/* * * (C) 2013-25 - ntop.org * * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software Foundation, * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * */ #include "ntop_includes.h" #ifdef __APPLE__ #include #endif /* Lua.cpp */ extern int ntop_lua_cli_print(lua_State *vm); extern int ntop_lua_check(lua_State *vm, const char *func, int pos, int expected_type); static bool help_printed = false; // #define DEBUG_FLOW_CHECKS 1 #define IMPLEMENT_SMART_FRAGMENTS /* **************************************************** */ /* Method used for collateral activities */ NetworkInterface::NetworkInterface() : NetworkInterfaceAlertableEntity(this, alert_entity_interface) { if(trace_new_delete) ntop->getTrace()->traceEvent(TRACE_NORMAL, "[new] %s", __FILE__); init(NULL); } /* **************************************************** */ NetworkInterface::NetworkInterface(const char *name, const char *custom_interface_type) : NetworkInterfaceAlertableEntity(this, alert_entity_interface) { char _ifname[MAX_INTERFACE_NAME_LEN], buf[MAX_INTERFACE_NAME_LEN]; /* We need to do it as isView() is not yet initialized */ char pcap_error_buffer[PCAP_ERRBUF_SIZE]; char *alias = NULL; if(trace_new_delete) ntop->getTrace()->traceEvent(TRACE_NORMAL, "[new] %s", __FILE__); if (name == NULL) { if (!help_printed) ntop->getTrace()->traceEvent(TRACE_WARNING, "No capture interface specified"); printAvailableInterfaces(false, 0, NULL, 0); name = Utils::ntop_lookupdev(_ifname, sizeof(_ifname)); if (name == NULL) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to locate default interface (%s)\n", pcap_error_buffer); exit(0); } } else { if (isNumber(name)) { /* We need to convert this numeric index into an interface name */ int ifid = atoi(name); _ifname[0] = '\0'; printAvailableInterfaces(false, ifid, _ifname, sizeof(_ifname)); if (_ifname[0] == '\0') { ntop->getTrace()->traceEvent(TRACE_WARNING, "Unable to locate interface Id %d", ifid); printAvailableInterfaces(false, 0, NULL, 0); exit(0); } name = _ifname; } } init(name); customIftype = custom_interface_type; influxdb_ts_exporter = rrd_ts_exporter = NULL; flow_checks_executor = prev_flow_checks_executor = NULL; host_checks_executor = prev_host_checks_executor = NULL; flows_dump_json = true; /* Too early will be set by NetworkInterface::startFlowDumping() */ memset(ifMac, 0, sizeof(ifMac)); #ifdef WIN32 if (name == NULL) name = "1"; /* First available interface */ #endif scalingFactor = 1; if (strcmp(name, "-") == 0) name = "stdin"; if (strcmp(name, "-") == 0) name = "stdin"; id = Utils::ifname2id(name); /* if interface alias, set as custom name (if not set already) */ if (ntop->getPrefs()) { alias = ntop->getPrefs()->get_if_alias(get_id()); if (alias) setCustomName(alias, true); } purge_idle_flows_hosts = true; if (custom_interface_type) ifDescription = strdup(name); else ifDescription = strdup(Utils::getInterfaceDescription(ifname, buf, sizeof(buf))); if (strchr(name, ':') || strchr(name, '@') || (!strcmp(name, "dummy")) || strchr(name, '/') /* file path */ || strstr(name, ".pcap") /* pcap */ || (strncmp(name, "lo", 2) == 0) || (strcmp(name, SYSTEM_INTERFACE_NAME) == 0) #if !defined(__APPLE__) && !defined(WIN32) || (Utils::readIPv4((char *)name) == 0) #endif || custom_interface_type) { ; /* Don't setup MDNS on ZC or RSS interfaces */ } else { ipv4_network = ipv4_network_mask = 0; if (pcap_lookupnet(ifname, &ipv4_network, &ipv4_network_mask, pcap_error_buffer) == -1) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Unable to read IPv4 address of %s: %s", ifname, pcap_error_buffer); } else { if(ntop->getPrefs()->limitResourcesUsage()) { discovery = NULL, mdns = NULL; } else { try { discovery = new NetworkDiscovery(this); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Network Discovery initialized"); } catch (...) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Failure initializing Network Discovery"); discovery = NULL; } if (discovery) { try { mdns = new MDNS(this); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "MDNS initialized"); } catch (...) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Failure initializing MDNS"); mdns = NULL; } } } } } #if defined(NTOPNG_PRO) pMap = NULL, sMap = NULL; acl_flow = NULL; #endif if (id >= 0) { last_pkt_rcvd = last_pkt_rcvd_remote = 0, pollLoopCreated = false, flowDumpLoopCreated = false, hostAlertsDequeueLoopCreated = flowAlertsDequeueLoopCreated = false, bridge_interface = false; next_idle_flow_purge = next_idle_host_purge = next_idle_other_purge = 0; cpu_affinity = -1 /* no affinity */, has_vlan_packets = has_ebpf_events = false; running = false, inline_interface = false; checkIdle(); ifSpeed = Utils::getMaxIfSpeed(name); ifMTU = Utils::getIfMTU(name), mtuWarningShown = false; reloadDhcpRanges(); } else /* id < 0 */ { ifSpeed = 0; } networkStats = NULL; #ifdef NTOPNG_PRO policer = NULL; /* possibly instantiated by subclass PacketBridge */ #ifndef HAVE_NEDGE flow_profiles = ntop->getPro()->has_valid_license() ? new (std::nothrow) FlowProfiles(id) : NULL; if (flow_profiles) flow_profiles->loadProfiles(); shadow_flow_profiles = NULL; #endif /* Lazy, instantiated on demand */ custom_app_stats = NULL; flow_interfaces_stats = NULL; #endif loadScalingFactorPrefs(); statsManager = NULL, alertStore = NULL, alertsQueue = NULL; ndpiStats = NULL; dscpStats = NULL; host_pools = new (std::nothrow) HostPools(this); bcast_domains = new (std::nothrow) BroadcastDomains(this); #ifdef __linux__ /* A bit aggressive but as people usually ignore warnings let's be proactive */ if (ifname && (!isView()) && (!strstr(ifname, ":")) && (!strstr(ifname, ".pcap")) && strcmp(ifname, "dummy") && strcmp(ifname, "any") && strcmp(ifname, "virbr") && strcmp(ifname, "wlan") && strncmp(ifname, "lo", 2) && strcmp(ifname, SYSTEM_INTERFACE_NAME)) { char ifaces[MAX_INTERFACE_NAME_LEN], *tmp, *iface; snprintf(ifaces, sizeof(ifaces), "%s", ifname); iface = strtok_r(ifaces, ",", &tmp); while (iface != NULL) { Utils::disableOffloads(iface); iface = strtok_r(NULL, ",", &tmp); } } #endif is_loopback = ((strcmp(ifname, "lo") == 0) || (strcmp(ifname, "lo0") == 0)); updateTrafficMirrored(); updateSmartRecording(); updateDynIfaceTrafficPolicy(); updateFlowDumpDisabled(); updateLbdIdentifier(); updateFlowsOnlyInterface(); updatePushFiltersSettings(); } /* **************************************************** */ void NetworkInterface::init(const char *interface_name) { ifname = interface_name ? strdup(interface_name) : NULL; inline_interface = false, has_vlan_packets = false, has_ebpf_events = false, has_seen_dhcp_addresses = false, has_seen_containers = false, has_seen_pods = false, has_external_alerts = false, last_pkt_rcvd = last_pkt_rcvd_remote = 0, next_idle_flow_purge = next_idle_host_purge = 0, running = false, shutting_down = false, customIftype = NULL, is_loopback = is_traffic_mirrored = false, lbd_serialize_by_mac = false, is_smart_recording_enabled = false; smart_recording_instance_name = NULL; flows_only_interface = false; numSubInterfaces = 0; ip_reassignment_alerts_enabled = false; pcap_datalink_type = DLT_EN10MB; /* default */ mtuWarningShown = false; purge_idle_flows_hosts = true; id = (u_int8_t)-1; last_remote_pps = 0,last_remote_bps = 0, last_remote_update = 0, has_vlan_packets = false, cpu_affinity = -1 /* no affinity */, interfaceStats = NULL, has_too_many_hosts = has_too_many_flows = false, flow_dump_disabled_by_user = false, flow_dump_disabled_by_backend = false, numL2Devices = 0, totalNumHosts = numTotalRxOnlyHosts = numLocalHosts = numLocalRxOnlyHosts = 0, arp_requests = arp_replies = 0, has_mac_addresses = false, checkpointPktCount = checkpointBytesCount = checkpointPktDropCount = checkpointDroppedAlertsCount = 0, checkpointTrafficSent = 0, checkpointTrafficRcvd = 0, checkpointPacketsSent = 0, checkpointPacketsRcvd = 0, pollLoopCreated = false, bridge_interface = false, mdns = NULL, discovery = NULL, ifDescription = NULL, flowHashingMode = flowhashing_none; num_new_flows = 0; last_obs_point_id = 0; flows_hash = NULL, hosts_hash = NULL; macs_hash = NULL, ases_hash = NULL, vlans_hash = NULL, obs_hash = NULL; countries_hash = NULL; gw_macs = NULL; is_dynamic_interface = false; dynamic_interface_master = NULL; dynamic_interface_criteria = 0; dynamic_interface_mode = flowhashing_none; show_dynamic_interface_traffic = false; push_host_filters = false; top_sites = NULL; reload_hosts_bcast_domain = false; hosts_bcast_domain_last_update = 0; ip_addresses = ""; networkStats = NULL; cpu_affinity = -1; gettimeofday(&last_periodic_stats_update, NULL); num_live_captures = 0; num_host_dropped_alerts = num_flow_dropped_alerts = num_other_dropped_alerts = 0; num_written_alerts = num_alerts_queries = 0; score_as_cli = score_as_srv = 0; memset(live_captures, 0, sizeof(live_captures)); memset(&num_alerts_engaged_notice, 0, sizeof(num_alerts_engaged_notice)), memset(&num_alerts_engaged_warning, 0, sizeof(num_alerts_engaged_warning)), memset(&num_alerts_engaged_error, 0, sizeof(num_alerts_engaged_error)), memset(&num_alerts_engaged_critical, 0, sizeof(num_alerts_engaged_critical)), memset(&num_alerts_engaged_emergency, 0, sizeof(num_alerts_engaged_emergency)); tot_num_anomalies.local_hosts = tot_num_anomalies.remote_hosts = 0; num_active_alerted_flows_notice = 0, num_active_alerted_flows_warning = 0, num_active_alerted_flows_error = 0; num_active_probes = 0; is_view = false; #ifdef NTOPNG_PRO viewed_by = NULL; #endif viewed_interface_id = 0; download_stats = upload_stats = NULL; db = NULL; #ifdef NTOPNG_PRO custom_app_stats = NULL; flow_interfaces_stats = NULL; policer = NULL; /* Behavior init variables */ next5MinPeriodicUpdate = nextMinPeriodicUpdate = 0; score_behavior = new BehaviorAnalysis(); traffic_tx_behavior = new BehaviorAnalysis(0.9 /* Alpha parameter */, 0.1 /* Beta parameter */, 0.05 /* Significance */, true /* Counter */, IFACE_BEHAVIOR_REFRESH); traffic_rx_behavior = new BehaviorAnalysis(0.9 /* Alpha parameter */, 0.1 /* Beta parameter */, 0.05 /* Significance */, true /* Counter */, IFACE_BEHAVIOR_REFRESH); #endif ndpiStats = NULL; dscpStats = NULL; statsManager = NULL, alertStore = NULL, ifSpeed = 0; host_pools = NULL; bcast_domains = NULL; ifMTU = CONST_DEFAULT_MAX_PACKET_SIZE, mtuWarningShown = false; #ifdef NTOPNG_PRO #ifndef HAVE_NEDGE flow_profiles = shadow_flow_profiles = NULL; sub_interfaces = NULL; #endif #endif #ifdef NTOPNG_PRO dhcp_last_sec_pkts = 0, last_sec_epoch = 0; #endif dhcp_ranges = dhcp_ranges_shadow = NULL; if (bridge_interface || is_dynamic_interface || isView()) ; else companionQueue = new (std::nothrow) ParsedFlow *[COMPANION_QUEUE_LEN](); next_compq_insert_idx = next_compq_remove_idx = 0; last_purge_idle = 0; idleFlowsToDump = activeFlowsToDump = NULL; flowAlertsQueue = new (std::nothrow) SPSCQueue(MAX_FLOW_CHECKS_QUEUE_LEN, "flowAlertsQueue"); hostAlertsQueue = new (std::nothrow) SPSCQueue(MAX_HOST_CHECKS_QUEUE_LEN, "hostAlertsQueue"); flow_serial = 0; alert_serial = 1; /* first assigned rowid is 1 */ /* nDPI handling */ ndpi_cleanup_needed = false; last_ndpi_reload = 0; ndpiReloadInProgress = false; ndpi_struct_shadow = NULL; ndpi_struct = initnDPIStruct(); ndpi_finalize_initialization(ndpi_struct); #if defined(HAVE_KAFKA) && defined(NTOPNG_PRO) kafka = NULL; #endif customFlowLuaScript_proto = customFlowLuaScript_periodic = customFlowLuaScript_end = NULL; customHostLuaScript = NULL; INTERFACE_PROFILING_INIT(); } /* **************************************************** */ void NetworkInterface::cleanShadownDPI() { if(ndpi_struct_shadow != NULL) { ntop->getTrace()->traceEvent(TRACE_INFO, "%s(%p)", __FUNCTION__, ndpi_struct_shadow); ndpi_exit_detection_module(ndpi_struct_shadow); ndpi_struct_shadow = NULL; } } /* ******************** */ u_int16_t NetworkInterface::getnDPIProtoByName(const char *name) { return (ndpi_get_proto_by_name(get_ndpi_struct(), name)); } /* ********************** */ struct ndpi_keys_struct { const char *proto, *key, *value; }; struct ndpi_detection_module_struct *NetworkInterface::initnDPIStruct() { struct ndpi_detection_module_struct *ndpi_s = ndpi_init_detection_module(NULL); ndpi_port_range d_port[MAX_DEFAULT_PORTS]; NDPI_PROTOCOL_BITMASK all; ndpi_cfg_error rc; const char* dirs[] = { "/usr/share/ndpi/public_suffix_list.dat", "/usr/local/share/ndpi/public_suffix_list.dat", "../nDPI/lists/public_suffix_list.dat", NULL }; const struct ndpi_keys_struct ndpi_keys[] = { { NULL, "flow.track_payload", "1" }, { "tls", "metadata.ja4r_fingerprint", "1" }, { NULL, "packets_limit_per_flow", "64" }, { "stun", "monitoring", "1" }, { "stun", "max_packets_extra_dissection", "32" }, { NULL, NULL, NULL } }; if (ndpi_s == NULL) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to initialize nDPI"); exit(-1); } // enable all protocols NDPI_BITMASK_SET_ALL(all); ndpi_set_protocol_detection_bitmask2(ndpi_s, &all); for(int i=0; ndpi_keys[i].key != NULL; i++) { rc = ndpi_set_config(ndpi_s, ndpi_keys[i].proto, ndpi_keys[i].key, ndpi_keys[i].value); if (rc != NDPI_CFG_OK) ntop->getTrace()->traceEvent(TRACE_ERROR, "Error ndpi_set_config(%s/%s/%s): %d", ndpi_keys[i].proto ? ndpi_keys[i].proto : "NULL", ndpi_keys[i].key ? ndpi_keys[i].key : "NULL", ndpi_keys[i].value, rc); } if(ntop->getPrefs() && ntop->getPrefs()->is_dns_cache_enabled()) { rc = ndpi_set_config(ndpi_s, NULL, "dpi.address_cache_size", "512000"); if (rc != NDPI_CFG_OK) ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to enable the DNS cache: %d", rc); } if (ntop->getCustomnDPIProtos() != NULL) ndpi_load_protocols_file(ndpi_s, ntop->getCustomnDPIProtos()); /* Load public domain suffixes (part of the ndpi package) */ for(int i=0; dirs[i] != NULL; i++) { struct stat buf; if(stat(dirs[i], &buf) == 0) { if(ndpi_load_domain_suffixes(ndpi_s, (char*)dirs[i]) == 0) ntop->getTrace()->traceEvent(TRACE_INFO, "Successfully loaded %s", dirs[i]); else ntop->getTrace()->traceEvent(TRACE_WARNING, "Error while loading %s", dirs[i]); break; } } memset(d_port, 0, sizeof(d_port)); ndpi_set_proto_defaults(ndpi_s, 0, 0, NDPI_PROTOCOL_UNRATED, NTOPNG_NDPI_OS_PROTO_ID, (char *)"Operating System", NDPI_PROTOCOL_CATEGORY_SYSTEM_OS, NDPI_PROTOCOL_QOE_CATEGORY_UNSPECIFIED, d_port, d_port); // load custom protocols loadProtocolsAssociations(ndpi_s); return (ndpi_s); } /* **************************************************** */ /* Operations are performed in the following order: * * 1. initnDPIReload() * 2. ... nDPILoadIPCategory/nDPILoadHostnameCategory() ... * 3. finalizenDPIReload() * 4. cleanShadownDPI() */ bool NetworkInterface::initnDPIReload() { ntop->getTrace()->traceEvent(TRACE_INFO, "Started nDPI reload %s", ndpiReloadInProgress ? "[IN PROGRESS]" : ""); if (ndpiReloadInProgress) { /* Do not display this alert for subinterfaces as they might have been created on the fly and thus trigger this alert */ if (!isSubInterface()) ntop->getTrace()->traceEvent(TRACE_ERROR, "Internal error: nested nDPI category reload"); return (false); } ndpiReloadInProgress = true; cleanShadownDPI(); /* No need to dedicate another variable for the reload, we can use the shadow * itself */ //ntop->getTrace()->traceEvent(TRACE_NORMAL, "nDPI reload started"); ndpi_struct_shadow = initnDPIStruct(); return (true); } /* **************************************************** */ void NetworkInterface::finalizenDPIReload() { ntop->getTrace()->traceEvent(TRACE_INFO, "%s(%p)", __FUNCTION__, ndpi_struct_shadow); if (!ndpiReloadInProgress) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Internal error: nested nDPI category reload"); return; } if (ndpi_struct_shadow) { struct ndpi_detection_module_struct *old_struct; int rc; ntop->getTrace()->traceEvent(TRACE_INFO, "Going to reload custom categories"); /* The new categories were loaded on the current ndpi_struct_shadow */ ndpi_enable_loaded_categories(ndpi_struct_shadow); rc = ndpi_finalize_initialization(ndpi_struct_shadow); if (rc != 0) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Error ndpi_finalize_initialization: %d", rc); } ntop->getTrace()->traceEvent(TRACE_INFO, "nDPI finalizing reload..."); old_struct = ndpi_struct; ndpi_struct = ndpi_struct_shadow; ndpi_struct_shadow = old_struct; /* Update hosts policies based on nDPi configuration */ reloadHostsBlacklist(); ntop->getTrace()->traceEvent(TRACE_DEBUG, "nDPI reload completed"); ndpiReloadInProgress = false; } } /* ******************************************* */ void NetworkInterface::loadProtocolsAssociations(struct ndpi_detection_module_struct *ndpi_str) { char **keys, **values; Redis *redis = ntop->getRedis(); int rc; if (!redis) return; rc = redis->hashGetAll(CUSTOM_NDPI_PROTOCOLS_ASSOCIATIONS_HASH, &keys, &values); if (rc > 0) { for (int i = 0; i < rc; i++) { if (keys[i] && values[i]) { u_int16_t protoId = atoi(keys[i]); ndpi_protocol_category_t protoCategory = (ndpi_protocol_category_t)atoi(values[i]); setnDPIProtocolCategory(ndpi_str, protoId, protoCategory); } if (values[i]) free(values[i]); if (keys[i]) free(keys[i]); } free(keys); free(values); } } /* *************************************** */ bool NetworkInterface::nDPILoadIPCategory(char *what, u_int16_t id, char *list_name) { bool success = true; // ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s [%s][%s][id: %u]", __FUNCTION__, what, list_name, id); if (what && ndpi_struct_shadow) success = (ndpi_load_ip_category(ndpi_struct_shadow, what, (ndpi_protocol_category_t)id, (void *)list_name) == 0); return success; } /* *************************************** */ bool NetworkInterface::nDPILoadHostnameCategory(char *what, u_int16_t id, char *list_name) { bool success = true; // ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s(%p) [%s]", __FUNCTION__, ndpi_struct_shadow, what); if (what && ndpi_struct_shadow) success = (ndpi_load_hostname_category(ndpi_struct_shadow, what, (ndpi_protocol_category_t)id) == 0); else ntop->getTrace()->traceEvent(TRACE_WARNING, "Internal error: invalid nDPI state"); return success; } /* *************************************** */ int NetworkInterface::setDomainMask(const char *domain, u_int64_t domain_mask) { // ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s(%p) [%s]", __FUNCTION__, ndpi_struct_shadow, domain); if(domain && ndpi_struct_shadow) return(ndpi_add_host_risk_mask(ndpi_struct_shadow, (char*)domain, domain_mask)); else ntop->getTrace()->traceEvent(TRACE_WARNING, "Internal error: invalid nDPI state"); return(-1); } /* *************************************** */ int NetworkInterface::addTrustedIssuerDN(const char *dn) { if(dn && ndpi_struct_shadow) return(ndpi_add_trusted_issuer_dn(ndpi_struct_shadow, (char*)dn)); return(-1); } /* *************************************** */ ndpi_protocol_category_t NetworkInterface::get_ndpi_proto_category(u_int16_t protoid) { ndpi_protocol proto; protoid = ndpi_map_user_proto_id_to_ndpi_id(get_ndpi_struct(), protoid); proto.proto.app_protocol = NDPI_PROTOCOL_UNKNOWN; proto.proto.master_protocol = protoid; proto.category = NDPI_PROTOCOL_CATEGORY_UNSPECIFIED; return (get_ndpi_proto_category(proto)); } /* *************************************** */ void NetworkInterface::setnDPIProtocolCategory(struct ndpi_detection_module_struct *ndpi_str, u_int16_t protoId, ndpi_protocol_category_t protoCategory) { u_int16_t mappedProtoId = ndpi_map_user_proto_id_to_ndpi_id(ndpi_str, protoId); if(mappedProtoId == 0) ntop->getTrace()->traceEvent(TRACE_INFO, "Internal error: ID %d (mapped to %u)", protoId, mappedProtoId); else { ntop->getTrace()->traceEvent(TRACE_INFO, "Setting protocol association: ID %d (mapped to %u) -> category %d", protoId, mappedProtoId, protoCategory); ndpi_set_proto_category(ndpi_str, mappedProtoId, protoCategory); } } /* **************************************************** */ #ifdef NTOPNG_PRO void NetworkInterface::initL7Policer() { /* Instantiate the policer */ policer = new (std::nothrow) L7Policer(this); } #endif /* **************************************************** */ void NetworkInterface::checkDisaggregationMode() { char rkey[128], rsp[64]; if (customIftype || isSubInterface()) return; snprintf(rkey, sizeof(rkey), CONST_IFACE_DYN_IFACE_MODE_PREFS, id); if ((!ntop->getRedis()->get(rkey, rsp, sizeof(rsp))) && (rsp[0] != '\0')) { if (getIfType() == interface_type_ZMQ) { /* ZMQ interface */ if (!strcmp(rsp, DISAGGREGATION_PROBE_IP)) flowHashingMode = flowhashing_probe_ip; else if (!strcmp(rsp, DISAGGREGATION_IFACE_ID)) flowHashingMode = flowhashing_iface_idx; else if (!strcmp(rsp, DISAGGREGATION_INGRESS_IFACE_ID)) flowHashingMode = flowhashing_ingress_iface_idx; else if (!strcmp(rsp, DISAGGREGATION_INGRESS_PROBE_IP_AND_IFACE_ID)) flowHashingMode = flowhashing_probe_ip_and_ingress_iface_idx; else if (!strcmp(rsp, DISAGGREGATION_INGRESS_VRF_ID)) flowHashingMode = flowhashing_vrfid; else if (!strcmp(rsp, DISAGGREGATION_VLAN)) flowHashingMode = flowhashing_vlan; else if (strcmp(rsp, DISAGGREGATION_NONE)) ntop->getTrace()->traceEvent( TRACE_ERROR, "Unknown aggregation value for interface %s [rsp: %s]", get_type(), rsp); } else { /* non-ZMQ interface */ if (!strcmp(rsp, DISAGGREGATION_VLAN)) flowHashingMode = flowhashing_vlan; else if (strcmp(rsp, DISAGGREGATION_NONE)) ntop->getTrace()->traceEvent( TRACE_ERROR, "Unknown aggregation value for interface %s [rsp: %s]", get_type(), rsp); } } /* Populate ignored interfaces */ rsp[0] = '\0'; if ((!ntop->getRedis()->get((char *)CONST_RUNTIME_PREFS_IGNORED_INTERFACES, rsp, sizeof(rsp))) && (rsp[0] != '\0')) { char *token; char *rest = rsp; while ((token = strtok_r(rest, ",", &rest))) flowHashingIgnoredInterfaces.insert(atoi(token)); } #ifdef NTOPNG_PRO #ifndef HAVE_NEDGE sub_interfaces = ntop->getPrefs()->is_enterprise_m_edition() ? new (std::nothrow) SubInterfaces(this) : NULL; #endif #endif } /* **************************************************** */ void NetworkInterface::loadScalingFactorPrefs() { if (ntop->getRedis() != NULL) { char rkey[128], rsp[16]; u_int32_t scaling_factor = 0; snprintf(rkey, sizeof(rkey), CONST_IFACE_SCALING_FACTOR_PREFS, id); if ((ntop->getRedis()->get(rkey, rsp, sizeof(rsp)) == 0) && (rsp[0] != '\0')) scaling_factor = atol(rsp); if (scaling_factor > 0) setScalingFactor(scaling_factor); } } /* **************************************************** */ /* Whether the initialization procedure is completed for this interface */ bool NetworkInterface::isStartingUp() const { return !running && !ntop->getGlobals()->isShutdownRequested() && !ntop->getGlobals()->isShutdown(); } /* **************************************************** */ /* Whether the interface is running and ntopng is not shutting down */ bool NetworkInterface::isRunning() const { return running && !ntop->getGlobals()->isShutdownRequested() && !ntop->getGlobals()->isShutdown(); } /* **************************************************** */ /* Whether the interface is running and shutting down (flows/hosts purge already * completed) */ bool NetworkInterface::isShuttingDown() const { return !running && shutting_down; } /* **************************************************** */ void NetworkInterface::setCustomName(char *alias, bool set_on_empty) { char pref_buf[CONST_MAX_LEN_REDIS_KEY]; char rsp[MAX_INTERFACE_NAME_LEN]; if (ntop->getRedis()) { snprintf(pref_buf, sizeof(pref_buf), CONST_INTERFACE_CUSTOM_NAME, get_id()); if (!set_on_empty || ntop->getRedis()->get(pref_buf, rsp, sizeof(rsp)) != 0) ntop->getRedis()->set(pref_buf, alias); } } /* **************************************************** */ bool NetworkInterface::getInterfaceBooleanPref(const char *pref_key, bool default_pref_value) const { char pref_buf[CONST_MAX_LEN_REDIS_KEY], rsp[2] = {0}; bool interface_pref = default_pref_value; if (ntop->getRedis()) { snprintf(pref_buf, sizeof(pref_buf), pref_key, get_id()); if ((ntop->getRedis()->get(pref_buf, rsp, sizeof(rsp)) == 0) && (rsp[0] != '\0')) { if (rsp[0] == '1') interface_pref = true; else if (rsp[0] == '0') interface_pref = false; } } #if 0 ntop->getTrace()->traceEvent(TRACE_NORMAL, "Reading pref [%s][ifid: %i][rsp: %s][actual_value: %d]", pref_buf, get_id(), rsp, interface_pref ? 1 : 0); #endif return interface_pref; } /* **************************************************** */ void NetworkInterface::updateTrafficMirrored() { is_traffic_mirrored = getInterfaceBooleanPref(CONST_MIRRORED_TRAFFIC_PREFS, CONST_DEFAULT_MIRRORED_TRAFFIC); } /* **************************************************** */ void NetworkInterface::updateSmartRecording() { char redis_key[CONST_MAX_LEN_REDIS_KEY]; char redis_val[CONST_MAX_LEN_REDIS_VALUE]; char *instance = NULL, *prev_instance; bool enable = false; if (!ntop->getRedis()) return; enable = getInterfaceBooleanPref(CONST_SMART_RECORDING_PREFS, CONST_DEFAULT_SMART_RECORDING); if (enable) { snprintf(redis_key, sizeof(redis_key), CONST_SMART_RECORDING_INSTANCE_PREFS, get_id()); redis_val[0] = '\0'; if (ntop->getRedis()->get(redis_key, redis_val, CONST_MAX_LEN_REDIS_VALUE) == 0 && strlen(redis_val) > 0) { instance = strdup(redis_val); } else { enable = false; /* missing instance name (required) */ } } /* Update instance name */ prev_instance = smart_recording_instance_name; smart_recording_instance_name = instance; if (prev_instance) { usleep(100); free(prev_instance); } /* Toggle smart recording */ is_smart_recording_enabled = enable; } /* **************************************************** */ bool NetworkInterface::isSmartRecordingEnabled() const { return is_smart_recording_enabled && ntop->getPrefs()->is_enterprise_xl_edition(); } /* **************************************************** */ void NetworkInterface::updateDynIfaceTrafficPolicy() { show_dynamic_interface_traffic = getInterfaceBooleanPref(CONST_SHOW_DYN_IFACE_TRAFFIC_PREFS, CONST_DEFAULT_SHOW_DYN_IFACE_TRAFFIC); } /* **************************************************** */ void NetworkInterface::updatePushFiltersSettings() { push_host_filters = getInterfaceBooleanPref(CONST_PUSH_HOST_FILTERS_PREFS, CONST_DEFAULT_PUSH_HOST_FILTERS); } /* **************************************************** */ void NetworkInterface::updateFlowDumpDisabled() { flow_dump_disabled_by_user = getInterfaceBooleanPref(CONST_DISABLED_FLOW_DUMP_PREFS, false); } /* **************************************** */ void NetworkInterface::updateLbdIdentifier() { lbd_serialize_by_mac = getInterfaceBooleanPref(CONST_LBD_SERIALIZATION_PREFS, CONST_DEFAULT_LBD_SERIALIZE_AS_MAC); } /* **************************************** */ void NetworkInterface::updateFlowsOnlyInterface() { flows_only_interface = getInterfaceBooleanPref(CONST_FLOWS_ONLY_INTERFACE, CONST_DEFAULT_FLOWS_ONLY_INTERFACE); } /* **************************************************** */ bool NetworkInterface::checkIdle() { is_idle = false; if (ifname != NULL) { char rkey[128], rsp[16]; snprintf(rkey, sizeof(rkey), "ntopng.prefs.%s_not_idle", ifname); if ((ntop->getRedis()->get(rkey, rsp, sizeof(rsp)) == 0) && (rsp[0] != '\0')) { int val = atoi(rsp); if (val == 0) is_idle = true; } } return (is_idle); } /* **************************************************** */ void NetworkInterface::deleteDataStructures() { if (flows_hash) { delete (flows_hash); flows_hash = NULL; } if (hosts_hash) { delete (hosts_hash); hosts_hash = NULL; } if (ases_hash) { delete (ases_hash); ases_hash = NULL; } if (obs_hash) { delete (obs_hash); obs_hash = NULL; } if (countries_hash) { delete (countries_hash); countries_hash = NULL; } if (vlans_hash) { delete (vlans_hash); vlans_hash = NULL; } if (macs_hash) { delete (macs_hash); macs_hash = NULL; } if (gw_macs) { delete (gw_macs); gw_macs = NULL; } if (download_stats) { delete (download_stats); download_stats = NULL; } if (upload_stats) { delete (upload_stats); upload_stats = NULL; } if (companionQueue) { for (u_int16_t i = 0; i < COMPANION_QUEUE_LEN; i++) if (companionQueue[i]) delete companionQueue[i]; delete[] companionQueue; companionQueue = NULL; } } /* **************************************************** */ NetworkInterface::~NetworkInterface() { std::map,InterfaceMemberAlertableEntity *>::iterator it; std::map::iterator it_o; if(trace_new_delete) ntop->getTrace()->traceEvent(TRACE_NORMAL, "[delete] %s", __FILE__); #ifdef INTERFACE_PROFILING u_int64_t n = ethStats.getNumIngressPackets(); if (isPacketInterface() && n > 0) { for (u_int i = 0; i < INTERFACE_PROFILING_NUM_SECTIONS; i++) { if (INTERFACE_PROFILING_SECTION_LABEL(i) != NULL) ntop->getTrace()->traceEvent( TRACE_NORMAL, "[PROFILING] Section #%d '%s': AVG %llu ticks", i, INTERFACE_PROFILING_SECTION_LABEL(i), INTERFACE_PROFILING_SECTION_AVG(i, n)); } } #endif cleanup(); deleteDataStructures(); if (idleFlowsToDump) delete idleFlowsToDump; if (activeFlowsToDump) delete activeFlowsToDump; if (db) { db->shutdown(); delete db; } if (host_pools) delete host_pools; /* note: this requires ndpi_struct */ if (bcast_domains) delete bcast_domains; if (ifDescription) free(ifDescription); if (discovery) delete discovery; if (statsManager) delete statsManager; if (alertStore) delete alertStore; if (alertsQueue) delete alertsQueue; if (ndpiStats) delete ndpiStats; if (dscpStats) delete dscpStats; if (networkStats) { u_int32_t numNetworks = ntop->getNumLocalNetworks(); for (u_int32_t i = 0; i < numNetworks; i++) delete networkStats[i]; delete[] networkStats; } if (interfaceStats) delete interfaceStats; if (customFlowLuaScript_proto) delete customFlowLuaScript_proto; if (customFlowLuaScript_periodic) delete customFlowLuaScript_periodic; if (customFlowLuaScript_end) delete customFlowLuaScript_end; if (customHostLuaScript) delete customHostLuaScript; #if defined(NTOPNG_PRO) if (pMap) delete pMap; if (sMap) delete sMap; if (acl_flow) delete acl_flow; if (score_behavior) delete (score_behavior); if (traffic_tx_behavior) delete (traffic_tx_behavior); if (traffic_rx_behavior) delete (traffic_rx_behavior); #endif for (it = external_alerts.begin(); it != external_alerts.end(); ++it) delete it->second; external_alerts.clear(); #ifdef NTOPNG_PRO if (policer) delete (policer); #ifndef HAVE_NEDGE if (flow_profiles) delete (flow_profiles); if (shadow_flow_profiles) delete (shadow_flow_profiles); if (sub_interfaces) delete (sub_interfaces); #endif if (custom_app_stats) delete custom_app_stats; if (flow_interfaces_stats) delete flow_interfaces_stats; #endif if (influxdb_ts_exporter) delete influxdb_ts_exporter; if (rrd_ts_exporter) delete rrd_ts_exporter; if (dhcp_ranges) delete[] dhcp_ranges; if (dhcp_ranges_shadow) delete[] dhcp_ranges_shadow; if (mdns) delete mdns; /* Leave it at the end so the mdns resolver has time to initialize */ if (ifname) free(ifname); if (flowAlertsQueue) delete flowAlertsQueue; if (hostAlertsQueue) delete hostAlertsQueue; addRedisSitesKey(); if (top_sites) delete top_sites; if (prev_flow_checks_executor) delete prev_flow_checks_executor; if (flow_checks_executor) delete flow_checks_executor; if (prev_host_checks_executor) delete prev_host_checks_executor; if (host_checks_executor) delete host_checks_executor; /* Note do not need to delete kafka as it's shared with db */ if (ndpi_struct) { ndpi_exit_detection_module(ndpi_struct); ndpi_struct = NULL; } cleanShadownDPI(); if (smart_recording_instance_name) free(smart_recording_instance_name); } /* **************************************************** */ /* Enqueue flow alert to a queue for processing and later delivery to recipients */ bool NetworkInterface::enqueueFlowAlert(FlowAlert *alert) { Flow *f = alert->getFlow(); bool ret = false; /* Perform the actual enqueue */ if (flowAlertsQueue && flowAlertsQueue->enqueue(alert, true)) { /* If enqueue was successful, increase the flow reference counter. Reference counter will be deleted when doing the dequeue. */ f->incUses(); /* Signal the waiter on the condition variable */ flow_checks_condvar.signal(); ret = true; } else { delete alert; } return ret; } /* **************************************************** */ /* Enqueue host alert to a queue for processing and later delivery to recipients */ bool NetworkInterface::enqueueHostAlert(HostAlert *alert) { HostAlertReleasedPair alert_info(alert, alert->isReleased()); Host *h = alert->getHost(); bool ret = false; if (!ntop->getPrefs()->dontEmitHostAlerts() && hostAlertsQueue && hostAlertsQueue->enqueue(alert_info, true)) { /* If enqueue was successful, increase the host reference counter. Reference counter will be deleted when doing the dequeue. */ h->incUses(); /* Signal the waiter on the condition variable */ host_checks_condvar.signal(); ret = true; } else { if (alert->isReleased()) delete alert; } return ret; } /* **************************************************** */ int NetworkInterface::dumpFlow(time_t when, Flow *f) { int rc = -1; /* Asynchronous dump via a thread */ if (f->get_state() == hash_entry_state_idle) { /* Last flow dump before delete * Note: this never happens in 'direct' mode */ if (idleFlowsToDump && idleFlowsToDump->enqueue(f, true)) { f->incUses(), f->set_dump_in_progress(); /* Signal there's work to do. Don't signal for view interfaces, they use sleep. */ #ifndef WIN32 if (!isViewed()) dump_condition.signal(); #endif #if DEBUG_FLOW_DUMP ntop->getTrace()->traceEvent(TRACE_NORMAL, "[%s] Queueing flow to dump [IDLE]", __FUNCTION__); #endif } else { incNumQueueDroppedFlows(1); } } else { /* Partial dump if active flows */ if (activeFlowsToDump && activeFlowsToDump->enqueue(f, true)) { f->incUses(), f->set_dump_in_progress(); /* Signal there's work to do. */ #ifndef WIN32 if (!isViewed()) dump_condition.signal(); #endif #if DEBUG_FLOW_DUMP ntop->getTrace()->traceEvent(TRACE_NORMAL, "[%s] Queueing flow to dump [ACTIVE]", __FUNCTION__); #endif } else { incNumQueueDroppedFlows(1); } } return (rc); } /* **************************************************** */ #ifdef NTOPNG_PRO void NetworkInterface::flushFlowDump() { if (db) db->flush(); } #endif /* **************************************************** */ static bool local_hosts_2_redis_walker(GenericHashEntry *h, void *user_data, bool *matched) { Host *host = (Host *)h; if (host && (host->isLocalHost() || host->isSystemHost())) *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ u_int32_t NetworkInterface::getHostsHashSize() { return (hosts_hash ? hosts_hash->getNumEntries() : 0); } /* **************************************************** */ u_int32_t NetworkInterface::getASesHashSize() { return (ases_hash ? ases_hash->getNumEntries() : 0); } /* **************************************************** */ u_int32_t NetworkInterface::getObsHashSize() { return (obs_hash ? obs_hash->getNumEntries() : 0); } /* **************************************************** */ u_int32_t NetworkInterface::getCountriesHashSize() { return (countries_hash ? countries_hash->getNumEntries() : 0); } /* **************************************************** */ u_int32_t NetworkInterface::getVLANsHashSize() { return (vlans_hash ? vlans_hash->getNumEntries() : 0); } /* **************************************************** */ u_int32_t NetworkInterface::getFlowsHashSize() { return (flows_hash ? flows_hash->getNumEntries() : 0); } /* **************************************************** */ u_int32_t NetworkInterface::getMacsHashSize() { return (macs_hash ? macs_hash->getNumEntries() : 0); } /* **************************************************** */ bool NetworkInterface::walker(u_int32_t *begin_slot, bool walk_all, WalkerType wtype, bool (*walker)(GenericHashEntry *h, void *user_data, bool *matched), void *user_data) { bool ret = false; if (id == SYSTEM_INTERFACE_ID) return (false); switch (wtype) { case walker_hosts: ret = hosts_hash ? hosts_hash->walk(begin_slot, walk_all, walker, user_data) : false; break; case walker_flows: ret = flows_hash ? flows_hash->walk(begin_slot, walk_all, walker, user_data) : false; break; case walker_macs: ret = macs_hash ? macs_hash->walk(begin_slot, walk_all, walker, user_data) : false; break; case walker_ases: ret = ases_hash ? ases_hash->walk(begin_slot, walk_all, walker, user_data) : false; break; case walker_obs: ret = obs_hash ? obs_hash->walk(begin_slot, walk_all, walker, user_data) : false; break; case walker_countries: ret = countries_hash ? countries_hash->walk(begin_slot, walk_all, walker, user_data) : false; break; case walker_vlans: ret = vlans_hash ? vlans_hash->walk(begin_slot, walk_all, walker, user_data) : false; break; } return (ret); } /* **************************************************** */ Flow *NetworkInterface::getFlow(int32_t if_index, Mac *src_mac, Mac *dst_mac, u_int16_t vlan_id, u_int16_t observation_domain_id, u_int32_t private_flow_id, u_int32_t deviceIP, u_int32_t inIndex, u_int32_t outIndex, const ICMPinfo *const icmp_info, IpAddress *src_ip, IpAddress *dst_ip, u_int16_t src_port, u_int16_t dst_port, u_int8_t l4_proto, bool *src2dst_direction, time_t first_seen, time_t last_seen, u_int32_t len_on_wire, bool *new_flow, bool create_if_missing, u_int8_t *view_cli_mac, u_int8_t *view_srv_mac) { Flow *ret, *unswapped_flow; Mac *primary_mac; Host *srcHost = NULL, *dstHost = NULL; if (!flows_hash) return (NULL); if (vlan_id != 0) setSeenVLANTaggedPackets(); if (!hasSeenMacAddresses()) { if ((src_mac && Utils::macHash(src_mac->get_mac()) != 0) || (dst_mac && Utils::macHash(dst_mac->get_mac()) != 0)) setSeenMacAddresses(); } INTERFACE_PROFILING_SECTION_ENTER("NetworkInterface::getFlow: flows_hash->find", 1); ret = flows_hash->find(src_mac, dst_mac, src_ip, dst_ip, src_port, dst_port, vlan_id, observation_domain_id, private_flow_id, l4_proto, icmp_info, src2dst_direction, true /* Inline call */, &unswapped_flow); INTERFACE_PROFILING_SECTION_EXIT(1); if ((ret == NULL) && (unswapped_flow != NULL)) { /* We have found this flow but with the wrong direction and we're waiting it to be swapped. */ ret = unswapped_flow; /* 1 - Use the new flow */ ret->swap(); /* 2 - Swap flow keys */ *src2dst_direction = ((ntohs(src_port) == ret->get_cli_port()) && (ntohs(dst_port) == ret->get_srv_port())); } if (ret == NULL) { if (!create_if_missing) return (NULL); *new_flow = true; num_new_flows++; if (!flows_hash->hasEmptyRoom()) { // ntop->getTrace()->traceEvent(TRACE_WARNING, "Too many flows"); has_too_many_flows = true; return (NULL); } try { INTERFACE_PROFILING_SECTION_ENTER("NetworkInterface::getFlow: new Flow", 2); ret = new Flow(this, if_index, vlan_id, observation_domain_id, private_flow_id, l4_proto, src_mac, src_ip, src_port, dst_mac, dst_ip, dst_port, icmp_info, first_seen, last_seen, view_cli_mac, view_srv_mac); INTERFACE_PROFILING_SECTION_EXIT(2); } catch (std::bad_alloc &ba) { static bool oom_warning_sent = false; if (!oom_warning_sent) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Not enough memory"); oom_warning_sent = true; } has_too_many_flows = true; return (NULL); } if (flows_hash->add(ret, false /* Don't lock, we're inline with the purgeIdle */)) { *src2dst_direction = true; } else { /* Note: this should never happen as we are checking hasEmptyRoom() */ delete ret; // ntop->getTrace()->traceEvent(TRACE_WARNING, "Too many flows"); has_too_many_flows = true; return (NULL); } } else { *new_flow = false; has_too_many_flows = false; } if (src_mac) { if ((srcHost = (*src2dst_direction) ? ret->get_cli_host() : ret->get_srv_host())) { if ((!src_mac->isSpecialMac()) && (primary_mac = srcHost->getMac()) && (primary_mac != src_mac) /* Mac address has changed */) { #ifdef MAC_DEBUG char buf[32], bufm1[32], bufm2[32]; ntop->getTrace()->traceEvent(TRACE_NORMAL, "Detected mac address [new MAC: %s] [old host: %s/primary mac: " "%s][pkts: %u]", Utils::formatMac(src_mac->get_mac(), bufm1, sizeof(bufm1)), srcHost->get_ip()->print(buf, sizeof(buf)), Utils::formatMac(primary_mac->get_mac(), bufm2, sizeof(bufm2)), getNumPackets()); #endif if (srcHost->getMac()->isSpecialMac()) { if (getIfType() == interface_type_NETFILTER) { /* This is the first *reply* packet of a flow so we need to increment it with the initial packet that was missed as NetFilter did not report the (destination) MAC. From now on, all flow peers are known */ /* NOTE: in nEdge, stats are updated into Flow::update_hosts_stats */ #ifndef HAVE_NEDGE if (ret->get_packets_cli2srv() == 1 /* first packet */) src_mac->incRcvdStats(getTimeLastPktRcvd(), 1, ret->get_bytes_cli2srv() /* size of the last packet */); #endif } } #ifdef HANDLE_MAC_SWAP srcHost->set_mac(src_mac); /* We're changing host's special mac with a non-special one */ /* incNumHosts(srcHost, srcHost->isRxOnlyHost()); */ srcHost->updateHostPool(true /* Inline */); #endif } } } if (dst_mac) { if ((dstHost = (*src2dst_direction) ? ret->get_srv_host() : ret->get_cli_host())) { if ((!dst_mac->isSpecialMac()) && (primary_mac = dstHost->getMac()) && (primary_mac != dst_mac)) { #ifdef MAC_DEBUG char buf[32], bufm1[32], bufm2[32]; ntop->getTrace()->traceEvent(TRACE_NORMAL, "Detected mac address [new MAC: %s] [old host: %s/primary mac: %s]", Utils::formatMac(dst_mac->get_mac(), bufm1, sizeof(bufm1)), dstHost->get_ip()->print(buf, sizeof(buf)), Utils::formatMac(primary_mac->get_mac(), bufm2, sizeof(bufm2))); #endif #ifdef HANDLE_MAC_SWAP dstHost->set_mac(dst_mac); /* We're changing host's special mac with a non-special one */ /* incNumHosts(dstHost, dstHost->isRxOnlyHost()); */ dstHost->updateHostPool(true /* Inline */); #endif } } } return (ret); } /* **************************************************** */ void NetworkInterface::setSubInterface(NetworkInterface *master_iface, FlowHashingEnum mode, u_int64_t criteria) { dynamic_interface_mode = mode; dynamic_interface_criteria = criteria; dynamic_interface_master = master_iface; is_dynamic_interface = true; }; /* **************************************************** */ /* NOTE: the interface is deleted when this method returns false */ bool NetworkInterface::registerSubInterface(NetworkInterface *sub_iface, u_int64_t criteria) { /* registerInterface deletes the interface on failure */ if (!ntop->registerInterface(sub_iface)) return false; sub_iface->setSubInterface(this, flowHashingMode, criteria); /* allocateStructures must be called after registering the interface. * This is needed because StoreManager calles ntop->getInterfaceById. */ sub_iface->allocateStructures(); ntop->initInterface(sub_iface); sub_iface->startPacketPolling(); /* Won't actually start a thread, just mark this interface as running */ flowHashing[criteria] = sub_iface; /* Add it to the hash */ numSubInterfaces++; ntop->getRedis()->set(CONST_STR_RELOAD_LISTS, (const char *)"1"); return true; } /* **************************************************** */ NetworkInterface *NetworkInterface::getDynInterface(u_int64_t criteria, bool parser_interface) { NetworkInterface *sub_iface = NULL; #ifndef HAVE_NEDGE std::map::iterator subIface = flowHashing.find(criteria); char buf[64], buf1[48]; const char *vIface_type; if (subIface != flowHashing.end()) { sub_iface = subIface->second; return sub_iface; } /* Interface not found */ if ((numSubInterfaces >= MAX_NUM_VIRTUAL_INTERFACES) || (ntop->get_num_interfaces() >= MAX_NUM_DEFINED_INTERFACES)) { static bool too_many_interfaces_error = false; if (!too_many_interfaces_error) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Too many subinterfaces defined"); too_many_interfaces_error = true; } return (NULL); } switch (flowHashingMode) { case flowhashing_vlan: vIface_type = CONST_INTERFACE_TYPE_VLAN; snprintf(buf, sizeof(buf), "%s [VLAN Id: %u]", ifname, (unsigned int)criteria); break; case flowhashing_probe_ip: vIface_type = CONST_INTERFACE_TYPE_FLOW; snprintf(buf, sizeof(buf), "%s [Probe IP: %s]", ifname, Utils::intoaV4((unsigned int)criteria, buf1, sizeof(buf1))); break; case flowhashing_iface_idx: vIface_type = CONST_INTERFACE_TYPE_FLOW; snprintf(buf, sizeof(buf), "%s [IfIdx: %u]", ifname, (unsigned int)criteria); break; case flowhashing_ingress_iface_idx: vIface_type = CONST_INTERFACE_TYPE_FLOW; snprintf(buf, sizeof(buf), "%s [InIfIdx: %u]", ifname, (unsigned int)criteria); break; case flowhashing_probe_ip_and_ingress_iface_idx: { /* 64 bit value: upper 32 bit is nProbe IP, lower 32 bit ifIdx */ u_int32_t nprobe_ip = (u_int32_t)(criteria >> 32); u_int32_t if_id = (u_int32_t)(criteria & 0xFFFFFFFF); vIface_type = CONST_INTERFACE_TYPE_FLOW; snprintf(buf, sizeof(buf), "%s [Probe IP: %s][InIfIdx: %u]", ifname, Utils::intoaV4(nprobe_ip, buf1, sizeof(buf1)), if_id); } break; case flowhashing_vrfid: vIface_type = CONST_INTERFACE_TYPE_FLOW; snprintf(buf, sizeof(buf), "%s [VRF Id: %u]", ifname, (unsigned int)criteria); break; default: return (NULL); break; } #ifdef HAVE_ZMQ if (dynamic_cast(this)) sub_iface = new (std::nothrow) ZMQParserInterface(buf, vIface_type); else #endif if (dynamic_cast(this)) sub_iface = new (std::nothrow) SyslogParserInterface(buf, vIface_type); else sub_iface = new (std::nothrow) NetworkInterface(buf, vIface_type); if (sub_iface == NULL) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Failure allocating interface: not enough memory?"); return (NULL); } if (!this->registerSubInterface(sub_iface, criteria)) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Failure registering sub-interface"); sub_iface = NULL; /* NOTE: interface deleted by registerSubInterface */ return (NULL); } #endif return (sub_iface); } /* **************************************************** */ bool NetworkInterface::processPacket(int32_t if_index, u_int32_t bridge_iface_idx, int datalink_type, bool *ingressPacket, const struct bpf_timeval *when, const u_int64_t packet_time, struct ndpi_ethhdr *eth, u_int16_t vlan_id, struct ndpi_iphdr *iph, struct ndpi_ipv6hdr *ip6, u_int16_t ip_offset, u_int16_t encapsulation_overhead, u_int32_t len_on_wire, const struct pcap_pkthdr *h, const u_char *packet, u_int16_t *ndpiProtocol, Host **srcHost, Host **dstHost, Flow **hostFlow) { u_int16_t trusted_ip_len = max_val(0, (int)h->caplen - ip_offset); u_int16_t trusted_payload_len = 0; u_int32_t private_flow_id = 0; bool src2dst_direction; u_int8_t l4_proto; Flow *flow; Mac *srcMac = NULL, *dstMac = NULL; IpAddress src_ip, dst_ip; ICMPinfo icmp_info; u_int16_t frame_padding = 0; u_int16_t src_port = 0, dst_port = 0; struct ndpi_tcphdr *tcph = NULL; struct ndpi_udphdr *udph = NULL; struct sctphdr *sctph = NULL; u_int16_t trusted_l4_packet_len; u_int8_t *l4, tcp_flags = 0, *payload = NULL; u_int8_t *ip; bool is_fragment = false, new_flow; bool pass_verdict = true; u_int16_t l4_len = 0, fragment_offset = 0; #ifndef HAVE_NEDGE #ifdef IMPLEMENT_SMART_FRAGMENTS u_int16_t fragment_extra_overhead = 0; #endif #endif u_int8_t tos; #ifdef NTOPNG_PRO u_int8_t tcp_window_scale = 0; #endif *hostFlow = NULL; if (!isSubInterface()) { bool processed = false; #ifdef NTOPNG_PRO #ifndef HAVE_NEDGE /* Custom disaggregation */ if (sub_interfaces && (sub_interfaces->getNumSubInterfaces() > 0)) { processed = sub_interfaces->processPacket(if_index, bridge_iface_idx, datalink_type, *ingressPacket, when, packet_time, eth, vlan_id, iph, ip6, ip_offset, encapsulation_overhead, len_on_wire, h, packet, ndpiProtocol, srcHost, dstHost, hostFlow); } #endif #endif if ((!processed) && (flowHashingMode != flowhashing_none)) { /* VLAN disaggregation */ if ((flowHashingMode == flowhashing_vlan) && (vlan_id > 0)) { NetworkInterface *vIface; if ((vIface = getDynInterface((u_int32_t)vlan_id, false)) != NULL) { vIface->setTimeLastPktRcvd(h->ts.tv_sec); pass_verdict = vIface->processPacket(if_index, bridge_iface_idx, datalink_type, ingressPacket, when, packet_time, eth, vlan_id, iph, ip6, ip_offset, encapsulation_overhead, len_on_wire, h, packet, ndpiProtocol, srcHost, dstHost, hostFlow); processed = true; } } } if (processed && !showDynamicInterfaceTraffic()) { incStats(*ingressPacket, when->tv_sec, ETHERTYPE_IP, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); return (pass_verdict); } } if (eth == NULL) { incStats(*ingressPacket, when->tv_sec, ETHERTYPE_IP, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); return (pass_verdict); } if ((srcMac = getMac(eth->h_source, true /* Create if missing */, true /* Inline call */))) { /* NOTE: in nEdge, stats are updated into Flow::update_hosts_stats */ #ifndef HAVE_NEDGE srcMac->incSentStats(getTimeLastPktRcvd(), 1, len_on_wire); #endif srcMac->setSeenIface(bridge_iface_idx); #ifdef HAVE_NEDGE u_int16_t mac_pool = 0; char bufMac[24]; char *mac_str; /* When captive portal is disabled, use the auto_assigned_pool_id as the * default MAC pool */ if (host_pools && (ntop->getPrefs()->get_auto_assigned_pool_id() != NO_HOST_POOL_ID) && (!ntop->getPrefs()->isCaptivePortalEnabled()) && (srcMac->locate() == located_on_lan_interface)) { if (!host_pools->findMacPool(srcMac->get_mac(), &mac_pool) || (mac_pool == NO_HOST_POOL_ID)) { mac_str = Utils::formatMac(srcMac->get_mac(), bufMac, sizeof(bufMac)); ntop->addToPool(mac_str, ntop->getPrefs()->get_auto_assigned_pool_id()); } } #endif } if ((dstMac = getMac(eth->h_dest, true /* Create if missing */, true /* Inline call */))) { /* NOTE: in nEdge, stats are updated into Flow::update_hosts_stats */ #ifndef HAVE_NEDGE dstMac->incRcvdStats(getTimeLastPktRcvd(), 1, len_on_wire); #endif } if (iph != NULL) { u_int16_t ip_len, ip_tot_len; /* IPv4 */ if ((trusted_ip_len < 20) || ((ip_len = iph->ihl * 4) == 0)) { incStats(*ingressPacket, when->tv_sec, ETHERTYPE_IP, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); return (pass_verdict); } /* NOTE: ip_tot_len is not trusted as may be forged */ ip_tot_len = ntohs(iph->tot_len); if(ip_tot_len > (h->caplen - ip_offset)) { /* Invalid lenght */ incStats(*ingressPacket, when->tv_sec, ETHERTYPE_IP, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); return (pass_verdict); } tos = iph->tos; /* Use the actual h->len and not the h->caplen to determine whether a packet is fragmented. */ if (ip_len > (int)h->len - ip_offset || (int)h->len - ip_offset < ip_tot_len || (iph->frag_off & htons(0x1FFF /* IP_OFFSET */)) || (iph->frag_off & htons(0x2000 /* More Fragments: set */))) { is_fragment = true; fragment_offset = ((ntohs(iph->frag_off) & 0x3fff) & 0x1FFF) * 8; #ifdef IMPLEMENT_SMART_FRAGMENTS if (fragment_offset) { incStats(*ingressPacket, when->tv_sec, ETHERTYPE_IP, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); return (pass_verdict); } #endif } l4_proto = iph->protocol; l4 = ((u_int8_t *)iph + ip_len); l4_len = ip_tot_len - ip_len; /* use len from the ip header to compute sequence numbers */ ip = (u_int8_t *)iph; /* An ethernet frame can contain padding at the end of the packet. * Such padding can be identified by comparing the total packet length * reported into the IP header with the ethernet frame size. Such padding * should not be accounted in the L4 size. */ if (packet + h->caplen > ip + ip_tot_len) frame_padding = packet + h->caplen - ip - ip_tot_len; tos = iph->tos; } else { /* IPv6 */ u_int ipv6_shift = sizeof(const struct ndpi_ipv6hdr); u_int32_t *tos_ptr = (u_int32_t *)ip6; if (trusted_ip_len < sizeof(const struct ndpi_ipv6hdr)) { incStats(*ingressPacket, when->tv_sec, ETHERTYPE_IPV6, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); return (pass_verdict); } l4_proto = ip6->ip6_hdr.ip6_un1_nxt; if ((l4_proto == 0x3C /* IPv6 destination option */) || (l4_proto == 0x0 /* Hop-by-hop option */)) { u_int8_t *options = (u_int8_t *)ip6 + ipv6_shift; l4_proto = options[0]; ipv6_shift += 8 * (options[1] + 1); if (trusted_ip_len < ipv6_shift) { incStats(*ingressPacket, when->tv_sec, ETHERTYPE_IPV6, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); return (pass_verdict); } } l4 = (u_int8_t *)ip6 + ipv6_shift; l4_len = packet + h->len - l4; ip = (u_int8_t *)ip6; tos = ((ntohl(*tos_ptr) & 0xFF00000) >> 20) & 0xFF; } if((packet + h->caplen) > l4) trusted_l4_packet_len = packet + h->caplen - l4; else { /* Invalid lenght */ incStats(*ingressPacket, when->tv_sec, ETHERTYPE_IPV6, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); return (pass_verdict); } if (trusted_l4_packet_len > frame_padding) trusted_l4_packet_len -= frame_padding; if (l4_proto == IPPROTO_TCP) { if (trusted_l4_packet_len >= sizeof(struct ndpi_tcphdr)) { u_int tcp_len; /* TCP */ tcph = (struct ndpi_tcphdr *)l4; src_port = tcph->source, dst_port = tcph->dest; tcp_flags = l4[13]; tcp_len = min_val(4 * tcph->doff, trusted_l4_packet_len); payload = &l4[tcp_len]; trusted_payload_len = trusted_l4_packet_len - tcp_len; #ifdef NTOPNG_PRO if(tcp_flags & TH_SYN) { if(tcp_len > sizeof(struct ndpi_tcphdr)) { u_int8_t *options = (u_int8_t*)(&l4[sizeof(struct ndpi_tcphdr)]); u_int8_t options_len = tcp_len - sizeof(struct ndpi_tcphdr); for(u_int i=0; igetTrace()->traceEvent(TRACE_INFO, "Invalid TCP packet received [%u bytes long]", trusted_l4_packet_len); incStats(*ingressPacket, when->tv_sec, iph ? ETHERTYPE_IP : ETHERTYPE_IPV6, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); return (pass_verdict); } } else if (l4_proto == IPPROTO_UDP) { if (trusted_l4_packet_len >= sizeof(struct ndpi_udphdr)) { /* UDP */ udph = (struct ndpi_udphdr *)l4; src_port = udph->source, dst_port = udph->dest; payload = &l4[sizeof(struct ndpi_udphdr)]; trusted_payload_len = trusted_l4_packet_len - sizeof(struct ndpi_udphdr); #ifndef HAVE_NEDGE #ifdef IMPLEMENT_SMART_FRAGMENTS if (is_fragment) fragment_extra_overhead = ntohs(udph->len) - l4_len + sizeof(struct ndpi_iphdr); #endif #endif } else { /* Packet too short: this is a faked packet */ ntop->getTrace()->traceEvent(TRACE_INFO, "Invalid UDP packet received [%u bytes long]", trusted_l4_packet_len); incStats(*ingressPacket, when->tv_sec, iph ? ETHERTYPE_IP : ETHERTYPE_IPV6, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); return (pass_verdict); } } else if (l4_proto == IPPROTO_SCTP) { if (trusted_l4_packet_len >= sizeof(struct sctphdr)) { /* SCTP */ sctph = (struct sctphdr *)l4; src_port = sctph->sport, dst_port = sctph->dport; payload = &l4[sizeof(struct sctphdr)]; trusted_payload_len = trusted_l4_packet_len - sizeof(struct sctphdr); } else { /* Packet too short: this is a faked packet */ ntop->getTrace()->traceEvent(TRACE_INFO, "Invalid SCTP packet received [%u bytes long]", trusted_l4_packet_len); incStats(*ingressPacket, when->tv_sec, iph ? ETHERTYPE_IP : ETHERTYPE_IPV6, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); return (pass_verdict); } } else if (l4_proto == IPPROTO_ICMP) { icmp_info.dissectICMP(trusted_l4_packet_len, l4); } else { /* non TCP/UDP protocols */ } if (iph != NULL) src_ip.set(iph->saddr), dst_ip.set(iph->daddr); else src_ip.set(&ip6->ip6_src), dst_ip.set(&ip6->ip6_dst); #if defined(WIN32) && defined(DEMO_WIN32) if (this->ethStats.getNumPackets() > MAX_NUM_PACKETS) { static bool showMsg = false; if (!showMsg) { ntop->getTrace()->traceEvent(TRACE_NORMAL, "-----------------------------------------------------------"); ntop->getTrace()->traceEvent(TRACE_NORMAL, "WARNING: this demo application is a limited ntopng version able to"); ntop->getTrace()->traceEvent(TRACE_NORMAL, "capture up to %d packets. If you are interested", MAX_NUM_PACKETS); ntop->getTrace()->traceEvent(TRACE_NORMAL, "in the full version please have a look at the ntop"); ntop->getTrace()->traceEvent(TRACE_NORMAL, "home page http://www.ntop.org/."); ntop->getTrace()->traceEvent(TRACE_NORMAL, "-----------------------------------------------------------"); ntop->getTrace()->traceEvent(TRACE_NORMAL, ""); showMsg = true; } return (pass_verdict); } #endif /* We need to populate the private_flow_id with protocol-specific such as DNS.... Unfortunately nDPI has not yet seen this packet so we need to implement a micro-DPI code here */ if ((l4_proto == IPPROTO_UDP) && (trusted_payload_len > 20)) { u_int16_t fiftythree = htons(53); if ((src_port == fiftythree) || (dst_port == fiftythree)) { /* Looks like DNS */ u_int16_t dns_transaction_id = (payload[0] << 8) + payload[1]; // ntop->getTrace()->traceEvent(TRACE_NORMAL, "%04X", dns_transaction_id); private_flow_id = (u_int32_t)dns_transaction_id; } } INTERFACE_PROFILING_SECTION_ENTER("NetworkInterface::processPacket: getFlow", 0); pre_get_flow: /* Updating Flow */ flow = getFlow(if_index, srcMac, dstMac, vlan_id, 0 /* observationPointId */, private_flow_id, 0, 0, 0, l4_proto == IPPROTO_ICMP ? &icmp_info : NULL, &src_ip, &dst_ip, src_port, dst_port, l4_proto, &src2dst_direction, last_pkt_rcvd, last_pkt_rcvd, len_on_wire, &new_flow, true, eth->h_source, eth->h_dest /* Eth lvl, used just in view interfaces to add MAC */); INTERFACE_PROFILING_SECTION_EXIT(0); if (flow == NULL) { incStats(*ingressPacket, when->tv_sec, iph ? ETHERTYPE_IP : ETHERTYPE_IPV6, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, l4_proto, len_on_wire, 1); return (pass_verdict); } else { #ifdef HAVE_NEDGE if (new_flow) flow->setIngress2EgressDirection(*ingressPacket); #endif if (flow->is_swap_requested() /* This guarantees that at least a packet has been observed in both directions, and that we are in the dst->src direction of the flow that is being swapped */ && (!src2dst_direction)) { #if 0 char buf[256]; ntop->getTrace()->traceEvent(TRACE_NORMAL, "Swapping %s", flow->print(buf, sizeof(buf))); #endif flow->set_swap_done(); /* Mark the old flow with the swap done */ goto pre_get_flow; /* Start over */ } *srcHost = src2dst_direction ? flow->get_cli_host() : flow->get_srv_host(); *dstHost = src2dst_direction ? flow->get_srv_host() : flow->get_cli_host(); *hostFlow = flow; flow->setTOS(tos, src2dst_direction); switch (l4_proto) { case IPPROTO_TCP: #if defined(NTOPNG_PRO) flow->updateTCPAck(when, src2dst_direction, ntohl(tcph->ack_seq)); if((tcp_flags & TH_SYN) == 0) flow->updateTCPWin(src2dst_direction, ntohs(tcph->window)); else flow->updateTCPWinScale(src2dst_direction, tcp_window_scale); #endif flow->updateTcpFlags(when, tcp_flags, src2dst_direction, new_flow); /* This is the heuristic "For TCP flows for which the 3WH has not been observed..." at https://github.com/ntop/ntopng/issues/5058 So, only for the first packet check if this flow is a swap candidate. The only condition that should NOT be checked for swap is when there's a SYN and not an ACK, i.e., when we see the first packet of the TWH that allows to reliably determine the direction. */ if (new_flow && (!(tcp_flags & TH_SYN) || (tcp_flags & TH_ACK))) flow->check_swap(); if ((tcp_flags & (TH_RST | TH_FIN)) == 0) { /* Ignore Zero-window on flow termination as this case is not necessary a zero windon indication */ flow->updateTcpWindow(ntohs(tcph->window), src2dst_direction); } flow->updateTcpSeqNum(when, ntohl(tcph->seq), ntohl(tcph->ack_seq), ntohs(tcph->window), tcp_flags, l4_len - (4 * tcph->doff), src2dst_direction); break; case IPPROTO_UDP: #ifdef NTOPNG_PRO flow->updateUDPTimestamp(src2dst_direction, &h->ts); #endif break; case IPPROTO_ICMP: case IPPROTO_ICMPV6: if (trusted_l4_packet_len > 2) { u_int8_t icmp_type = l4[0]; u_int8_t icmp_code = l4[1]; flow->setICMP(src2dst_direction, icmp_type, icmp_code, l4); flow->updateICMPFlood(when, src2dst_direction); flow->setICMPPayloadSize(trusted_l4_packet_len); trusted_payload_len = trusted_l4_packet_len, payload = l4; } break; default: /* NOTE: for non TCP-flows, the swap heuristic is always checked on the first packet */ if (new_flow) flow->check_swap(); break; } #ifndef HAVE_NEDGE #ifdef __OpenBSD__ struct timeval tv_ts; tv_ts.tv_sec = h->ts.tv_sec; tv_ts.tv_usec = h->ts.tv_usec; flow->incStats(src2dst_direction, len_on_wire - encapsulation_overhead - frame_padding, payload, trusted_payload_len, l4_proto, is_fragment, tcp_flags, &tv_ts, fragment_extra_overhead); #else flow->incStats(src2dst_direction, len_on_wire - encapsulation_overhead - frame_padding, payload, trusted_payload_len, l4_proto, is_fragment, tcp_flags, &h->ts, fragment_extra_overhead); #endif #endif } /* In case of a traffic mirror with no MAC gateway address configured the traffic direction is set based on the local (-m) host (** 2 **). See also (** 1 **). */ if (isTrafficMirrored() && (!isGwMacConfigured())) { bool cli_local = src_ip.isLocalHost(); bool srv_local = dst_ip.isLocalHost(); if (cli_local && (!srv_local)) *ingressPacket = false; else if ((!cli_local) && srv_local) *ingressPacket = true; else ; /* Leave as is */ #if 0 char a[32], b[32]; ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s (%s) -> %s (%s) [%s]", src_ip.print(a, sizeof(a)), src_ip.isLocalHost() ? "L" : "R", dst_ip.print(b, sizeof(b)), dst_ip.isLocalHost() ? "L" : "R", *ingressPacket ? "IN" : "OUT"); #endif } /* Protocol Detection */ /* This is now incremented in Flow::hosts_periodic_stats_update * by calling iface->incLocalStats */ // flow->updateInterfaceLocalStats(src2dst_direction, 1, len_on_wire); if (!flow->isDetectionCompleted() || flow->needsExtraDissection()) { if ((!is_fragment) #ifdef IMPLEMENT_SMART_FRAGMENTS || (fragment_offset == 0) #endif ) flow->processPacket(src2dst_direction, h, ip, trusted_ip_len, packet_time, payload, trusted_payload_len, src_port); else { // FIX - only handle unfragmented packets // ntop->getTrace()->traceEvent(TRACE_WARNING, "IP fragments are not // handled yet!"); } } if (flow->isDetectionCompleted() && (!isSampledTraffic())) { switch (ndpi_get_lower_proto(flow->get_detected_protocol())) { case NDPI_PROTOCOL_DHCP: if (*srcHost) { Mac *mac = (*srcHost)->getMac(), *payload_cli_mac; if (mac && (trusted_payload_len > 240)) { struct dhcp_packet *dhcpp = (struct dhcp_packet *)payload; if (dhcpp->msgType == 0x01) { /* Request */ ; // mac->setDhcpHost(); mac->incNumDHCPRequests(); } else if (dhcpp->msgType == 0x02) { /* Reply */ checkMacIPAssociation(false, dhcpp->chaddr, dhcpp->yiaddr, mac); checkDhcpIPRange(mac, dhcpp, vlan_id); setDHCPAddressesSeen(); mac->incNumDHCPReplies(); } for (u_int32_t i = 240; (i+1) < trusted_payload_len;) { u_int8_t id = payload[i], len = payload[i + 1]; if (len == 0) break; #ifdef DHCP_DEBUG ntop->getTrace()->traceEvent(TRACE_WARNING, "[DHCP] [id=%u][len=%u]", id, len); #endif if((i + 2 + len) < trusted_payload_len) { if (id == 12 /* Host Name */) { char name[64], buf[24], *client_mac, key[64]; int j; j = ndpi_min(len, sizeof(name) - 1); strncpy((char *)name, (char *)&payload[i + 2], j); name[j] = '\0'; client_mac = Utils::formatMac(&payload[28], buf, sizeof(buf)); ntop->getTrace()->traceEvent(TRACE_INFO, "[DHCP] %s = '%s'", client_mac, name); snprintf(key, sizeof(key), DHCP_CACHE, get_id(), client_mac); ntop->getRedis()->set(key, name, 86400 /* 1d duration */); if ((payload_cli_mac = getMac(&payload[28], false /* Do not create if missing */, true /* Inline call */))) payload_cli_mac->inlineSetDHCPName(name); #ifdef DHCP_DEBUG ntop->getTrace()->traceEvent(TRACE_WARNING, "[DHCP] %s = '%s'", client_mac, name); #endif } else if (id == 0xFF) break; /* End of options */ } else break; /* Invalid lenght */ i += len + 2; } } } if (*dstHost) { char host_name[64]; Mac *dst_mac = (*dstHost)->getMac(); if (dst_mac && !(dst_mac->isBroadcast())) { flow->setDHCPHostName(dst_mac->getDHCPNameNotLowerCase(host_name, sizeof(host_name))); } } break; case NDPI_PROTOCOL_DHCPV6: if (*srcHost && *dstHost) { Mac *src_mac = (*srcHost)->getMac(); Mac *dst_mac = (*dstHost)->getMac(); if (src_mac && dst_mac && (trusted_payload_len > 20) && dst_mac->isMulticast()) ; // src_mac->setDhcpHost(); } break; case NDPI_PROTOCOL_NETBIOS: flow->dissectNetBIOS(payload, trusted_payload_len); break; case NDPI_PROTOCOL_BITTORRENT: if ((flow->getBitTorrentHash() == NULL) && (l4_proto == IPPROTO_UDP) && (flow->get_packets() < 8)) flow->dissectBittorrent((char *)payload, trusted_payload_len); break; case NDPI_PROTOCOL_HTTP: if (trusted_payload_len > 0) flow->dissectHTTP(src2dst_direction, (char *)payload, trusted_payload_len); break; case NDPI_PROTOCOL_SSDP: if (trusted_payload_len > 0) flow->dissectSSDP(src2dst_direction, (char *)payload, trusted_payload_len); break; case NDPI_PROTOCOL_DNS: /* DNS-over-TCP flows may carry zero-payload TCP segments e.g., during three-way-handshake, or when acknowledging. Make sure only non-zero-payload segments are processed. */ /* However still consider for a possible DNS flood */ #ifdef NTOPNG_PRO flow->updateDNSFlood(when, src2dst_direction); #endif if ((trusted_payload_len > 0) && payload) { flow->dissectDNS(src2dst_direction, (char *)payload, trusted_payload_len); /* DNS-over-TCP has a 2-bytes field with DNS payload length at the beginning. See RFC1035 section 4.2.2. TCP usage. */ } break; case NDPI_PROTOCOL_SNMP: #ifdef NTOPNG_PRO flow->updateSNMPFlood(when, src2dst_direction); #endif break; case NDPI_PROTOCOL_IEC60870: if ((trusted_payload_len > 0) && payload) { flow->processIEC60870Packet((htons(src_port) == 2404) ? true : false, payload, trusted_payload_len, h); } break; #ifdef NTOPNG_PRO case NDPI_PROTOCOL_MODBUS: if ((trusted_payload_len > 0) && payload) { flow->processModbusPacket((htons(dst_port) == 502) ? true : false, payload, trusted_payload_len, h); } break; #endif case NDPI_PROTOCOL_MDNS: #ifdef MDNS_TEST extern void _dissectMDNS(u_char * buf, u_int buf_len, char *out, u_int out_len); char outbuf[1024]; _dissectMDNS(payload, trusted_payload_len, outbuf, sizeof(outbuf)); ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s", outbuf); #endif flow->dissectMDNS(payload, trusted_payload_len); if (discovery && iph) discovery->queueMDNSResponse(iph->saddr, payload, trusted_payload_len); break; case NDPI_PROTOCOL_ZOOM: case NDPI_PROTOCOL_RTP: case NDPI_PROTOCOL_SRTP: if (flow->isZoomRTP()) { if (payload[0] == 5 /* RTCP/RTP */) { u_int8_t encoding_type = payload[8]; /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "Zoom [%d]", encoding_type); */ switch (encoding_type) { case 13: /* Screen Share */ case 30: /* Screen Share */ flow->setRTPStreamType(ndpi_multimedia_screen_sharing_flow); break; case 15: /* Audio */ flow->setRTPStreamType(ndpi_multimedia_audio_flow); break; case 16: /* Video */ flow->setRTPStreamType(ndpi_multimedia_video_flow); break; } } } else if (flow->getRTPStreamType() == ndpi_multimedia_unknown_flow) { if (flow->get_ndpi_flow() != NULL) { flow->setRTPStreamType(flow->get_ndpi_flow()->flow_multimedia_types); } } #ifdef NTOPNG_PRO if ((trusted_payload_len > 0) && payload) { flow->processRTPPacket(payload, trusted_payload_len, h, src2dst_direction); } #endif break; } #ifdef HAVE_NEDGE if (is_bridge_interface()) { struct tm now; time_t t_now = time(NULL); localtime_r(&t_now, &now); pass_verdict = flow->checkPassVerdict(&now); if (pass_verdict) { TrafficShaper *shaper_ingress, *shaper_egress; char buf[64]; flow->getFlowShapers(src2dst_direction, &shaper_ingress, &shaper_egress); ntop->getTrace()->traceEvent(TRACE_DEBUG, "[%s] %u / %u ", flow->get_detected_protocol_name(buf, sizeof(buf)), shaper_ingress, shaper_egress); pass_verdict = passShaperPacket(shaper_ingress, shaper_egress, (struct pcap_pkthdr *)h); } else { flow->incFlowDroppedCounters(); } } #endif } #ifdef NTOPNG_PRO if(flow && flow->isQUIC() && (trusted_payload_len > 0)) flow->updateQUICStats(src2dst_direction, &h->ts, payload, trusted_payload_len); #endif // ntop->getTrace()->traceEvent(TRACE_NORMAL, "direction: %s / len: %u", *ingressPacket ? "IN" : "OUT", len_on_wire); if(flow->isDetectionCompleted()) { /* Trick to avoid updating interface protocols when the flow application protocol has nit been detected hence account traffic as Unknown. See Flow::~Flow() */ if(flow->isFlowAccounted()) { incnDPIStats(when->tv_sec, flow->getStatsProtocol(), flow->get_protocol_category(), 0, len_on_wire, 0, 1); } else { incnDPIStats(when->tv_sec, flow->getStatsProtocol(), flow->get_protocol_category(), flow->get_bytes_cli2srv(), flow->get_bytes_srv2cli(), flow->get_packets_cli2srv(), flow->get_packets_srv2cli()); flow->setFlowAccounted(); /* Set the flow as accounted */ } } incStats(*ingressPacket, when->tv_sec, iph ? ETHERTYPE_IP : ETHERTYPE_IPV6, flow->getStatsProtocol(), flow->get_protocol_category(), l4_proto, len_on_wire, 1); /* For large flows, a periodic_stats_update is performed straight after processing a packet. Conditions checked to determine 'a large flow' are (1) A minimum number of bytes transferred since the previous periodic_stats_update (checked using get_current_*) (2) A minimum number of milliseconds elapsed since the previous periodic_stats_update (checked using get_current_update_time()) Conditions above are necessary as: (1) ensures that large flows perform faster periodic_stats_update, without having to wait for purgeIdle to visit the whole hash table (this may take up to PURGE_FRACTION seconds). (2) ensures that periodic_stats_update is not performed too frequently as it could be detrimental for performances and lead to packet drops. */ if (flow->get_current_bytes_cli2srv() + flow->get_current_bytes_srv2cli() >= PERIODIC_STATS_UPDATE_MIN_REFRESH_BYTES && Utils::msTimevalDiff(when, flow->get_current_update_time()) >= PERIODIC_STATS_UPDATE_MIN_REFRESH_MS) { flow->periodic_stats_update(when); } return (pass_verdict); } /* **************************************************** */ void NetworkInterface::purgeIdle(time_t when, bool force_idle, bool full_scan) { u_int n, m, o; last_pkt_rcvd = when; bcast_domains->reloadBroadcastDomains(full_scan /* Force a reload only if a full scan is requested */); if ((n = purgeIdleFlows(force_idle, full_scan)) > 0) ntop->getTrace()->traceEvent(TRACE_DEBUG, "Purged %u/%u idle flows on %s", n, getNumFlows(), ifname); if ((m = purgeIdleHosts(force_idle, full_scan)) > 0) ntop->getTrace()->traceEvent(TRACE_DEBUG, "Purged %u/%u idle hosts on %s", m, getNumHosts(), ifname); if ((o = purgeIdleMacsASesCountriesVLANs(force_idle, full_scan)) > 0) ntop->getTrace()->traceEvent(TRACE_DEBUG, "Purged %u idle ASs, MAC, Countries, VLANs... on %s", o, ifname); for (std::map::iterator it = flowHashing.begin(); it != flowHashing.end(); ++it) it->second->purgeIdle(when, force_idle, full_scan); #if defined(NTOPNG_PRO) if (pMap) pMap->purgeIdle(when); if (sMap) sMap->purgeIdle(when); #endif if (gw_macs_reload_requested) reloadGwMacs(); } /* ****************************************************** */ u_int16_t NetworkInterface::guessEthType(const u_char *p, u_int len, u_int8_t *is_ethernet) { if (len >= sizeof(struct ndpi_ethhdr)) { /* Check if this looks like ethernet */ struct ndpi_ethhdr *ehdr = (struct ndpi_ethhdr *)p; u_int16_t eth_type = ntohs(ehdr->h_proto); switch (eth_type) { case ETHERTYPE_IP: case ETHERTYPE_IPV6: case ETHERTYPE_PPPoE: case ETHERTYPE_VLAN: *is_ethernet = 1; return (eth_type); } } *is_ethernet = 0; if (len >= sizeof(struct ndpi_iphdr)) { struct ndpi_iphdr *ipv4 = (struct ndpi_iphdr *)p; if (ipv4->version == 4) return (ETHERTYPE_IP); } if (len >= sizeof(struct ip6_hdr)) { struct ip6_hdr *ipv6 = (struct ip6_hdr *)p; if (((ipv6->ip6_vfc >> 4) & 0x0f) == 6) return (ETHERTYPE_IPV6); } return (1 /* Unknown */); } /* **************************************************** */ bool NetworkInterface::dissectPacket(int32_t if_index, u_int32_t bridge_iface_idx, int datalink_type, bool ingressPacket, u_int8_t *sender_mac, const struct pcap_pkthdr *h, const u_char *packet, u_int16_t *ndpiProtocol, Host **srcHost, Host **dstHost, Flow **flow) { struct ndpi_ethhdr *ethernet = NULL, dummy_ethernet; u_int64_t time; u_int16_t eth_type, ip_offset = 0, vlan_id = 0, eth_offset = 0, encapsulation_overhead = 0; u_int32_t null_type; bool pass_verdict = true; u_int32_t len_on_wire = h->len * getScalingFactor(); *flow = NULL; /* Note dummy ethernet is always 0 unless sender_mac is set (Netfilter only) */ memset(&dummy_ethernet, 0, sizeof(dummy_ethernet)); /* Periodic housekeeping activites put here to avoid locks during code execution */ pollQueuedeCompanionEvents(); bcast_domains->reloadBroadcastDomains(); /* Netfilter interfaces don't report MAC addresses on packets */ if (getIfType() == interface_type_NETFILTER) len_on_wire += sizeof(struct ndpi_ethhdr); if (h->len == 0) { incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, 0, 1); goto dissect_packet_end; } else if (h->len > ifMTU) { if (!mtuWarningShown) { #ifdef __linux__ ntop->getTrace()->traceEvent(TRACE_NORMAL, "Packets exceeding the expected max size have been received " "[%s][len: %u][max len: %u].", get_name(), h->len, ifMTU); if (!read_from_pcap_dump()) { ntop->getTrace()->traceEvent(TRACE_WARNING, "If TSO/GRO is enabled, please disable it for best accuracy"); if(strchr(ifname, ':') == NULL) { /* print ethtool command for standard interfaces only */ ntop->getTrace()->traceEvent(TRACE_WARNING, "using: sudo ethtool -K %s gro off gso off tso off", (strchr(ifname, ',') == NULL) ? ifname : ""); } } #endif mtuWarningShown = true; } } setTimeLastPktRcvd(h->ts.tv_sec); if (last_purge_idle != (u_int32_t)h->ts.tv_sec) { /* Purge idle flows unless read from a pcap interface */ if (!read_from_pcap_dump()) purgeIdle(h->ts.tv_sec); last_purge_idle = h->ts.tv_sec; } time = ((uint64_t)h->ts.tv_sec) * 1000 + h->ts.tv_usec / 1000; datalink_check: if (datalink_type == DLT_NULL) { if (h->caplen < sizeof(u_int32_t)) { incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, h->len, 1); goto dissect_packet_end; } if((eth_offset + sizeof(u_int32_t)) <= h->caplen) memcpy(&null_type, &packet[eth_offset], sizeof(u_int32_t)); else { incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, h->len, 1); goto dissect_packet_end; } switch (null_type) { case BSD_AF_INET: eth_type = ETHERTYPE_IP; break; case BSD_AF_INET6_BSD: case BSD_AF_INET6_FREEBSD: case BSD_AF_INET6_DARWIN: eth_type = ETHERTYPE_IPV6; break; default: incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; /* Any other non IP protocol */ } ethernet = (struct ndpi_ethhdr *)&dummy_ethernet; if (sender_mac) memcpy(&dummy_ethernet.h_source, sender_mac, 6); ip_offset = 4 + eth_offset; } else if (datalink_type == DLT_EN10MB) { if (h->caplen < sizeof(ndpi_ethhdr)) { incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, h->len, 1); goto dissect_packet_end; } ethernet = (struct ndpi_ethhdr *)&packet[eth_offset]; ip_offset = sizeof(struct ndpi_ethhdr) + eth_offset; eth_type = ntohs(ethernet->h_proto); } else if (datalink_type == 113 /* Linux Cooked Capture */) { if (h->caplen < 16) { incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, h->len, 1); goto dissect_packet_end; } ethernet = (struct ndpi_ethhdr *)&dummy_ethernet; if (sender_mac) memcpy(&dummy_ethernet.h_source, sender_mac, 6); eth_type = (packet[eth_offset + 14] << 8) + packet[eth_offset + 15]; ip_offset = 16 + eth_offset; #ifdef DLT_RAW } else if (datalink_type == DLT_RAW /* Linux TUN/TAP device in TUN mode; Raw IP capture */ || datalink_type == 14 /* raw IP DLT_RAW on OpenBSD captures */) { if (h->caplen < sizeof(u_int32_t)) { incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, h->len, 1); goto dissect_packet_end; } switch ((packet[eth_offset] & 0xf0) >> 4) { case 4: eth_type = ETHERTYPE_IP; break; case 6: eth_type = ETHERTYPE_IPV6; break; default: incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; /* Unknown IP protocol version */ } if (sender_mac) memcpy(&dummy_ethernet.h_source, sender_mac, 6); ethernet = (struct ndpi_ethhdr *)&dummy_ethernet; ip_offset = eth_offset; #endif /* DLT_RAW */ } else if (datalink_type == DLT_ENC) { if (packet[0] == 2 /* IPv4 */) { eth_type = ETHERTYPE_IP; ethernet = (struct ndpi_ethhdr *)&dummy_ethernet; ip_offset = 12; } /* TODO support IPv6 encapsulation one day */ } else if (datalink_type == DLT_IPV4) { eth_type = ETHERTYPE_IP; if (sender_mac) memcpy(&dummy_ethernet.h_source, sender_mac, 6); ethernet = (struct ndpi_ethhdr *)&dummy_ethernet; ip_offset = 0; } else { incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } /* Make sure this label is BEFORE detunneling of VLAN or MPLS traffic. Otherwise, VLAN or MPLS traffic carried inside other tunnels, i.e., GRE or ERSPAN, won't be detunneled. */ decode_packet_eth: while (ip_offset < h->caplen) { if((eth_type == 0x8100 /* VLAN */) && ((ip_offset + sizeof(Ether80211q)) < h->caplen)) { Ether80211q *qType = (Ether80211q *)&packet[ip_offset]; vlan_id = ntohs(qType->vlanId) & 0xFFF; eth_type = (packet[ip_offset + 2] << 8) + packet[ip_offset + 3]; ip_offset += 4; } else if((eth_type == 0x8847 /* MPLS */) && ((unsigned int)(ip_offset + 2) < h->caplen)) { u_int8_t bos; /* bottom_of_stack */ bos = (((u_int8_t)packet[ip_offset + 2]) & 0x1), ip_offset += 4; if (bos) { if(h->caplen > (sizeof(struct ndpi_ethhdr) + ip_offset)) { u_int8_t is_ethernet; eth_type = guessEthType((const u_char *)&packet[ip_offset], h->caplen - ip_offset, &is_ethernet); if (is_ethernet) ip_offset += sizeof(struct ndpi_ethhdr); break; } else { incStats(ingressPacket, h->ts.tv_sec, ETHERTYPE_IP, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } } } else break; } /* Setting traffic direction based on MAC (** 1 **). See also (** 2 **) */ if (ethernet) { if (isTrafficMirrored() && isGwMacConfigured()) { /* Mirror */ if (isGwMac(ethernet->h_dest)) ingressPacket = false; } else if (!areTrafficDirectionsSupported()) { /* Interface with no direction info */ if (isInterfaceMac(ethernet->h_source)) ingressPacket = false; } } switch (eth_type) { case ETHERTYPE_PPPoE: ip_offset += 6 /* PPPoE */; /* Now we need to skip the PPP header */ if (packet[ip_offset] == 0x0) eth_type = packet[ip_offset + 1], ip_offset += 2; /* 2 Byte protocol */ else eth_type = packet[ip_offset], ip_offset += 1; /* 1 Byte protocol */ switch (eth_type) { case 0x21: eth_type = ETHERTYPE_IP; break; case 0x57: eth_type = ETHERTYPE_IPV6; break; default: incStats(ingressPacket, h->ts.tv_sec, ETHERTYPE_IP, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } goto decode_packet_eth; break; case ETHERTYPE_IP: if (h->caplen >= ip_offset + sizeof(struct ndpi_iphdr)) { u_int16_t frag_off; struct ndpi_iphdr *iph = (struct ndpi_iphdr *)&packet[ip_offset]; u_short ip_len = ((u_short)iph->ihl * 4); struct ndpi_ipv6hdr *ip6 = NULL; if((ip_len == 0) /* Invalid lenght */ || (iph->version != 4) /* This is not IPv4 */) { incStats(ingressPacket, h->ts.tv_sec, ETHERTYPE_IP, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } else frag_off = ntohs(iph->frag_off); if (ntop->getGlobals()->decode_tunnels() && (iph->protocol == IPPROTO_GRE) && ((frag_off & 0x3FFF /* IP_MF | IP_OFFSET */) == 0) && h->caplen >= ip_offset + ip_len + sizeof(struct grev1_header)) { struct grev1_header gre; u_int offset = ip_offset + ip_len + sizeof(struct grev1_header); memcpy(&gre, &packet[ip_offset + ip_len], sizeof(struct grev1_header)); gre.flags_and_version = ntohs(gre.flags_and_version); gre.proto = ntohs(gre.proto); if (gre.flags_and_version & (GRE_HEADER_CHECKSUM | GRE_HEADER_ROUTING)) offset += 4; if (gre.flags_and_version & GRE_HEADER_KEY) offset += 4; if (gre.flags_and_version & GRE_HEADER_SEQ_NUM) offset += 4; if (h->caplen >= offset) { if (gre.proto == 0x6558 /* Transparent Ethernet Bridging */) { eth_offset = offset; goto datalink_check; } else if (gre.proto == ETHERTYPE_IP) { ip_offset = offset, encapsulation_overhead = offset; goto decode_packet_eth; } else if (gre.proto == ETHERTYPE_IPV6) { eth_type = ETHERTYPE_IPV6; ip_offset = offset, encapsulation_overhead = offset; goto decode_packet_eth; } } /* ERSPAN Type 2 has an 8-byte header https://tools.ietf.org/html/draft-foschiano-erspan-00 */ if (h->caplen >= offset + sizeof(struct ndpi_ethhdr) + 8) { if (gre.proto == ETH_P_ERSPAN /* ERSPAN type II */) { offset += 8; eth_offset = offset, encapsulation_overhead = offset; ethernet = (struct ndpi_ethhdr *)&packet[eth_offset]; ip_offset = eth_offset + sizeof(struct ndpi_ethhdr); eth_type = ntohs(ethernet->h_proto); goto decode_packet_eth; } else if (gre.proto == ETH_P_ERSPAN2 /* ERSPAN version 2 (type III) */) { if (h->caplen >= offset + sizeof(struct ndpi_ethhdr) + 20) { offset += 20; eth_offset = offset, encapsulation_overhead = offset; ethernet = (struct ndpi_ethhdr *)&packet[eth_offset]; ip_offset = eth_offset + sizeof(struct ndpi_ethhdr); eth_type = ntohs(ethernet->h_proto); goto decode_packet_eth; } } else { /* Unknown encapsulation */ } } } else if (ntop->getGlobals()->decode_tunnels() && iph->protocol == IPPROTO_IPV6 && h->caplen >= ip_offset + ip_len + sizeof(struct ndpi_ipv6hdr)) { /* Detunnel 6in4 tunnel */ ip_offset += ip_len; eth_type = ETHERTYPE_IPV6; encapsulation_overhead = ip_offset; goto decode_packet_eth; } else if (ntop->getGlobals()->decode_tunnels() && (iph->protocol == IPPROTO_UDP) && ((frag_off & 0x3FFF /* IP_MF | IP_OFFSET */) == 0)) { if(ip_offset + ip_len + sizeof(struct ndpi_udphdr) > h->caplen) { incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } struct ndpi_udphdr *udp = (struct ndpi_udphdr *)&packet[ip_offset + ip_len]; u_int16_t sport = ntohs(udp->source), dport = ntohs(udp->dest); if ((sport == GTP_U_V1_PORT) || (dport == GTP_U_V1_PORT)) { /* Check if it's GTPv1 */ u_int offset = (u_int)(ip_offset + ip_len + sizeof(struct ndpi_udphdr)); u_int8_t flags = packet[offset]; u_int8_t message_type = packet[offset + 1]; if ((((flags & 0xE0) >> 5) == 1 /* GTPv1 */) && (message_type == 0xFF /* T-PDU */)) { ip_offset = ip_offset + ip_len + sizeof(struct ndpi_udphdr) + 8 /* GTPv1 header len */; if (flags & 0x04) ip_offset += 1 + 3 /* pad */ + 4 /* next extension (TODO better decoding) */; /* next_ext_header is present */ if (flags & 0x02) ip_offset += 4; /* sequence_number is present (it also includes next_ext_header and pdu_number) */ if (flags & 0x01) ip_offset += 1; /* pdu_number is present */ iph = (struct ndpi_iphdr *)&packet[ip_offset]; if (iph->version != 4) { /* FIX - Add IPv6 support */ incStats(ingressPacket, h->ts.tv_sec, ETHERTYPE_IPV6, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } } } else if ((sport == EOIP_PORT) && (dport == EOIP_PORT)) { u_int offset = ip_offset + ip_len + sizeof(struct ndpi_udphdr) + 12; eth_offset = offset; goto datalink_check; } else if ((sport == L2TP_PORT) && (dport == L2TP_PORT)) { u_int offset = ip_offset + ip_len + sizeof(struct ndpi_udphdr); if((offset+sizeof(struct l2tp_header)) > h->caplen) { struct l2tp_header *l2tp = (struct l2tp_header*)&packet[offset]; u_int16_t proto, flags = ntohs(l2tp->flags); if((flags & 0x8002) == 0x02) { /* L2TP v2 Data packet */ u_int8_t have_length_bit = (flags & 0x4000) == 0x4000; u_int8_t have_sequence_bit = (flags & 0x0800) == 0x0800; u_int8_t have_offset_bit = (flags & 0x0200) == 0x0200; offset += sizeof(struct l2tp_header); if(have_length_bit) offset += sizeof(u_int16_t); if(have_sequence_bit) offset += 2; if(have_offset_bit) offset += 2; } /* PPP */ offset += 2; proto = (packet[offset] << 8) + packet[offset+1]; offset += 2; /* Skip proto */ if(proto == 0x0021) { /* IPv4 */ eth_type = ETHERTYPE_IP; iph = (struct ndpi_iphdr *)&packet[offset]; } else if(proto == 0x0057) { /* IPv6 */ eth_type = ETHERTYPE_IPV6; ip6 = (struct ndpi_ipv6hdr *)&packet[offset]; } else { incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } } else { incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } } else if ((sport == TZSP_PORT) || (dport == TZSP_PORT)) { /* https://en.wikipedia.org/wiki/TZSP */ u_int offset = ip_offset + ip_len + sizeof(struct ndpi_udphdr); u_int8_t version = packet[offset]; u_int8_t type = packet[offset + 1]; u_int16_t encapsulates = ntohs(*((u_int16_t *)&packet[offset + 2])); if ((version == 1) && (type == 0) && (encapsulates == 1)) { u_int8_t stop = 0; offset += 4; while ((!stop) && (offset < h->caplen)) { u_int8_t tag_type = packet[offset]; u_int8_t tag_len; switch (tag_type) { case 0: /* PADDING Tag */ tag_len = 1; break; case 1: /* END Tag */ tag_len = 1, stop = 1; break; default: tag_len = packet[offset + 1]; break; } offset += tag_len; if (offset >= h->caplen) { incStats(ingressPacket, h->ts.tv_sec, ETHERTYPE_IPV6, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } else { eth_offset = offset; goto datalink_check; } } } } else if (dport == VXLAN_PORT) { eth_offset = ip_offset + ip_len + sizeof(struct ndpi_udphdr) + sizeof(struct ndpi_vxlanhdr); goto datalink_check; } if ((sport == CAPWAP_DATA_PORT) || (dport == CAPWAP_DATA_PORT)) { /* Control And Provisioning of Wireless Access Points https://www.rfc-editor.org/rfc/rfc5415.txt CAPWAP Header - variable length (5 MSB of byte 2 of header) IEEE 802.11 Data Flags - 24 bytes Logical-Link Control - 8 bytes Total = CAPWAP_header_length + 24 + 8 */ u_short eth_type; ip_offset = ip_offset + ip_len + sizeof(struct ndpi_udphdr); u_int8_t capwap_header_len = ((*(u_int8_t *)&packet[ip_offset + 1]) >> 3) * 4; ip_offset = ip_offset + capwap_header_len + 24 + 8; if (ip_offset >= h->len) { incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } eth_type = ntohs(*(u_int16_t *)&packet[ip_offset - 2]); switch (eth_type) { case ETHERTYPE_IP: iph = (struct ndpi_iphdr *)&packet[ip_offset]; break; case ETHERTYPE_IPV6: iph = NULL; ip6 = (struct ndpi_ipv6hdr *)&packet[ip_offset]; break; default: incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } } } else if (ntop->getGlobals()->decode_tunnels() && (iph->protocol == IPPROTO_IP_IN_IP)) { u_short ip_len = ((u_short)iph->ihl * 4); ip_offset += ip_len, eth_type = ETHERTYPE_IP; if((ip_len == 0) || (ip_offset > h->caplen)) { incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } else goto decode_packet_eth; } if (vlan_id && ntop->getPrefs()->do_ignore_vlans()) vlan_id = 0; if ((vlan_id == 0) && ntop->getPrefs()->do_simulate_vlans()) { vlan_id = (ip6 ? ip6->ip6_src.u6_addr.u6_addr8[15] + ip6->ip6_dst.u6_addr.u6_addr8[15] : iph->saddr + iph->daddr) % SIMULATE_VLANS_MAX_VALUE; } if (ntop->getPrefs()->do_ignore_macs()) ethernet = &dummy_ethernet; else if (unlikely(ntop->getPrefs()->do_simulate_macs())) { dummy_ethernet.h_source[0] = 0xb8, dummy_ethernet.h_source[1] = 0x27, dummy_ethernet.h_source[2] = 0xeb, dummy_ethernet.h_source[3] = 0xfd, dummy_ethernet.h_source[4] = 0x8e, dummy_ethernet.h_source[5] = rand() % 8; dummy_ethernet.h_dest[0] = 0xb8, dummy_ethernet.h_dest[1] = 0x27, dummy_ethernet.h_dest[2] = 0xeb, dummy_ethernet.h_dest[3] = 0xfd, dummy_ethernet.h_dest[4] = 0x8e, dummy_ethernet.h_dest[5] = rand() % 8; ethernet = &dummy_ethernet; } try { pass_verdict = processPacket(if_index, bridge_iface_idx, datalink_type, &ingressPacket, &h->ts, time, ethernet, vlan_id, iph, ip6, ip_offset, encapsulation_overhead, len_on_wire, h, packet, ndpiProtocol, srcHost, dstHost, flow); } catch (std::bad_alloc &ba) { static bool oom_warning_sent = false; if (!oom_warning_sent) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Not enough memory"); oom_warning_sent = true; } } } break; case ETHERTYPE_IPV6: if (h->caplen >= ip_offset + sizeof(struct ndpi_ipv6hdr)) { struct ndpi_iphdr *iph = NULL; struct ndpi_ipv6hdr *ip6 = (struct ndpi_ipv6hdr *)&packet[ip_offset]; if ((ntohl(ip6->ip6_hdr.ip6_un1_flow) & 0xF0000000) != 0x60000000) { /* This is not IPv6 */ incStats(ingressPacket, h->ts.tv_sec, ETHERTYPE_IPV6, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } else { u_int ipv6_shift = sizeof(const struct ndpi_ipv6hdr); u_int8_t l4_proto = ip6->ip6_hdr.ip6_un1_nxt; if (l4_proto == 0x3C /* IPv6 destination option */) { u_int8_t *options = (u_int8_t *)ip6 + ipv6_shift; l4_proto = options[0]; ipv6_shift = 8 * (options[1] + 1); } if (ntop->getGlobals()->decode_tunnels() && (l4_proto == IPPROTO_GRE) && h->caplen >= ip_offset + ipv6_shift + sizeof(struct grev1_header)) { struct grev1_header gre; u_int offset = ip_offset + ipv6_shift + sizeof(struct grev1_header); memcpy(&gre, &packet[ip_offset + ipv6_shift], sizeof(struct grev1_header)); gre.flags_and_version = ntohs(gre.flags_and_version); gre.proto = ntohs(gre.proto); if (gre.flags_and_version & (GRE_HEADER_CHECKSUM | GRE_HEADER_ROUTING)) offset += 4; if (gre.flags_and_version & GRE_HEADER_KEY) offset += 4; if (gre.flags_and_version & GRE_HEADER_SEQ_NUM) offset += 4; if (h->caplen >= offset) { if (gre.proto == ETHERTYPE_IP) { eth_type = ETHERTYPE_IP; ip_offset = offset, encapsulation_overhead = offset; goto decode_packet_eth; } else if (gre.proto == ETHERTYPE_IPV6) { ip_offset = offset, encapsulation_overhead = offset; goto decode_packet_eth; } } if (h->caplen >= offset + sizeof(struct ndpi_ethhdr) + 8 /* ERSPAN Type 2 header */) { if (gre.proto == ETH_P_ERSPAN) { offset += 8; eth_offset = offset, encapsulation_overhead = offset; ethernet = (struct ndpi_ethhdr *)&packet[eth_offset]; ip_offset = eth_offset + sizeof(struct ndpi_ethhdr); eth_type = ntohs(ethernet->h_proto); goto decode_packet_eth; } else if (gre.proto == ETH_P_ERSPAN2) { ; /* TODO: support ERSPAN Type 3 */ } else { /* Unknown encapsulation */ } } } else if (ntop->getGlobals()->decode_tunnels() && (l4_proto == IPPROTO_UDP)) { struct ndpi_udphdr *udp; u_int16_t sport, dport; if ((ip_offset + ipv6_shift + sizeof(struct ndpi_udphdr)) > h->caplen) { incStats(ingressPacket, h->ts.tv_sec, ETHERTYPE_IPV6, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } udp = (struct ndpi_udphdr *)&packet[ip_offset + ipv6_shift]; sport = udp->source, dport = udp->dest; if ((sport == CAPWAP_DATA_PORT) || (dport == CAPWAP_DATA_PORT)) { /* Control And Provisioning of Wireless Access Points https://www.rfc-editor.org/rfc/rfc5415.txt CAPWAP Header - variable length (5 MSB of byte 2 of header) IEEE 802.11 Data Flags - 24 bytes Logical-Link Control - 8 bytes Total = CAPWAP_header_length + 24 + 8 */ u_short eth_type; u_int8_t capwap_header_len; ip_offset = ip_offset + ipv6_shift + sizeof(struct ndpi_udphdr); if(((unsigned int) ip_offset + 1) < h->caplen) { capwap_header_len = ((*(u_int8_t *)&packet[ip_offset + 1]) >> 3) * 4; ip_offset = ip_offset + capwap_header_len + 24 + 8; if (ip_offset >= h->caplen) { incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } eth_type = ntohs(*(u_int16_t *)&packet[ip_offset - 2]); switch (eth_type) { case ETHERTYPE_IP: iph = (struct ndpi_iphdr *)&packet[ip_offset]; ip6 = NULL; break; case ETHERTYPE_IPV6: ip6 = (struct ndpi_ipv6hdr *)&packet[ip_offset]; break; default: incStats(ingressPacket, h->ts.tv_sec, 0, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } } } } else if (ntop->getGlobals()->decode_tunnels() && (l4_proto == IPPROTO_IP_IN_IP)) { eth_type = ETHERTYPE_IP; ip_offset += sizeof(struct ndpi_ipv6hdr); encapsulation_overhead = ip_offset; goto decode_packet_eth; } if (vlan_id && ntop->getPrefs()->do_ignore_vlans()) vlan_id = 0; if ((vlan_id == 0) && ntop->getPrefs()->do_simulate_vlans()) vlan_id = (ip6 ? ip6->ip6_src.u6_addr.u6_addr8[15] + ip6->ip6_dst.u6_addr.u6_addr8[15] : iph->saddr + iph->daddr) % SIMULATE_VLANS_MAX_VALUE; if (ntop->getPrefs()->do_ignore_macs()) ethernet = &dummy_ethernet; try { pass_verdict = processPacket(if_index, bridge_iface_idx, datalink_type, &ingressPacket, &h->ts, time, ethernet, vlan_id, iph, ip6, ip_offset, encapsulation_overhead, len_on_wire, h, packet, ndpiProtocol, srcHost, dstHost, flow); } catch (std::bad_alloc &ba) { static bool oom_warning_sent = false; if (!oom_warning_sent) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Not enough memory"); oom_warning_sent = true; } } } } break; default: /* No IPv4 nor IPv6 */ if (ntop->getPrefs()->do_ignore_macs()) ethernet = &dummy_ethernet; if (ethernet == NULL) { incStats(ingressPacket, h->ts.tv_sec, eth_type, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); goto dissect_packet_end; } Mac *srcMac = getMac(ethernet->h_source, true /* Create if missing */, true /* Inline call */); Mac *dstMac = getMac(ethernet->h_dest, true /* Create if missing */, true /* Inline call */); /* NOTE: in nEdge, stats are updated into Flow::update_hosts_stats */ #ifndef HAVE_NEDGE if (srcMac) srcMac->incSentStats(h->ts.tv_sec, 1, len_on_wire); if (dstMac) dstMac->incRcvdStats(h->ts.tv_sec, 1, len_on_wire); #endif if ((eth_type == ETHERTYPE_ARP) && (h->caplen >= (ip_offset + sizeof(arp_header)))) { struct arp_header *arpp = (struct arp_header *)&packet[ip_offset]; u_int16_t arp_opcode = ntohs(arpp->ar_op); /* Updates interface broadcast domains, according to what is seen in * this ARP that glues together L2 and L3 */ updateBroadcastDomains(vlan_id, ethernet->h_source, ethernet->h_dest, ntohl(arpp->arp_spa), ntohl(arpp->arp_tpa)); if (srcMac && dstMac && (!srcMac->isNull() || !dstMac->isNull())) { setSeenMacAddresses(); srcMac->setSourceMac(); if (arp_opcode == 0x1 /* ARP request */) { arp_requests++; srcMac->incSentArpRequests(); dstMac->incRcvdArpRequests(); } else if (arp_opcode == 0x2 /* ARP reply */) { arp_replies++; srcMac->incSentArpReplies(); dstMac->incRcvdArpReplies(); checkMacIPAssociation(true, arpp->arp_sha, arpp->arp_spa, srcMac); checkMacIPAssociation(true, arpp->arp_tha, arpp->arp_tpa, dstMac); } } } incStats(ingressPacket, h->ts.tv_sec, eth_type, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, 0, len_on_wire, 1); break; } dissect_packet_end: /* Live packet dump to mongoose */ if (num_live_captures > 0) deliverLiveCapture(h, packet, *flow); return(pass_verdict); } /* **************************************************** */ void NetworkInterface::pollQueuedeCompanionEvents() { if (companionQueue) { ParsedFlow *dequeued = NULL; while (dequeueFlowFromCompanion(&dequeued)) { Flow *flow = NULL; bool src2dst_direction, new_flow; flow = getFlow(UNKNOWN_PKT_IFACE_IDX, NULL /* srcMac */, NULL /* dstMac */, dequeued->vlan_id, 0 /* observationPointId */, dequeued->get_private_flow_id(), 0 /* deviceIP */, 0 /* inIndex */, 1 /* outIndex */, NULL /* ICMPinfo */, &dequeued->src_ip, &dequeued->dst_ip, dequeued->src_port, dequeued->dst_port, dequeued->l4_proto, &src2dst_direction, 0, 0, 0, &new_flow, true /* create_if_missing */, NULL, NULL); if (flow) { if (new_flow) flow->updateSeen(); if (dequeued->getAdditionalFieldsJSON()) { flow->setJSONInfo(dequeued->getAdditionalFieldsJSON()); } if (dequeued->getExternalAlert()) { /* Flow from SyslogParserInterface (Suricata) */ enum json_tokener_error jerr = json_tokener_success; json_object *o = json_tokener_parse_verbose(dequeued->getExternalAlert(), &jerr); if (o) flow->setExternalAlert(o); } if (dequeued->process_info_set || dequeued->container_info_set || dequeued->tcp_info_set) { /* Flow from ZMQParserInterface (nProbe Agent) */ flow->setParsedeBPFInfo(dequeued, false /* TODO */); } } delete dequeued; } return; } } /* **************************************************** */ /* Dequeue alerted flows from checks (and enqueue to recipients) */ u_int64_t NetworkInterface::dequeueFlowAlerts(u_int budget) { u_int64_t num_done = 0; while (flowAlertsQueue->isNotEmpty()) { FlowAlert *alert = flowAlertsQueue->dequeue(); Flow *f = alert->getFlow(); /* Enqueue alert to recipients */ f->enqueueAlertToRecipients(alert); #if DEBUG_FLOW_CHECKS ntop->getTrace()->traceEvent(TRACE_NORMAL, "Dequeued flow alert"); #endif /* Now that the job is done, the reference counter to the flow can be decreased. */ f->decUses(); num_done++; if (budget > 0 /* Budget requested */ && num_done >= budget /* Budget exceeded */) break; } return num_done; } /* **************************************************** */ /* Same as above but for hosts */ u_int64_t NetworkInterface::dequeueHostAlerts(u_int budget) { u_int64_t num_done = 0; while (hostAlertsQueue->isNotEmpty()) { HostAlertReleasedPair alert_info = hostAlertsQueue->dequeue(); HostAlert *alert = alert_info.first; bool released = alert_info.second; Host *h = alert->getHost(); /* Enqueue alert to recipients */ h->enqueueAlertToRecipients(alert, released); #if DEBUG_HOST_CHECKS ntop->getTrace()->traceEvent(TRACE_NORMAL, "Dequeued host alert"); #endif /* Now that the job is done, the reference counter to the host can be decreased. */ h->decUses(); num_done++; if (budget > 0 /* Budget requested */ && num_done >= budget /* Budget exceeded */) break; } return num_done; } /* **************************************************** */ u_int64_t NetworkInterface::dequeueFlowAlertsFromChecks(u_int budget) { u_int64_t num_done; num_done = dequeueFlowAlerts(budget); #if DEBUG_FLOW_CHECKS if (num_done > 0) ntop->getTrace()->traceEvent(TRACE_NORMAL, "Dequeued flows total [%u]", num_done); #endif return num_done; } /* **************************************************** */ u_int64_t NetworkInterface::dequeueHostAlertsFromChecks(u_int budget) { u_int64_t num_done; num_done = dequeueHostAlerts(budget); #if DEBUG_HOST_CHECKS if (num_done > 0) ntop->getTrace()->traceEvent(TRACE_NORMAL, "Dequeued hosts total [%u]", num_done); #endif return num_done; } /* **************************************************** */ void NetworkInterface::incNumQueueDroppedFlows(u_int32_t num) { /* For viewed interface, the dumper database is the one belonging to the overlying view interface. */ DB *dumper; #ifdef NTOPNG_PRO dumper = isViewed() ? viewedBy()->getDB() : getDB(); #else dumper = getDB(); #endif if (dumper) dumper->incNumQueueDroppedFlows(num); }; /* **************************************************** */ /* Dequeues flows enqueued for dump and perform the actual dump. A budget can be specified for idle and active flows. Specify a budget of 0 to indicate an unlimited budget. NOTE: in case of view interfaces, this method is called sequentially by the view interface on all the viewed interfaces. This function is called from a dedicated thread, only spawned when flow dump is enabled with -F. */ u_int64_t NetworkInterface::dequeueFlowsForDump(u_int idle_flows_budget, u_int active_flows_budget) { /* For viewed interface, the dumper database is the one belonging to the overlying view interface. */ DB *dumper; u_int64_t idle_flows_done = 0, active_flows_done = 0; time_t when = time(NULL); #ifdef NTOPNG_PRO dumper = isViewed() ? viewedBy()->getDB() : getDB(); #else dumper = getDB(); #endif #ifdef HAVE_ZMQ #ifndef HAVE_NEDGE if (ntop->get_export_interface() == NULL) #endif #endif if (dumper == NULL) { ntop->getTrace()->traceEvent(TRACE_INFO, "WARNING: Something is broken with flow dump"); return (0); } /* Process high-priority idle flows (they're high priority as an idle flow not dumped is lost) */ while (idleFlowsToDump->isNotEmpty()) { Flow *f = idleFlowsToDump->dequeue(); dumpFlowOut(dumper, when, f); // delete f; idle_flows_done++; if (idle_flows_budget > 0 /* Budget requested */ && idle_flows_done >= idle_flows_budget /* Budget exceeded */) break; } if (dumper) { /* Process low-priority active flows (they're low priority there can still be chances of dumping active flows later) */ while (activeFlowsToDump->isNotEmpty()) { Flow *f = activeFlowsToDump->dequeue(); if(dumpFlowOut(dumper, when, f)) active_flows_done++; if (active_flows_budget > 0 /* Budget requested */ && active_flows_done >= active_flows_budget /* Budget exceeded */) break; } } /* Wait until there's some work to do. Don't wait for viewed interfaces to prevent one viewed interface to block all the other interfaces. For viewed interfaces, this method is called sequentially in ViewInterface::dumpFlowLoop */ u_int64_t num_done = idle_flows_done + active_flows_done; #ifndef WIN32 if (!isViewed() && num_done == 0) { /* Do a timedwait to avoid blocking indefinitely. Failing to do this, for interfaces with no traffic, would cause the calling thread to wait and ignore shutdown. */ struct timespec dump_wait_expire; dump_wait_expire.tv_sec = time(NULL) + 2; dump_wait_expire.tv_nsec = 0; dump_condition.timedWait(&dump_wait_expire); } #endif #ifdef NTOPNG_PRO /* Flush possibly pending flows (avoids interfaces with almost no traffic to have their flows waiting in dump queues for too long) */ // flushFlowDump(); #endif if (dumper) dumper->checkIdle(when); return (num_done); } /* **************************************************** */ /* Thos method finally dumps a flow */ bool NetworkInterface::dumpFlowOut(DB *dumper, time_t when, Flow *f) { char *json = NULL; bool rc = true; if(dumper == NULL) { #ifdef NTOPNG_PRO dumper = isViewed() ? viewedBy()->getDB() : getDB(); #else dumper = getDB(); #endif if(dumper == NULL) return(false); } /* Checkpoint flow traffic counters for the dump */ f->update_partial_traffic_stats_db_dump(); /* Prepare the JSON - if requested */ if (flows_dump_json) { json = f->serialize(true /* Use JSON labels */); // ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s", json); } #ifdef HAVE_ZMQ #ifndef HAVE_NEDGE if (ntop->get_export_interface() && (json != NULL)) ntop->get_export_interface()->export_data(json); #endif #endif if (f->get_partial_bytes()) /* Make sure data is not at zero */ rc = dumper->dumpFlow(when, f, json); /* Finally dump this flow */ if (json) free(json); #if DEBUG_FLOW_DUMP ntop->getTrace()->traceEvent(TRACE_NORMAL, "Dumped active flow"); #endif if (!rc) incDBNumDroppedFlows(dumper); f->decUses(); /* Add done, decrease the reference counter */ f->set_dump_done(); return(true); } /* **************************************************** */ void NetworkInterface::flowAlertsDequeueLoop() { u_int64_t n; char buf[16]; snprintf(buf, sizeof(buf), "ntopng-%d-fchek", get_id()); Utils::setThreadName(buf); ntop->getTrace()->traceEvent(TRACE_NORMAL, "Started flow user script hooks loop on interface '%s' [id: %u]...", get_description(), get_id()); /* Wait until it starts up */ while (!isRunning()) { /* Control-C during startup */ if (ntop->getGlobals()->isShutdownRequested()) return; _usleep(10000); } /* Now operational */ while (!isShuttingDown()) { /* Dequeue flows for dump. */ n = dequeueFlowAlertsFromChecks(32 /* budget */); if (n == 0) { /* No flow was dequeued. Let's wait. */ #ifdef WIN32 /* If windows, sleep if nothing was done during the previous cycle. */ _usleep(10000); #else /* On non-windows, signal/waits are implemented to throttle the speed Wait for at most 1-2s. Cannot wait indefinitely as we must ensure purgeQueuedIdleFlows() gets executed, and also to exit when it's time to shutdown. */ struct timespec hooks_wait_expire; hooks_wait_expire.tv_sec = time(NULL) + 2; hooks_wait_expire.tv_nsec = 0; flow_checks_condvar.timedWait(&hooks_wait_expire); #endif } } /* Make sure all alerts have been dequeued and processed */ dequeueFlowAlertsFromChecks(0 /* unlimited budget */); ntop->getTrace()->traceEvent( TRACE_NORMAL, "Flow alerts dump thread terminated for %s", get_name()); } /* **************************************************** */ void NetworkInterface::hostAlertsDequeueLoop() { u_int64_t n; char buf[16]; snprintf(buf, sizeof(buf), "ntopng-%d-hcheck", get_id()); Utils::setThreadName(buf); ntop->getTrace()->traceEvent(TRACE_NORMAL, "Started host user script hooks loop on interface '%s' [id: %u]...", get_description(), get_id()); /* Wait until it starts up */ while (!isRunning()) { /* Control-C during startup */ if (ntop->getGlobals()->isShutdownRequested()) return; _usleep(10000); } /* Now operational */ while (!isShuttingDown()) { /* Dequeue hosts for dump. */ n = dequeueHostAlertsFromChecks(32 /* budget */); if (n == 0) { /* No host was dequeued. Let's wait. */ #ifdef WIN32 /* If windows, sleep if nothing was done during the previous cycle. On non-windows, there's nothing do to as signal/waits are implemented to throttle the speed */ _usleep(10000); #else /* Wait for at most 1-2s. Cannot wait indefinitely as we must ensure purgeQueuedIdleHosts() gets executed, and also to exit when it's time to shutdown. */ struct timespec hooks_wait_expire; hooks_wait_expire.tv_sec = time(NULL) + 2; hooks_wait_expire.tv_nsec = 0; host_checks_condvar.timedWait(&hooks_wait_expire); #endif } } /* Make sure all alerts have been dequeued and processed */ dequeueHostAlertsFromChecks(0 /* unlimited budged */); ntop->getTrace()->traceEvent( TRACE_NORMAL, "Host alerts dump thread terminated for %s", get_name()); } /* **************************************************** */ void NetworkInterface::dumpFlowLoop() { char buf[16]; snprintf(buf, sizeof(buf), "ntopng-%d-fdump", get_id()); Utils::setThreadName(buf); ntop->getTrace()->traceEvent(TRACE_NORMAL, "Started flow dump loop on interface %s [id: %u]...", get_description(), get_id()); /* Wait until it starts up */ while (!isRunning()) { /* Control-C during startup */ if (ntop->getGlobals()->isShutdownRequested()) return; _usleep(10000); } /* Now operational */ while (!isShuttingDown()) { /* Dequeue flows for dump. Use an unlimited budget for idle flows as they're high-priority and thus we want to keep processing them if they're in the queue. */ u_int64_t n = dequeueFlowsForDump(0 /* Unlimited budget for idle flows */, MAX_ACTIVE_FLOW_QUEUE_LEN /* Limited budged for active flows */); if (n == 0) { #ifdef WIN32 _usleep(10000); #endif } } /* Make sure all flows have been dumper */ dequeueFlowsForDump(0 /* Unlimited budget for idle flows */, 0 /* Unlimited budged for active flows */); ntop->getTrace()->traceEvent(TRACE_NORMAL, "Flow dump thread completed for %s", get_name()); } /* **************************************************** */ static void *flowChecksLoop(void *ptr) { NetworkInterface *_if = (NetworkInterface *)ptr; _if->flowAlertsDequeueLoop(); return (NULL); } /* **************************************************** */ static void *hostChecksLoop(void *ptr) { NetworkInterface *_if = (NetworkInterface *)ptr; _if->hostAlertsDequeueLoop(); return (NULL); } /* **************************************************** */ static void *flowDumper(void *ptr) { NetworkInterface *_if = (NetworkInterface *)ptr; _if->dumpFlowLoop(); return (NULL); } /* **************************************************** */ void NetworkInterface::startFlowDumping() { idleFlowsToDump = new (std::nothrow)SPSCQueue(MAX_IDLE_FLOW_QUEUE_LEN, "idleFlowsToDump"); activeFlowsToDump = new (std::nothrow)SPSCQueue(MAX_ACTIVE_FLOW_QUEUE_LEN, "activeFlowsToDump"); ntop->getPrefs()->do_dump_flows_on_es() || ntop->getPrefs()->do_dump_flows_on_syslog(); if (!isViewed()) { /* Do not spawn the dumper thread for viewed interfaces - it's the view interface that has the dumper thread */ if (idleFlowsToDump && activeFlowsToDump) { pthread_create(&flowDumpLoop, NULL, flowDumper, (void *)this); flowDumpLoopCreated = true; } else { if (idleFlowsToDump) { delete idleFlowsToDump; idleFlowsToDump = NULL; } if (activeFlowsToDump) { delete activeFlowsToDump; activeFlowsToDump = NULL; } } } } /* **************************************************** */ void NetworkInterface::startPacketPolling() { if (pollLoopCreated) { if ((cpu_affinity != -1) && (ntop->getNumCPUs() > 1)) { if (Utils::setThreadAffinity(pollLoop, cpu_affinity)) ntop->getTrace()->traceEvent(TRACE_WARNING, "Couldn't set affinity of interface %s to core %d", get_description(), cpu_affinity); else ntop->getTrace()->traceEvent(TRACE_NORMAL, "Setting affinity of interface %s to core %d", get_description(), cpu_affinity); } #ifdef __linux__ char buf[16]; snprintf(buf, sizeof(buf), "ntopng-%d-pkt", get_id()); pthread_setname_np(pollLoop, buf); #endif } ntop->getTrace()->traceEvent(TRACE_NORMAL, "Started packet polling on interface '%s' [id: %u]...", get_description(), get_id()); running = true; } /* **************************************************** */ void NetworkInterface::shutdown() { void *res; if (running) { running = false; if (pollLoopCreated) pthread_join(pollLoop, &res); /* purgeIdle one last time to make sure all entries will be marked as idle */ purgeIdle(time(NULL), true, true); /* stop host/flow alerts dump threads */ shutting_down = true; /* Shut down dump threads (after purging flows/hosts to flush engaged * alerts to the database) */ if (flowDumpLoopCreated) pthread_join(flowDumpLoop, &res); if (flowAlertsDequeueLoopCreated) pthread_join(flowChecksLoop, &res); if (hostAlertsDequeueLoopCreated) pthread_join(hostChecksLoop, &res); if (db) db->flush(); } } /* **************************************************** */ void NetworkInterface::cleanup() { if (this != ntop->getSystemInterface()) ntop->getTrace()->traceEvent(TRACE_NORMAL, "Cleanup interface %s", get_description()); next_idle_flow_purge = next_idle_host_purge = 0; cpu_affinity = -1, has_vlan_packets = false, has_ebpf_events = false, has_mac_addresses = false; has_seen_dhcp_addresses = false; running = false, inline_interface = false; has_seen_containers = false, has_seen_pods = false; has_external_alerts = false; getStats()->cleanup(); if (flows_hash) flows_hash->cleanup(); if (hosts_hash) hosts_hash->cleanup(); if (ases_hash) ases_hash->cleanup(); if (obs_hash) obs_hash->cleanup(); if (countries_hash) countries_hash->cleanup(); if (vlans_hash) vlans_hash->cleanup(); if (macs_hash) macs_hash->cleanup(); } /* **************************************************** */ /* Used by ViewInterface to find hosts bound to flows in the ViewInterface */ void NetworkInterface::findFlowHosts(int32_t iface_idx, u_int16_t vlan_id, u_int16_t observation_domain_id, u_int32_t private_flow_id, Mac *src_mac, IpAddress *_src_ip, Host **src, Mac *dst_mac, IpAddress *_dst_ip, Host **dst) { if (!hosts_hash) { *src = *dst = NULL; return; } INTERFACE_PROFILING_SECTION_ENTER( "NetworkInterface::findFlowHosts: hosts_hash->get", 3); /* Do not look on sub interfaces, Flows are always created in the same * interface of its hosts */ (*src) = hosts_hash->get(vlan_id, _src_ip, src_mac, true /* Inline call */, observation_domain_id); INTERFACE_PROFILING_SECTION_EXIT(3); if ((*src) == NULL) { if (!hosts_hash->hasEmptyRoom()) { *src = *dst = NULL; has_too_many_hosts = true; return; } if (_src_ip && (_src_ip->isLocalHost() || _src_ip->isLocalInterfaceAddress() || ntop->isInLocalASN(_src_ip))) { INTERFACE_PROFILING_SECTION_ENTER("NetworkInterface::findFlowHosts: new LocalHost", 4); (*src) = new (std::nothrow)LocalHost(this, iface_idx, src_mac, vlan_id, observation_domain_id, _src_ip); INTERFACE_PROFILING_SECTION_EXIT(4); } else { INTERFACE_PROFILING_SECTION_ENTER("NetworkInterface::findFlowHosts: new RemoteHost", 5); (*src) = new (std::nothrow)RemoteHost(this, iface_idx, src_mac, vlan_id, observation_domain_id, _src_ip); INTERFACE_PROFILING_SECTION_EXIT(5); } if (*src) { INTERFACE_PROFILING_SECTION_ENTER("NetworkInterface::findFlowHosts: hosts_hash->add", 6); bool add_res = hosts_hash->add(*src, false /* Don't lock, we're inline with the purgeIdle */); INTERFACE_PROFILING_SECTION_EXIT(6); if (!add_res) { // ntop->getTrace()->traceEvent(TRACE_WARNING, "Too many hosts in interface %s", ifname); delete *src; *src = *dst = NULL; has_too_many_hosts = true; return; } has_too_many_hosts = false; } } /* ***************************** */ INTERFACE_PROFILING_SECTION_ENTER("NetworkInterface::findFlowHosts: hosts_hash->get", 3); (*dst) = hosts_hash->get(vlan_id, _dst_ip, dst_mac, true /* Inline call */, observation_domain_id); INTERFACE_PROFILING_SECTION_EXIT(3); if ((*dst) == NULL) { if (!hosts_hash->hasEmptyRoom()) { *dst = NULL; has_too_many_hosts = true; return; } if (_dst_ip && (_dst_ip->isLocalHost() || _dst_ip->isLocalInterfaceAddress() || ntop->isInLocalASN(_dst_ip))) { INTERFACE_PROFILING_SECTION_ENTER("NetworkInterface::findFlowHosts: new LocalHost", 4); (*dst) = new (std::nothrow)LocalHost(this, iface_idx, dst_mac, vlan_id, observation_domain_id, _dst_ip); INTERFACE_PROFILING_SECTION_EXIT(4); } else { INTERFACE_PROFILING_SECTION_ENTER("NetworkInterface::findFlowHosts: new RemoteHost", 5); (*dst) = new (std::nothrow)RemoteHost(this, iface_idx, dst_mac, vlan_id, observation_domain_id, _dst_ip); INTERFACE_PROFILING_SECTION_EXIT(5); } if (*dst) { INTERFACE_PROFILING_SECTION_ENTER("NetworkInterface::findFlowHosts: hosts_hash->add", 6); bool add_res = hosts_hash->add(*dst, false /* Don't lock, we're inline with the purgeIdle */); INTERFACE_PROFILING_SECTION_EXIT(6); if (!add_res) { // ntop->getTrace()->traceEvent(TRACE_WARNING, "Too many hosts in // interface %s", ifname); delete *dst; *dst = NULL; has_too_many_hosts = true; return; } has_too_many_hosts = false; } } } /* **************************************************** */ bool NetworkInterface::viewEnqueue(time_t t, Flow *f) { /* Enqueue is only performed when the interface is 'viewed'. Enqueue needs to know the viewed interface id. */ #ifdef NTOPNG_PRO if (isViewed()) return viewedBy()->viewEnqueue(t, f, getViewedId()); #endif return false; } /* **************************************************** */ bool NetworkInterface::checkPeriodicStatsUpdateTime(const struct timeval *tv) { float diff = Utils::msTimevalDiff(tv, &last_periodic_stats_update) / 1000; if (diff < 0 /* Need a reset */ || diff >= periodicStatsUpdateFrequency() || read_from_pcap_dump_done()) { memcpy(&last_periodic_stats_update, tv, sizeof(last_periodic_stats_update)); return true; } return false; } /* **************************************************** */ u_int32_t NetworkInterface::periodicStatsUpdateFrequency() const { return ntop->getPrefs()->get_housekeeping_frequency(); } /* **************************************************** */ struct timeval NetworkInterface::periodicUpdateInitTime() const { struct timeval tv; if (getIfType() != interface_type_PCAP_DUMP) gettimeofday(&tv, NULL); else tv.tv_sec = last_pkt_rcvd, tv.tv_usec = 0; return tv; } /* **************************************************** */ u_int32_t NetworkInterface::getFlowMaxIdle() { return (ntop->getPrefs()->get_pkt_ifaces_flow_max_idle()); } /* **************************************************** */ void NetworkInterface::periodicStatsUpdate() { #if 0 ntop->getTrace()->traceEvent(TRACE_NORMAL, "[%s][%s]", __FUNCTION__, get_name()); #endif struct timeval tv = periodicUpdateInitTime(); if (db) db->updateStats(&tv); checkReloadHostsBroadcastDomain(); if (!checkPeriodicStatsUpdateTime(&tv)) return; /* Not yet the time to perform an update */ #ifdef NTOPNG_PRO if (getHostPools()) getHostPools()->checkPoolsStatsReset(); #endif updatePacketsStats(); bytes_thpt.updateStats(&tv, getNumBytes()); pkts_thpt.updateStats(&tv, getNumPackets()); ethStats.updateStats(&tv); download_stats->addPoint((u_int32_t)ethStats.getIngressBytesThpt() / 1000); upload_stats->addPoint((u_int32_t)ethStats.getEgressBytesThpt() / 1000); /* Use KB instead of Bytes */ if (ndpiStats) ndpiStats->updateStats(&tv); if (ntop->getGlobals()->isShutdownRequested()) return; #ifdef PERIODIC_STATS_UPDATE_DEBUG_TIMING ntop->getTrace()->traceEvent(TRACE_NORMAL, "MySQL dump took %d seconds", time(NULL) - tdebug.tv_sec); gettimeofday(&tdebug, NULL); #endif if (host_pools) host_pools->updateStats(&tv); for (u_int32_t network_id = 0; network_id < ntop->getNumLocalNetworks(); network_id++) { NetworkStats *ns = getNetworkStats(network_id); if(ns != NULL) ns->updateStats(&tv); } #ifdef PERIODIC_STATS_UPDATE_DEBUG_TIMING ntop->getTrace()->traceEvent(TRACE_NORMAL, "Timeseries update took %d seconds", time(NULL) - tdebug.tv_sec); gettimeofday(&tdebug, NULL); #endif #ifdef PERIODIC_STATS_UPDATE_DEBUG_TIMING gettimeofday(&tdebug, NULL); ntop->getTrace()->traceEvent(TRACE_NORMAL, "Stats update done [took: %d]", tdebug.tv_sec - tdebug_init.tv_sec); #endif #ifdef NTOPNG_PRO if(tv.tv_sec >= nextMinPeriodicUpdate) { updateBehaviorStats(&tv); nextMinPeriodicUpdate = tv.tv_sec + MIN_IFACE_BEHAVIOR_REFRESH; } if (tv.tv_sec >= next5MinPeriodicUpdate) { /* 5 minute periodic update */ if (sMap) sMap->purgeIdle(next5MinPeriodicUpdate); if (pMap) pMap->purgeIdle(next5MinPeriodicUpdate); next5MinPeriodicUpdate = tv.tv_sec + IFACE_BEHAVIOR_REFRESH; } #endif } /* **************************************************** */ /* Frees the memory (destructors) of all idle hash table entries except flows */ u_int64_t NetworkInterface::purgeQueuedIdleEntries() { u_int64_t num_purged = 0; #if 0 ntop->getTrace()->traceEvent(TRACE_NORMAL, "Updating hash tables [%s]", get_name()); #endif GenericHash *ghs[] = {hosts_hash, flows_hash, ases_hash, countries_hash, vlans_hash, macs_hash, obs_hash}; /* Delete all idle entries */ for (u_int i = 0; i < sizeof(ghs) / sizeof(ghs[0]); i++) { if (ghs[i]) num_purged += ghs[i]->purgeQueuedIdleEntries(); } return num_purged; } /* **************************************************** */ struct update_host_pool_l7policy { bool update_pool_id; bool update_l7policy; }; static bool update_host_host_pool_l7policy(GenericHashEntry *node, void *user_data, bool *matched) { Host *h = (Host *)node; update_host_pool_l7policy *up = (update_host_pool_l7policy *)user_data; #ifdef HOST_POOLS_DEBUG char buf[128]; u_int16_t cur_pool_id = h->get_host_pool(); #endif *matched = true; if (up->update_pool_id) h->updateHostPool(false /* Not inline with traffic processing */); #ifdef NTOPNG_PRO if (up->update_l7policy) h->resetBlockedTrafficStatus(); #endif #ifdef HOST_POOLS_DEBUG ntop->getTrace()->traceEvent( TRACE_NORMAL, "Going to refresh pool for %s " "[refresh pool id: %i] " "[refresh l7policy: %i] " "[host pool id before refresh: %i] " "[host pool id after refresh: %i] ", h->get_ip()->print(buf, sizeof(buf)), up->update_pool_id ? 1 : 0, up->update_l7policy ? 1 : 0, cur_pool_id, h->get_host_pool()); #endif return (false); /* false = keep on walking */ } static bool update_l2_device_host_pool(GenericHashEntry *node, void *user_data, bool *matched) { Mac *m = (Mac *)node; #ifdef HOST_POOLS_DEBUG u_int16_t cur_pool_id = m->get_host_pool(); #endif *matched = true; m->updateHostPool(false /* Not inline with traffic processing */); #ifdef HOST_POOLS_DEBUG char buf[24]; ntop->getTrace()->traceEvent(TRACE_NORMAL, "Going to refresh pool for %s " "[host pool id before refresh: %i] " "[host pool id after refresh: %i] ", Utils::formatMac(m->get_mac(), buf, sizeof(buf)), cur_pool_id, m->get_host_pool()); #endif return (false); /* false = keep on walking */ } /* **************************************************** */ void NetworkInterface::refreshHostPools() { u_int32_t begin_slot = 0; bool walk_all = true; struct update_host_pool_l7policy update_host; update_host.update_pool_id = true; update_host.update_l7policy = false; #ifdef NTOPNG_PRO if (is_bridge_interface() && getL7Policer()) { /* Every pool is associated with a set of L7 rules so a refresh must be triggered to seal this association */ getL7Policer()->refreshL7Rules(); /* Must refresh host l7policies as a change in the host pool id may determine an l7policy change for that host */ update_host.update_l7policy = true; } #endif if (hosts_hash) { begin_slot = 0; walker(&begin_slot, walk_all, walker_hosts, update_host_host_pool_l7policy, &update_host); } if (macs_hash) { begin_slot = 0; walker(&begin_slot, walk_all, walker_macs, update_l2_device_host_pool, NULL); } #ifdef HAVE_NEDGE if (update_host.update_l7policy) updateFlowsL7Policy(); #endif } /* **************************************************** */ static bool count_open_server_ports(GenericHashEntry *node, void *user_data, bool *matched) { Host *h = (Host *)node; std::unordered_map *count = (std::unordered_map *)user_data; std::unordered_map::iterator it; for (u_int i = 0; i < 2; i++) { std::unordered_map *ports; if (i == 0) ports = h->getServerPorts(true); else ports = h->getServerPorts(false); if (ports) { for (it = ports->begin(); it != ports->end(); ++it) { u_int16_t port = it->first; std::unordered_map::iterator it1 = count->find(port); if (it1 == count->end()) (*count)[port] = 1; else (*count)[port] = (*count)[port] + 1; } } } *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ void NetworkInterface::localHostsServerPorts(lua_State *vm) { u_int32_t begin_slot = 0; std::unordered_map count; std::unordered_map::iterator it; walker(&begin_slot, true /* walk_all */, walker_hosts, count_open_server_ports, &count); lua_newtable(vm); for (it = count.begin(); it != count.end(); ++it) { char port[32]; snprintf(port, sizeof(port), "%u", it->first); lua_push_uint32_table_entry(vm, port, it->second); } } /* **************************************************** */ #ifdef HAVE_NEDGE static bool update_flow_l7_policy(GenericHashEntry *node, void *user_data, bool *matched) { Flow *f = (Flow *)node; *matched = true; f->updateFlowShapers(); return (false); /* false = keep on walking */ } /* **************************************************** */ void NetworkInterface::updateHostsL7Policy(u_int16_t host_pool_id) { u_int32_t begin_slot = 0; bool walk_all = true; if (isView()) return; struct update_host_pool_l7policy update_host; update_host.update_pool_id = false; update_host.update_l7policy = true; /* Pool id didn't change here so there's no need to walk on the macs as policies are set on the hosts */ if (hosts_hash) walker(&begin_slot, walk_all, walker_hosts, update_host_host_pool_l7policy, &update_host); } /* **************************************************** */ void NetworkInterface::updateFlowsL7Policy() { u_int32_t begin_slot = 0; bool walk_all = true; if (isView()) return; if (flows_hash) walker(&begin_slot, walk_all, walker_flows, update_flow_l7_policy, NULL); } /* **************************************************** */ struct resetPoolsStatsData { struct tm *now; u_int16_t pool_filter; }; static bool flow_recheck_quota_walker(GenericHashEntry *flow, void *user_data, bool *matched) { Flow *f = (Flow *)flow; struct tm *now = ((struct resetPoolsStatsData *)user_data)->now; *matched = true; f->recheckQuota(now); return (false); /* false = keep on walking */ } static bool host_reset_quotas(GenericHashEntry *host, void *user_data, bool *matched) { Host *h = (Host *)host; u_int16_t pool_filter = ((struct resetPoolsStatsData *)user_data)->pool_filter; if ((pool_filter == (u_int16_t)-1) || (h->get_host_pool() == pool_filter)) { *matched = true; h->resetQuotaStats(); h->resetBlockedTrafficStatus(); } return (false); /* false = keep on walking */ } #endif /* **************************************************** */ #ifdef NTOPNG_PRO void NetworkInterface::resetPoolsStats(u_int16_t pool_filter) { struct tm now; time_t t_now = time(NULL); localtime_r(&t_now, &now); if (host_pools) { host_pools->resetPoolsStats(pool_filter); #ifdef HAVE_NEDGE u_int32_t begin_slot = 0; bool walk_all = true; struct resetPoolsStatsData data; data.pool_filter = pool_filter; data.now = &now; walker(&begin_slot, walk_all, walker_hosts, host_reset_quotas, &data); begin_slot = 0; walker(&begin_slot, walk_all, walker_flows, flow_recheck_quota_walker, &data); #endif } } #endif /* **************************************************** */ struct host_find_info { char *host_to_find; u_int16_t vlan_id; u_int16_t observationPointId; Host *h; }; /* **************************************************** */ struct as_find_info { u_int32_t asn; AutonomousSystem *as; }; /* **************************************************** */ struct obs_point_find_info { u_int32_t obs_point_id; ObservationPoint *obs_point; }; /* **************************************************** */ struct vlan_find_info { u_int16_t vlan_id; VLAN *vl; }; /* **************************************************** */ struct country_find_info { const char *country_id; Country *country; }; /* **************************************************** */ struct mac_find_info { u_int8_t mac[6]; u_int16_t vlan_id; Mac *m; DeviceType dtype; lua_State *vm; }; /* **************************************************** */ static bool find_host_by_name(GenericHashEntry *h, void *user_data, bool *matched) { struct host_find_info *info = (struct host_find_info *)user_data; Host *host = (Host *)h; char ip_buf[32], name_buf[96]; name_buf[0] = '\0'; #ifdef DEBUG char buf[64]; ntop->getTrace()->traceEvent( TRACE_WARNING, "[%s][%s][%s]", host->get_ip() ? host->get_ip()->print(buf, sizeof(buf)) : "", host->get_name(), info->host_to_find); #endif if ((info->h == NULL) #if 0 /* Commented out because of (***) */ && (host->get_observation_point_id() == info->observationPointId) #endif && (host->get_vlan_id() == info->vlan_id)) { host->get_name(name_buf, sizeof(name_buf), false); if (strlen(name_buf) == 0 && host->get_ip()) { char *ipaddr = host->get_ip()->print(ip_buf, sizeof(ip_buf)); int rc = ntop->getRedis()->getAddress( ipaddr, name_buf, sizeof(name_buf), false /* Don't resolve it if not known */); if (rc == 0 /* found */ && strcmp(ipaddr, name_buf)) host->setResolvedName(name_buf); else name_buf[0] = '\0'; } if (!strcmp(name_buf, info->host_to_find)) { info->h = host; *matched = true; return (true); /* found */ } } return (false); /* false = keep on walking */ } /* **************************************************** */ static bool find_as_by_asn(GenericHashEntry *he, void *user_data, bool *matched) { struct as_find_info *info = (struct as_find_info *)user_data; AutonomousSystem *as = (AutonomousSystem *)he; if ((info->as == NULL) && info->asn == as->get_asn()) { info->as = as; *matched = true; return (true); /* found */ } return (false); /* false = keep on walking */ } /* **************************************************** */ static bool find_obs_point_by_id(GenericHashEntry *he, void *user_data, bool *matched) { struct obs_point_find_info *info = (struct obs_point_find_info *)user_data; ObservationPoint *_obs_point = (ObservationPoint *)he; if ((info->obs_point == NULL) && info->obs_point_id == _obs_point->getObsPoint()) { info->obs_point = _obs_point; *matched = true; return (true); /* found */ } return (false); /* false = keep on walking */ } /* **************************************************** */ static bool find_country(GenericHashEntry *he, void *user_data, bool *matched) { struct country_find_info *info = (struct country_find_info *)user_data; Country *country = (Country *)he; if ((info->country == NULL) && !strcmp(info->country_id, country->get_country_name())) { info->country = country; *matched = true; return (true); /* found */ } return (false); /* false = keep on walking */ } /* **************************************************** */ static bool find_vlan_by_vlan_id(GenericHashEntry *he, void *user_data, bool *matched) { struct vlan_find_info *info = (struct vlan_find_info *)user_data; VLAN *vl = (VLAN *)he; if ((info->vl == NULL) && info->vlan_id == vl->get_vlan_id()) { info->vl = vl; *matched = true; return (true); /* found */ } return (false); /* false = keep on walking */ } /* **************************************************** */ Host *NetworkInterface::getHostByIP(IpAddress *ip, u_int16_t vlan_id, u_int16_t observation_point_id, bool isInlineCall) { Host *h; h = hosts_hash ? hosts_hash->get(vlan_id, ip, NULL, isInlineCall, observation_point_id) : NULL; return (h); } /* **************************************************** */ Host *NetworkInterface::getHost(char *host_ip, u_int16_t vlan_id, u_int16_t observation_point_id, bool isInlineCall) { struct in_addr a4; struct in6_addr a6; Host *h = NULL; if (!host_ip) return (NULL); /* Check if address is invalid */ if ((inet_pton(AF_INET, (const char *)host_ip, &a4) == 0) && (inet_pton(AF_INET6, (const char *)host_ip, &a6) == 0)) { /* Looks like a symbolic name */ struct host_find_info info; u_int32_t begin_slot = 0; bool walk_all = true; memset(&info, 0, sizeof(info)); info.host_to_find = host_ip, info.vlan_id = vlan_id; info.observationPointId = observation_point_id; walker(&begin_slot, walk_all, walker_hosts, find_host_by_name, (void *)&info); h = info.h; } else { IpAddress ip; ip.set(host_ip); h = getHostByIP(&ip, vlan_id, observation_point_id, isInlineCall); } return (h); } /* **************************************************** */ #ifdef NTOPNG_PRO #ifndef HAVE_NEDGE static bool update_flow_profile(GenericHashEntry *h, void *user_data, bool *matched) { Flow *flow = (Flow *)h; flow->updateProfile(); *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ void NetworkInterface::updateFlowProfiles() { if (isView()) return; if (ntop->getPro()->has_valid_license()) { FlowProfiles *newP; u_int32_t begin_slot = 0; bool walk_all = true; if (shadow_flow_profiles) { delete shadow_flow_profiles; shadow_flow_profiles = NULL; } flow_profiles->dumpCounters(); shadow_flow_profiles = flow_profiles, newP = new (std::nothrow) FlowProfiles(id); newP->loadProfiles(); /* and reload */ flow_profiles = newP; /* Overwrite the current profiles */ if (flows_hash) walker(&begin_slot, walk_all, walker_flows, update_flow_profile, NULL); } } #endif #endif /* **************************************************** */ bool NetworkInterface::isHostActive(AddressTree *allowed_hosts, char *host_ip, u_int16_t vlan_id) { Host *h; h = findHostByIP(allowed_hosts, host_ip, vlan_id, 0 /* any observation point */); return !!h; } /* **************************************************** */ bool NetworkInterface::getHostInfo(lua_State *vm, AddressTree *allowed_hosts, char *host_ip, u_int16_t vlan_id) { Host *h; bool ret; h = findHostByIP(allowed_hosts, host_ip, vlan_id, getLuaVMUservalue(vm, observationPointId)); if (h) { h->lua(vm, allowed_hosts, true, true, true, false); ret = true; } else ret = false; return ret; } /* **************************************************** */ bool NetworkInterface::getHostMinInfo(lua_State *vm, AddressTree *allowed_hosts, char *host_ip, u_int16_t vlan_id, bool only_ndpi_stats) { Host *h; bool ret; h = findHostByIP(allowed_hosts, host_ip, vlan_id, getLuaVMUservalue(vm, observationPointId)); if (h) { lua_newtable(vm); if (only_ndpi_stats) h->lua_get_ndpi_info(vm); else h->lua_get_min_info(vm); ret = true; } else ret = false; return ret; } /* **************************************************** */ void NetworkInterface::checkReloadHostsBroadcastDomain() { time_t bcast_domains_last_update = bcast_domains->getLastUpdate(); if (hosts_bcast_domain_last_update < bcast_domains_last_update) reload_hosts_bcast_domain = true, hosts_bcast_domain_last_update = bcast_domains_last_update; else if (reload_hosts_bcast_domain) reload_hosts_bcast_domain = false; } /* **************************************************** */ void NetworkInterface::checkPointHostTalker(lua_State *vm, char *host_ip, u_int16_t vlan_id) { Host *h; if (host_ip && (h = getHost(host_ip, vlan_id, getLuaVMUservalue(vm, observationPointId), false /* Not an inline call */))) h->checkpoint(vm); else lua_pushnil(vm); } /* **************************************************** */ Host *NetworkInterface::findHostByIP(AddressTree *allowed_hosts, char *host_ip, u_int16_t vlan_id, u_int16_t observationPointId) { Host *h = NULL; if (host_ip == NULL) return (NULL); h = getHost(host_ip, vlan_id, observationPointId, false /* Not an inline call */); if (h == NULL) return (NULL); if (allowed_hosts && !h->match(allowed_hosts)) return (NULL); return h; } /* **************************************************** */ struct flowHostRetrieveList { Flow *flow; /* Value */ Host *hostValue; Mac *macValue; VLAN *vlanValue; AutonomousSystem *asValue; ObservationPoint *obsPointValue; Country *countryVal; u_int64_t numericValue; const char *stringValue; IpAddress *ipValue; }; struct flowHostRetriever { /* Search criteria */ AddressTree *allowed_hosts; Host *host, *talking_with_host, *server, *client; u_int16_t observationPointId; u_int8_t *mac, bridge_iface_idx; char *manufacturer; bool sourceMacsOnly, dhcpHostsOnly; time_t min_first_seen; char *country; char *flow_info; int ndpi_proto; /* Not used in flow_search_walker */ TrafficType traffic_type; /* Not used in flow_search_walker */ sortField sorter; TcpFlowStateFilter tcp_flow_state_filter; LocationPolicy location; /* Not used in flow_search_walker */ u_int8_t ipVersionFilter; /* Not used in flow_search_walker */ bool filteredHosts; /* Not used in flow_search_walker */ bool blacklistedHosts; /* Not used in flow_search_walker */ bool anomalousOnly; /* Not used in flow_search_walker */ bool dhcpOnly; /* Not used in flow_search_walker */ const AddressTree *cidr_filter; /* Not used in flow_search_walker */ u_int16_t vlan_id; ndpi_os osFilter; u_int32_t device_ip; u_int32_t asnFilter; u_int32_t uidFilter; u_int32_t pidFilter; int32_t networkFilter; u_int16_t poolFilter; u_int8_t devtypeFilter; u_int8_t locationFilter; int32_t iface_index; /* Return values */ u_int32_t maxNumEntries, actNumEntries; u_int64_t totBytesSent, totBytesRcvd, totThpt; struct flowHostRetrieveList *elems; #ifdef NTOPNG_PRO QoEStats *qoe; #endif bool only_traffic_stats; /* Used by getActiveFlowsStats */ nDPIStats *ndpi_stats; FlowStats *stats; /* Paginator */ Paginator *pag; }; /* **************************************************** */ static bool flow_matches(Flow *f, struct flowHostRetriever *retriever) { int ndpi_proto_master_proto, ndpi_proto_app_proto, ndpi_cat; u_int16_t port; int32_t local_network_id; u_int16_t vlan_id = 0; u_int16_t cli_pool, srv_pool, pool_filter; AlertLevelGroup flow_status_severity_filter = alert_level_group_none; ndpi_patricia_node_t *srv_target_node = NULL, *cli_target_node = NULL; IpAddress *cli_ip = (IpAddress *)f->get_srv_ip_addr(); IpAddress *srv_ip = (IpAddress *)f->get_cli_ip_addr(); u_int16_t alert_type_filter; u_int8_t ip_version; u_int8_t l4_protocol; u_int8_t *mac_filter; LocationPolicy client_policy; LocationPolicy server_policy; TcpFlowStateFilter tcp_flow_state_filter; bool unicast, unidirectional, alerted_flows, periodic_flows, cli_pool_found = false, srv_pool_found = false; u_int32_t asn_filter; char *username_filter; char *pidname_filter; char *wlan_ssid_filter; u_int32_t deviceIP = 0; int32_t iface_index = -1; u_int32_t inIndex, outIndex; u_int8_t icmp_type, icmp_code, dscp_filter; #ifdef NTOPNG_PRO u_int8_t qoe_filter; #ifndef HAVE_NEDGE char *traffic_profile_filter; #endif #endif char *container_filter, *pod_filter; #ifdef HAVE_NEDGE bool filtered_flows; #endif if (f && (!f->idle())) { if (f->get_observation_point_id() != retriever->observationPointId) { #if 0 ntop->getTrace()->traceEvent(TRACE_WARNING, "Skipping VLAN: %u-%u / observationPointId: %u-%u", f->get_vlan_id(), vlan_id, f->get_observation_point_id(), retriever->observationPointId); #endif return (false); } if (retriever->server) { if (!f->getInterface()->isViewed()) { if (retriever->server != f->get_srv_host()) { return (false); } } else { if (!(retriever->server->get_ip()->equal(f->get_srv_ip_addr()) && retriever->server->get_vlan_id() == f->get_vlan_id())) return (false); } } if (retriever->talking_with_host) { if (!f->getInterface()->isViewed()) { if (retriever->talking_with_host != f->get_cli_host() && retriever->talking_with_host != f->get_srv_host()) { return (false); } } else { if (!(retriever->talking_with_host->get_ip()->equal( f->get_cli_ip_addr()) && retriever->talking_with_host->get_vlan_id() == f->get_vlan_id()) && !(retriever->talking_with_host->get_ip()->equal( f->get_srv_ip_addr()) && retriever->talking_with_host->get_vlan_id() == f->get_vlan_id())) return (false); } } if (retriever->client) { if (!f->getInterface()->isViewed()) { if (retriever->client != f->get_cli_host()) { return (false); } } else { if (!(retriever->client->get_ip()->equal(f->get_cli_ip_addr()) && retriever->client->get_vlan_id() == f->get_vlan_id())) return (false); } } if (retriever->flow_info) { if (strcmp(retriever->flow_info, f->getFlowInfo(false).c_str())) return (false); } if (retriever->host) { if (!f->getInterface()->isViewed()) { /* For non-viewed interfaces it is safe to just check on pointers equality. Indeed, the retriever->host has been obtained with getHost(), which has returned the same pointer also used by the flow to identify its client / server hosts. */ if ((retriever->host != f->get_cli_host()) && (retriever->host != f->get_srv_host())) { #if 0 if(f->get_cli_host() && f->get_srv_host()) { char buf[128], buf2[128], buf3[128]; ntop->getTrace()->traceEvent(TRACE_WARNING, "Skipping Host: %s (%p) - %s (%p) / Talking Host: %s (%p)", f->get_cli_host() ? f->get_cli_host()->get_hostkey(buf, sizeof(buf)) : "NULL", f->get_cli_host(), f->get_srv_host() ? f->get_srv_host()->get_hostkey(buf2, sizeof(buf2)) : "NULL", f->get_srv_host(), retriever->host ? retriever->host->get_hostkey(buf3, sizeof(buf3)) : "NULL", retriever->host); } #endif return (false); } } else { /* In case of view interfaces, hosts are in the view interface whereas flows are in the underlying viewed interfaces so in this case it is not possible to just check on pointers equality. This need to use the retriever->host which comes from the view interface to retrieve an IpAddress, and check it against the flow ip address, along with the vlan. Indeed, flow ip addresses exist also when a flow doesn't have Host* as in the case of viewed interfaces. */ if (!(retriever->host->get_ip()->equal(f->get_cli_ip_addr()) && retriever->host->get_vlan_id() == f->get_vlan_id()) && !(retriever->host->get_ip()->equal(f->get_srv_ip_addr()) && retriever->host->get_vlan_id() == f->get_vlan_id())) return (false); } } if (retriever->pag && retriever->pag->l7protoFilter(&ndpi_proto_master_proto, &ndpi_proto_app_proto)) { bool is_ok = false; /* ntop->getTrace()->traceEvent(TRACE_WARNING, "Filtering app protocol: %u, master protocol: %u | Flow app protocol: %u, master protocol: %u", ndpi_proto_app_proto, ndpi_proto_master_proto, f->get_detected_protocol().proto.app_protocol, f->get_detected_protocol().proto.master_protocol); */ /* Case where both protocols are unknown */ if(ndpi_proto_master_proto == NDPI_PROTOCOL_UNKNOWN && ndpi_proto_app_proto == NDPI_PROTOCOL_UNKNOWN) { if( (ndpi_proto_master_proto == f->get_detected_protocol().proto.master_protocol) && (ndpi_proto_app_proto == f->get_detected_protocol().proto.app_protocol) ) /* We're good */ is_ok = true; } else if(ndpi_proto_master_proto == NDPI_PROTOCOL_UNKNOWN || ndpi_proto_app_proto == NDPI_PROTOCOL_UNKNOWN) { /* Case where one is unknown, the other can match both master and app of the flow */ if((ndpi_proto_master_proto == NDPI_PROTOCOL_UNKNOWN && (ndpi_proto_app_proto == f->get_detected_protocol().proto.app_protocol || ndpi_proto_app_proto == f->get_detected_protocol().proto.master_protocol)) || (ndpi_proto_app_proto == NDPI_PROTOCOL_UNKNOWN && (ndpi_proto_master_proto == f->get_detected_protocol().proto.app_protocol || ndpi_proto_master_proto == f->get_detected_protocol().proto.master_protocol))) /* We're good */ is_ok = true; } else { /* Case where both are not unknown */ if (((ndpi_proto_master_proto == NDPI_PROTOCOL_UNKNOWN) || (ndpi_proto_master_proto == f->get_detected_protocol().proto.master_protocol)) && ((ndpi_proto_app_proto == NDPI_PROTOCOL_UNKNOWN) || (ndpi_proto_app_proto == f->get_detected_protocol().proto.app_protocol)) ) /* We're good */ is_ok = true; } if(!is_ok) return (false); } if (retriever->pag && retriever->pag->l7categoryFilter(&ndpi_cat) && f->get_protocol_category() != ndpi_cat) return (false); if (retriever->pag && retriever->pag->tcpFlowStateFilter(&tcp_flow_state_filter) && ((f->get_protocol() != IPPROTO_TCP) || (tcp_flow_state_filter == tcp_flow_state_established && !f->isTCPEstablished()) || (tcp_flow_state_filter == tcp_flow_state_connecting && !f->isTCPConnecting()) || (tcp_flow_state_filter == tcp_flow_state_closed && !f->isTCPClosed()) || (tcp_flow_state_filter == tcp_flow_state_reset && !f->isTCPReset()))) return (false); if (retriever->pag && retriever->pag->ipVersion(&ip_version) && (((ip_version == 4) && (f->get_cli_ip_addr() && !f->get_cli_ip_addr()->isIPv4())) || ((ip_version == 6) && (f->get_cli_ip_addr() && !f->get_cli_ip_addr()->isIPv6())))) return (false); if (retriever->pag && retriever->pag->L4Protocol(&l4_protocol) && l4_protocol && l4_protocol != f->get_protocol()) return (false); if (retriever->pag && retriever->pag->deviceIpFilter(&deviceIP)) { if ((f->getFlowDeviceIP() != deviceIP) || (retriever->pag->inIndexFilter(&inIndex) && f->getFlowDeviceInIndex() != inIndex) || (retriever->pag->outIndexFilter(&outIndex) && f->getFlowDeviceOutIndex() != outIndex)) return (false); } if (retriever->pag && retriever->pag->containerFilter(&container_filter)) { const char *cli_container = f->getClientContainerInfo() ? f->getClientContainerInfo()->id : NULL; const char *srv_container = f->getServerContainerInfo() ? f->getServerContainerInfo()->id : NULL; if (!((cli_container && !strcmp(container_filter, cli_container)) || (srv_container && !strcmp(container_filter, srv_container)))) return (false); } if (retriever->pag && retriever->pag->podFilter(&pod_filter)) { const ContainerInfo *cli_cont = f->getClientContainerInfo(); const ContainerInfo *srv_cont = f->getServerContainerInfo(); const char *cli_pod = cli_cont && cli_cont->data_type == container_info_data_type_k8s ? cli_cont->data.k8s.pod : NULL; const char *srv_pod = srv_cont && srv_cont->data_type == container_info_data_type_k8s ? srv_cont->data.k8s.pod : NULL; if (!((cli_pod && !strcmp(pod_filter, cli_pod)) || (srv_pod && !strcmp(pod_filter, srv_pod)))) return (false); } #ifdef NTOPNG_PRO #ifndef HAVE_NEDGE if (retriever->pag && retriever->pag->trafficProfileFilter(&traffic_profile_filter) && (f->isMaskedFlow() || strcmp(traffic_profile_filter, f->get_profile_name()))) { return (false); } #endif #endif if (retriever->pag && retriever->pag->ifaceIndexFilter(&iface_index)) { if (f->getInterface() && f->getInterface()->get_id() != iface_index) { return (false); } } if (retriever->pag && retriever->pag->asnFilter(&asn_filter) && f->get_cli_host() && f->get_srv_host() && f->get_cli_host()->get_asn() != asn_filter && f->get_srv_host()->get_asn() != asn_filter) return (false); else if (retriever->pag && retriever->pag->asnFilter(&asn_filter) && f->get_cli_host() && f->get_srv_host()) ntop->getTrace()->traceEvent( TRACE_WARNING, "Filtering ASN: %u | Client ASN: %u | Server ASN: %u", asn_filter, f->get_cli_host()->get_asn(), f->get_srv_host()->get_asn()); if (retriever->pag && retriever->pag->usernameFilter(&username_filter) && (!f->get_user_name(true /* client uid */) || strcmp(f->get_user_name(true /* client uid */), username_filter)) && (!f->get_user_name(false /* server uid */) || strcmp(f->get_user_name(false /* server uid */), username_filter))) return (false); if (retriever->pag && retriever->pag->pidnameFilter(&pidname_filter) && (!f->get_proc_name(true /* client pid */) || strcmp(f->get_proc_name(true /* client pid */), pidname_filter)) && (!f->get_proc_name(false /* server pid */) || strcmp(f->get_proc_name(false /* server pid */), pidname_filter))) return (false); if (retriever->pag && retriever->pag->icmpValue(&icmp_type, &icmp_code)) { u_int8_t cur_type, cur_code; f->getICMP(&cur_code, &cur_type); if ((!f->isICMP()) || (cur_type != icmp_type) || (cur_code != icmp_code)) return (false); } if (retriever->pag && retriever->pag->wlanSSIDFilter(&wlan_ssid_filter) && (!f->getWLANSSID() || strcmp(f->getWLANSSID(), wlan_ssid_filter))) return (false); if (retriever->pag && retriever->pag->dscpFilter(&dscp_filter) && f->getCli2SrvDSCP() != dscp_filter && f->getSrv2CliDSCP() != dscp_filter) return (false); #ifdef NTOPNG_PRO if (retriever->pag && retriever->pag->QoEFilter(&qoe_filter) && ((u_int8_t) f->getQoEType()) != qoe_filter) return (false); #endif if (retriever->pag && retriever->pag->portFilter(&port) && f->get_cli_port() != port && f->get_srv_port() != port) return (false); if (retriever->pag && retriever->pag->localNetworkFilter(&local_network_id)) { if(local_network_id != (int32_t) CONST_MAX_NUM_NETWORKS + 1) { int32_t cli_local_network_id, srv_local_network_id; f->get_cli_ip_addr()->isLocalHost(&cli_local_network_id), f->get_srv_ip_addr()->isLocalHost(&srv_local_network_id); if (cli_local_network_id != local_network_id && srv_local_network_id != local_network_id) return (false); } } if (retriever->pag && retriever->pag->vlanIdFilter(&vlan_id) && (f->get_vlan_id() != vlan_id)) return (false); if (retriever->pag && retriever->pag->clientMode(&client_policy) && (((client_policy == location_local_only) && (!f->get_cli_ip_addr()->isLocalHost())) || ((client_policy == location_remote_only) && (f->get_cli_ip_addr()->isLocalHost())))) return (false); if (retriever->pag && retriever->pag->serverMode(&server_policy) && (((server_policy == location_local_only) && (!f->get_srv_ip_addr()->isLocalHost())) || ((server_policy == location_remote_only) && (f->get_srv_ip_addr()->isLocalHost())))) return (false); if (retriever->pag && retriever->pag->alertedFlows(&alerted_flows) && ((alerted_flows && !f->isFlowAlerted()) || (!alerted_flows && f->isFlowAlerted()))) return (false); if (retriever->pag && retriever->pag->periodicFlows(&periodic_flows) && ((periodic_flows && !f->isPeriodicFlow()) || (!periodic_flows && f->isPeriodicFlow()))) return (false); /* Flow Status filter */ if (retriever->pag && retriever->pag->flowStatusFilter(&alert_type_filter) && !f->getAlertsBitmap().isSetBit(alert_type_filter)) return (false); /* Flow Status severity filter */ if (retriever->pag && retriever->pag->flowStatusFilter(&flow_status_severity_filter)) { if (!f->isFlowAlerted() || f->getPredominantAlertSeverity() == alert_level_none || (flow_status_severity_filter == alert_level_group_notice_or_lower && f->getPredominantAlertSeverity() > alert_level_notice) || (flow_status_severity_filter == alert_level_group_warning && f->getPredominantAlertSeverity() != alert_level_warning) || (flow_status_severity_filter == alert_level_group_error && f->getPredominantAlertSeverity() < alert_level_error)) return (false); } #ifdef HAVE_NEDGE if (retriever->pag && retriever->pag->filteredFlows(&filtered_flows) && ((filtered_flows && f->isPassVerdict()) || (!filtered_flows && !f->isPassVerdict()))) return (false); #endif if (retriever->pag && retriever->pag->unidirectionalTraffic(&unidirectional) && ((unidirectional && !f->isOneWay()) || (!unidirectional && f->isBidirectional()))) return (false); /* Unicast: at least one between client and server is unicast address */ if (retriever->pag && retriever->pag->unicastTraffic(&unicast) && ((unicast && ((f->get_cli_ip_addr()->isMulticastAddress() || f->get_cli_ip_addr()->isBroadcastAddress() || f->get_srv_ip_addr()->isMulticastAddress() || f->get_srv_ip_addr()->isBroadcastAddress()))) || (!unicast && (!f->get_cli_ip_addr()->isMulticastAddress() && !f->get_cli_ip_addr()->isBroadcastAddress() && !f->get_srv_ip_addr()->isMulticastAddress() && !f->get_srv_ip_addr()->isBroadcastAddress())))) return (false); if (cli_ip && !f->get_cli_host()) cli_pool_found = f->getInterface()->getHostPools()->findIpPool( cli_ip, f->get_vlan_id(), &cli_pool, &cli_target_node); if (srv_ip && !f->get_srv_host()) srv_pool_found = f->getInterface()->getHostPools()->findIpPool( srv_ip, f->get_vlan_id(), &srv_pool, &srv_target_node); /* Pool filter */ if (retriever->pag && retriever->pag->poolFilter(&pool_filter) && !((f->get_cli_host() && f->get_cli_host()->get_host_pool() == pool_filter) || (f->get_srv_host() && f->get_srv_host()->get_host_pool() == pool_filter)) && !((cli_pool_found && cli_pool == pool_filter) || (srv_pool_found && srv_pool == pool_filter))) return (false); /* Mac filter - NOTE: must stay below the vlan_id filter */ if (retriever->pag && retriever->pag->macFilter(&mac_filter) && !((f->get_cli_host() && f->get_cli_host()->getMac() && f->get_cli_host()->getMac()->equal(mac_filter)) || (f->get_srv_host() && f->get_srv_host()->getMac() && f->get_srv_host()->getMac()->equal(mac_filter)))) return (false); if (f->match(retriever->allowed_hosts)) return (true); /* match */ } return (false); } /* **************************************************** */ static bool flow_search_walker(GenericHashEntry *h, void *user_data, bool *matched) { struct flowHostRetriever *retriever = (struct flowHostRetriever *)user_data; Flow *f = (Flow *)h; const TcpInfo *tcp_info; if (retriever->actNumEntries >= retriever->maxNumEntries) return (true); /* Limit reached - stop iterating */ if (flow_matches(f, retriever)) { retriever->elems[retriever->actNumEntries].flow = f; retriever->totBytesSent += f->get_bytes_cli2srv(); retriever->totBytesRcvd += f->get_bytes_srv2cli(); switch (retriever->sorter) { case column_client: if (f->getInterface()->isViewed()) { if(f->get_cli_ip_addr()) retriever->elems[retriever->actNumEntries++].ipValue = (IpAddress *)f->get_cli_ip_addr(); } else { if(f->get_cli_host()) retriever->elems[retriever->actNumEntries++].hostValue = f->get_cli_host(); } break; case column_server: if (f->getInterface()->isViewed()) { if(f->get_srv_ip_addr()) retriever->elems[retriever->actNumEntries++].ipValue = (IpAddress *)f->get_srv_ip_addr(); } else { if(f->get_srv_host()) retriever->elems[retriever->actNumEntries++].hostValue = f->get_srv_host(); } break; case column_vlan: retriever->elems[retriever->actNumEntries++].numericValue = f->get_vlan_id(); break; case column_proto_l4: retriever->elems[retriever->actNumEntries++].numericValue = f->get_protocol(); break; case column_ndpi: retriever->elems[retriever->actNumEntries++].numericValue = f->get_detected_protocol().proto.app_protocol; break; case column_protocol: retriever->actNumEntries++; break; case column_duration: retriever->elems[retriever->actNumEntries++].numericValue = f->get_duration(); break; case column_score: { u_int16_t s = f->getScore();; if(s == 0) { if(retriever->pag->a2zSortOrder()) s = 0xFFFF; else s = 0; } retriever->elems[retriever->actNumEntries++].numericValue = s; } break; case column_qoe: { u_int8_t qoe = #ifdef NTOPNG_PRO f->getQoEScore(); #else NTOP_QOE_UNKNOWN; #endif if(qoe == NTOP_QOE_UNKNOWN) { if(retriever->pag->a2zSortOrder()) qoe = 0xFF; else qoe = 0; } retriever->elems[retriever->actNumEntries++].numericValue = qoe; } break; case column_thpt: retriever->elems[retriever->actNumEntries++].numericValue = f->get_bytes_thpt(); break; case column_bytes: retriever->elems[retriever->actNumEntries++].numericValue = f->get_bytes(); break; case column_last_seen: retriever->elems[retriever->actNumEntries++].numericValue = f->get_last_seen(); break; case column_first_seen: retriever->elems[retriever->actNumEntries++].numericValue = f->get_first_seen(); break; case column_client_rtt: if ((tcp_info = f->getClientTcpInfo())) retriever->elems[retriever->actNumEntries++].numericValue = (u_int64_t)(tcp_info->rtt * 1000); else retriever->elems[retriever->actNumEntries++].numericValue = 0; break; case column_server_rtt: if ((tcp_info = f->getServerTcpInfo())) retriever->elems[retriever->actNumEntries++].numericValue = (u_int64_t)(tcp_info->rtt * 1000); else retriever->elems[retriever->actNumEntries++].numericValue = 0; break; case column_key: retriever->elems[retriever->actNumEntries++].numericValue = f->get_hash_entry_id(); break; case column_device_ip: retriever->elems[retriever->actNumEntries++].numericValue = f->getFlowDeviceIP(); break; case column_in_index: retriever->elems[retriever->actNumEntries++].numericValue = f->getFlowDeviceInIndex(); break; case column_out_index: retriever->elems[retriever->actNumEntries++].numericValue = f->getFlowDeviceOutIndex(); break; default: ntop->getTrace()->traceEvent(TRACE_WARNING, "Internal error: column %d not handled", retriever->sorter); break; } *matched = true; } return (false); /* false = keep on walking */ } /* **************************************************** */ static bool host_search_walker(GenericHashEntry *he, void *user_data, bool *matched) { char buf[64]; u_int8_t network_prefix = 0; IpAddress *ip_addr = NULL; struct flowHostRetriever *r = (struct flowHostRetriever *)user_data; Host *h = (Host *)he; if (r->actNumEntries >= r->maxNumEntries) return(true); /* Limit reached */ if (!h || h->idle() || !h->match(r->allowed_hosts) #if 0 /* (***) The check below is commented out as it has no sense for hosts as written in https://www.ntop.org/ntopng/data-aggregation-in-ntopng-host-pools-vs-observation-points/ In conclusion, the observation point is a way to logically aggregate flows whereas pool are used to aggregate hosts. For this reason they can be used simultaneously. */ || (h->get_observation_point_id() != r->observationPointId) #endif ) return (false); if ((r->location == location_local_only && (!h->isLocalUnicastHost())) || (r->location == location_local_only_no_tx && ((!h->isLocalUnicastHost()) || (!h->isRxOnlyHost()))) || (r->location == location_local_only_no_tcp_tx && ((!h->isLocalUnicastHost()) || (!h->isRxOnlyHost()) || (h->getNumBytesTCPSent() > 0) || (h->getNumBytesTCPRcvd() == 0))) || (r->location == location_remote_only && h->isLocalHost()) || (r->location == location_remote_only_no_tx && (h->isLocalHost() || (!h->isRxOnlyHost()))) || (r->location == location_remote_only_no_tcp_tx && (h->isLocalHost() || (!h->isRxOnlyHost()) || (h->getNumBytesTCPSent() > 0) || (h->getNumBytesTCPRcvd() == 0))) || (r->location == location_broadcast_domain_only && !h->isBroadcastDomainHost()) || (r->location == location_private_only && !h->isPrivateHost()) || (r->location == location_public_only && h->isPrivateHost()) || (r->location == location_public_only && h->isPrivateHost()) || (r->location == location_broadcat_multicast_only && !h->isBroadcastHost() && !h->isMulticastHost()) || ((r->vlan_id != (NO_VLAN)) && (r->vlan_id != h->get_vlan_id())) || ((r->ndpi_proto != -1) && (h->get_ndpi_stats()->getProtoBytes(r->ndpi_proto) == 0)) || ((r->asnFilter != (u_int32_t)-1) && (r->asnFilter != h->get_asn())) || ((r->networkFilter != -2) && (r->networkFilter != h->get_local_network_id())) || (r->mac && ((!h->getMac()) || (!h->getMac()->equal(r->mac)))) || ((r->poolFilter != (u_int16_t)-1) && (r->poolFilter != h->get_host_pool())) || (r->country && strlen(r->country) && strcmp(h->get_country(buf, sizeof(buf)), r->country)) || (r->osFilter != ndpi_os_MAX_OS && (h->getOS() != r->osFilter)) || (r->blacklistedHosts && !h->isBlacklisted()) || (r->anomalousOnly && !h->hasAnomalies()) || (r->dhcpOnly && !h->isDHCPHost()) || (r->cidr_filter && !h->match(r->cidr_filter)) || (r->traffic_type == traffic_type_unidirectional && !h->isUnidirectionalTraffic()) || (r->traffic_type == traffic_type_bidirectional && !h->isBidirectionalTraffic()) || (r->device_ip && h->getLastDeviceIp() && (r->device_ip != h->getLastDeviceIp())) || (r->dhcpHostsOnly && (!h->isDHCPHost())) || #ifdef NTOPNG_PRO (r->filteredHosts && !h->hasBlockedTraffic()) || #endif (r->ipVersionFilter && (((r->ipVersionFilter == 4) && (!h->get_ip()->isIPv4())) || ((r->ipVersionFilter == 6) && (!h->get_ip()->isIPv6())))) ) return (false); /* false = keep on walking */ r->elems[r->actNumEntries].hostValue = h; h->incUses(); /* (***) */ switch (r->sorter) { case column_ip: r->elems[r->actNumEntries++].hostValue = h; /* hostValue was already set */ break; case column_alerts: r->elems[r->actNumEntries++].numericValue = h->getNumEngagedAlerts(); break; case column_name: r->elems[r->actNumEntries++].stringValue = strdup(h->get_visual_name(buf, sizeof(buf))); break; case column_country: r->elems[r->actNumEntries++].stringValue = strdup(h->get_country(buf, sizeof(buf))); break; case column_os: r->elems[r->actNumEntries++].numericValue = h->getOS(); break; case column_vlan: r->elems[r->actNumEntries++].numericValue = h->get_vlan_id(); break; case column_since: r->elems[r->actNumEntries++].numericValue = h->get_first_seen(); break; case column_asn: r->elems[r->actNumEntries++].numericValue = h->get_asn(); break; case column_obs_point: r->elems[r->actNumEntries++].numericValue = h->get_observation_point_id(); break; case column_score: r->elems[r->actNumEntries++].numericValue = h->getScore(); break; case column_thpt: r->elems[r->actNumEntries++].numericValue = h->getBytesThpt(); break; case column_num_flows: r->elems[r->actNumEntries++].numericValue = h->getNumActiveFlows(); break; case column_num_dropped_flows: r->elems[r->actNumEntries++].numericValue = h->getNumDroppedFlows(); break; case column_traffic: r->elems[r->actNumEntries++].numericValue = h->getNumBytes(); break; case column_local_network_id: r->elems[r->actNumEntries++].numericValue = h->get_local_network_id(); break; case column_local_network: ntop->getLocalNetworkIp(h->get_local_network_id(), &ip_addr, &network_prefix); r->elems[r->actNumEntries].ipValue = ip_addr; r->elems[r->actNumEntries++].numericValue = network_prefix; break; case column_mac: r->elems[r->actNumEntries++].numericValue = Utils::macaddr_int(h->get_mac()); break; case column_pool_id: r->elems[r->actNumEntries++].numericValue = h->get_host_pool(); break; case column_tcp_udp_unresp_as_client: r->elems[r->actNumEntries++].numericValue = h->getNumContactedPeersAsClientTCPUDPNoTX(); break; case column_tcp_udp_unresp_as_server: r->elems[r->actNumEntries++].numericValue = h->getNumContactsFromPeersAsServerTCPUDPNoTX(); break; /* Criteria */ case column_traffic_sent: r->elems[r->actNumEntries++].numericValue = h->getNumBytesSent(); break; case column_traffic_rcvd: r->elems[r->actNumEntries++].numericValue = h->getNumBytesRcvd(); break; case column_traffic_unknown: r->elems[r->actNumEntries++].numericValue = h->get_ndpi_stats()->getProtoBytes(NDPI_PROTOCOL_UNKNOWN); break; case column_num_flows_as_client: r->elems[r->actNumEntries++].numericValue = h->getNumOutgoingFlows(); break; case column_num_flows_as_server: r->elems[r->actNumEntries++].numericValue = h->getNumIncomingFlows(); break; case column_total_num_alerted_flows_as_client: r->elems[r->actNumEntries++].numericValue = h->getTotalNumAlertedOutgoingFlows(); break; case column_total_num_alerted_flows_as_server: r->elems[r->actNumEntries++].numericValue = h->getTotalNumAlertedIncomingFlows(); break; case column_total_num_unreachable_flows_as_client: r->elems[r->actNumEntries++].numericValue = h->getTotalNumUnreachableOutgoingFlows(); break; case column_total_num_unreachable_flows_as_server: r->elems[r->actNumEntries++].numericValue = h->getTotalNumUnreachableIncomingFlows(); break; case column_total_num_retx_sent: r->elems[r->actNumEntries++].numericValue = h->getTcpPacketSentStats()->get_retr(); break; case column_total_num_retx_rcvd: r->elems[r->actNumEntries++].numericValue = h->getTcpPacketRcvdStats()->get_retr(); break; case column_total_alerts: r->elems[r->actNumEntries++].numericValue = h->getTotalAlerts(); break; case column_score_as_client: r->elems[r->actNumEntries++].numericValue = h->getScoreAsClient(); break; case column_score_as_server: r->elems[r->actNumEntries++].numericValue = h->getScoreAsServer(); break; default: ntop->getTrace()->traceEvent( TRACE_WARNING, "Internal error: column %d not handled", r->sorter); break; } *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ static bool mac_search_walker(GenericHashEntry *he, void *user_data, bool *matched) { struct flowHostRetriever *r = (struct flowHostRetriever *)user_data; Mac *m = (Mac *)he; if (r->actNumEntries >= r->maxNumEntries) return (true); /* Limit reached */ if (!m || m->idle() || (r->min_first_seen >= m->get_first_seen()) /* first seen must be greater than or equal to the minimum first seen */ || (r->sourceMacsOnly && !m->isSourceMac()) || ((r->devtypeFilter != (u_int8_t)-1) && (m->getDeviceType() != r->devtypeFilter)) #ifdef HAVE_NEDGE || ((r->locationFilter != (u_int8_t)-1) && (m->locate() != r->locationFilter)) #endif || ((r->poolFilter != (u_int16_t)-1) && (m->getInterface()->getHostPool(m) != r->poolFilter)) || (r->manufacturer && strcmp(r->manufacturer, m->get_manufacturer() ? m->get_manufacturer() : "") != 0)) return (false); /* false = keep on walking */ r->elems[r->actNumEntries].macValue = m; switch (r->sorter) { case column_mac: r->elems[r->actNumEntries++].numericValue = Utils::macaddr_int(m->get_mac()); break; case column_since: r->elems[r->actNumEntries++].numericValue = m->get_first_seen(); break; case column_thpt: r->elems[r->actNumEntries++].numericValue = m->getBytesThpt(); break; case column_traffic: r->elems[r->actNumEntries++].numericValue = m->getNumBytes(); break; case column_num_hosts: r->elems[r->actNumEntries++].numericValue = m->getNumHosts(); break; case column_manufacturer: r->elems[r->actNumEntries++].stringValue = m->get_manufacturer() ? (char *)m->get_manufacturer() : (char *)"zzz"; break; case column_device_type: r->elems[r->actNumEntries++].numericValue = m->getDeviceType(); break; case column_arp_total: r->elems[r->actNumEntries++].numericValue = m->getNumSentArp() + m->getNumRcvdArp(); break; case column_arp_sent: r->elems[r->actNumEntries++].numericValue = m->getNumSentArp(); break; case column_arp_rcvd: r->elems[r->actNumEntries++].numericValue = m->getNumRcvdArp(); break; default: ntop->getTrace()->traceEvent( TRACE_WARNING, "Internal error: column %d not handled", r->sorter); break; } *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ static bool as_search_walker(GenericHashEntry *he, void *user_data, bool *matched) { struct flowHostRetriever *r = (struct flowHostRetriever *)user_data; AutonomousSystem *as = (AutonomousSystem *)he; if (r->actNumEntries >= r->maxNumEntries) return (true); /* Limit reached */ if (!as || as->idle()) return (false); /* false = keep on walking */ r->elems[r->actNumEntries].asValue = as; switch (r->sorter) { case column_asn: r->elems[r->actNumEntries++].numericValue = as->get_asn(); break; case column_asname: r->elems[r->actNumEntries++].stringValue = as->get_asname() ? as->get_asname() : (char *)"zzz"; break; case column_score: r->elems[r->actNumEntries++].numericValue = as->getScore(); break; case column_alerted_flows: r->elems[r->actNumEntries++].numericValue = as->getTotalAlertedNumFlowsAsClient() + as->getTotalAlertedNumFlowsAsServer(); break; case column_since: r->elems[r->actNumEntries++].numericValue = as->get_first_seen(); break; case column_thpt: r->elems[r->actNumEntries++].numericValue = as->getBytesThpt(); break; case column_traffic: r->elems[r->actNumEntries++].numericValue = as->getNumBytes(); break; case column_num_hosts: r->elems[r->actNumEntries++].numericValue = as->getNumHosts(); break; default: ntop->getTrace()->traceEvent( TRACE_WARNING, "Internal error: column %d not handled", r->sorter); break; } *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ static bool obs_point_search_walker(GenericHashEntry *he, void *user_data, bool *matched) { struct flowHostRetriever *r = (struct flowHostRetriever *)user_data; ObservationPoint *obs_point = (ObservationPoint *)he; if (r->actNumEntries >= r->maxNumEntries) return (true); /* Limit reached */ if (!obs_point || obs_point->idle()) return (false); /* false = keep on walking */ r->elems[r->actNumEntries].obsPointValue = obs_point; switch (r->sorter) { case column_obs_point: r->elems[r->actNumEntries++].numericValue = obs_point->getObsPoint(); break; case column_score: r->elems[r->actNumEntries++].numericValue = obs_point->getScore(); break; case column_since: r->elems[r->actNumEntries++].numericValue = obs_point->get_first_seen(); break; case column_thpt: r->elems[r->actNumEntries++].numericValue = obs_point->getBytesThpt(); break; case column_traffic: r->elems[r->actNumEntries++].numericValue = obs_point->getNumBytes(); break; case column_num_hosts: r->elems[r->actNumEntries++].numericValue = obs_point->getNumHosts(); break; default: ntop->getTrace()->traceEvent( TRACE_WARNING, "Internal error: column %d not handled", r->sorter); break; } *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ static bool country_search_walker(GenericHashEntry *he, void *user_data, bool *matched) { struct flowHostRetriever *r = (struct flowHostRetriever *)user_data; Country *country = (Country *)he; if (r->actNumEntries >= r->maxNumEntries) return (true); /* Limit reached */ if (!country || country->idle()) return (false); /* false = keep on walking */ r->elems[r->actNumEntries].countryVal = country; /* Note: we don't have throughput information into the countries */ switch (r->sorter) { case column_country: r->elems[r->actNumEntries++].stringValue = country->get_country_name(); break; case column_since: r->elems[r->actNumEntries++].numericValue = country->get_first_seen(); break; case column_score: r->elems[r->actNumEntries++].numericValue = country->getScore(); break; case column_num_hosts: r->elems[r->actNumEntries++].numericValue = country->getNumHosts(); break; case column_thpt: r->elems[r->actNumEntries++].numericValue = country->getBytesThpt(); break; case column_traffic: r->elems[r->actNumEntries++].numericValue = country->getNumBytes(); break; default: ntop->getTrace()->traceEvent( TRACE_WARNING, "Internal error: column %d not handled", r->sorter); break; } *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ static bool vlan_search_walker(GenericHashEntry *he, void *user_data, bool *matched) { struct flowHostRetriever *r = (struct flowHostRetriever *)user_data; VLAN *vl = (VLAN *)he; if (r->actNumEntries >= r->maxNumEntries) return (true); /* Limit reached */ if (!vl || vl->idle()) return (false); /* false = keep on walking */ r->elems[r->actNumEntries].vlanValue = vl; switch (r->sorter) { case column_vlan: r->elems[r->actNumEntries++].numericValue = vl->get_vlan_id(); break; case column_since: r->elems[r->actNumEntries++].numericValue = vl->get_first_seen(); break; case column_thpt: r->elems[r->actNumEntries++].numericValue = vl->getBytesThpt(); break; case column_score: r->elems[r->actNumEntries++].numericValue = vl->getScore(); break; case column_traffic: r->elems[r->actNumEntries++].numericValue = vl->getNumBytes(); break; case column_num_hosts: r->elems[r->actNumEntries++].numericValue = vl->getNumHosts(); break; default: ntop->getTrace()->traceEvent( TRACE_WARNING, "Internal error: column %d not handled", r->sorter); break; } *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ int hostSorter(const void *_a, const void *_b) { struct flowHostRetrieveList *a = (struct flowHostRetrieveList *)_a; struct flowHostRetrieveList *b = (struct flowHostRetrieveList *)_b; if (!a || !b) return (true); return (a->hostValue->compare(b->hostValue)); } /* **************************************************** */ int ipNetworkSorter(const void *_a, const void *_b) { struct flowHostRetrieveList *a = (struct flowHostRetrieveList *)_a; struct flowHostRetrieveList *b = (struct flowHostRetrieveList *)_b; int rv; if (!a || !b || !a->ipValue || !b->ipValue) return (true); /* Compare network address first */ rv = a->ipValue->compare(b->ipValue); if (rv != 0) return rv; /* If the address matches, compare netmasks */ if (a->numericValue < b->numericValue) return (-1); else if (a->numericValue > b->numericValue) return (1); else return (0); } int ipSorter(const void *_a, const void *_b) { struct flowHostRetrieveList *a = (struct flowHostRetrieveList *)_a; struct flowHostRetrieveList *b = (struct flowHostRetrieveList *)_b; if (!a || !b || !a->ipValue || !b->ipValue) return (true); return (a->ipValue->compare(b->ipValue)); } int numericSorter(const void *_a, const void *_b) { struct flowHostRetrieveList *a = (struct flowHostRetrieveList *)_a; struct flowHostRetrieveList *b = (struct flowHostRetrieveList *)_b; if (!a || !b) return (true); if (a->numericValue < b->numericValue) return (-1); else if (a->numericValue > b->numericValue) return (1); else return (0); } int stringSorter(const void *_a, const void *_b) { struct flowHostRetrieveList *a = (struct flowHostRetrieveList *)_a; struct flowHostRetrieveList *b = (struct flowHostRetrieveList *)_b; if (!a || !b || !a->stringValue || !b->stringValue) return (true); return (strcasecmp(a->stringValue, b->stringValue)); } int protocolSorter(const void *_a, const void *_b) { struct flowHostRetrieveList *a = (struct flowHostRetrieveList *)_a; struct flowHostRetrieveList *b = (struct flowHostRetrieveList *)_b; if (!a || !b || !a->flow || !b->flow) return (true); char buf_a[64]; char buf_b[64]; char* a_protocol = a->flow->get_detected_protocol_name(buf_a, sizeof(buf_a)); char* b_protocol = b->flow->get_detected_protocol_name(buf_b, sizeof(buf_b)); int result = strcasecmp(a_protocol, b_protocol); if (result!=0) return (result); else { char* a_l4_protocol = Utils::l4proto2name(a->flow->get_protocol()); char* b_l4_protocol = Utils::l4proto2name(b->flow->get_protocol()); #if 0 ntop->getTrace()->traceEvent(TRACE_NORMAL, "Comparison between [%s]:[%s] and [%s]:[%s]", a_l4_protocol, a_protocol, b_l4_protocol, b_protocol); #endif return (strcasecmp(a_l4_protocol, b_l4_protocol)); } } /* **************************************************** */ int NetworkInterface::sortFlows(u_int32_t *begin_slot, bool walk_all, struct flowHostRetriever *retriever, AddressTree *allowed_hosts, Host *host, Host *client, Host *server, char *flow_info, Paginator *p, const char *sortColumn) { int (*sorter)(const void *_a, const void *_b); if (retriever == NULL) return (-1); retriever->pag = p; retriever->host = host, retriever->location = location_all; retriever->server = server; retriever->client = client; retriever->flow_info = flow_info; retriever->ndpi_proto = -1; retriever->iface_index = -1; retriever->actNumEntries = 0, retriever->maxNumEntries = getFlowsHashSize(), retriever->allowed_hosts = allowed_hosts; retriever->elems = (struct flowHostRetrieveList *)calloc(sizeof(struct flowHostRetrieveList), retriever->maxNumEntries); if (retriever->elems == NULL) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Out of memory :-("); return (-1); } if (!strcmp(sortColumn, "column_client")) retriever->sorter = column_client, sorter = (isViewed() || isView()) ? ipSorter : hostSorter; else if (!strcmp(sortColumn, "column_vlan")) retriever->sorter = column_vlan, sorter = numericSorter; else if (!strcmp(sortColumn, "column_server")) retriever->sorter = column_server, sorter = (isViewed() || isView()) ? ipSorter : hostSorter; else if (!strcmp(sortColumn, "column_proto_l4")) retriever->sorter = column_proto_l4, sorter = numericSorter; else if (!strcmp(sortColumn, "column_ndpi")) retriever->sorter = column_ndpi, sorter = numericSorter; else if (!strcmp(sortColumn, "column_protocol")) retriever->sorter = column_protocol, sorter = protocolSorter; else if (!strcmp(sortColumn, "column_duration")) retriever->sorter = column_duration, sorter = numericSorter; else if (!strcmp(sortColumn, "column_score")) retriever->sorter = column_score, sorter = numericSorter; else if (!strcmp(sortColumn, "column_score")) retriever->sorter = column_score, sorter = numericSorter; else if (!strcmp(sortColumn, "column_score_as_client")) retriever->sorter = column_score_as_client, sorter = numericSorter; else if (!strcmp(sortColumn, "column_score_as_server")) retriever->sorter = column_score_as_server, sorter = numericSorter; else if (!strcmp(sortColumn, "column_thpt")) retriever->sorter = column_thpt, sorter = numericSorter; else if (!strcmp(sortColumn, "column_last_seen")) retriever->sorter = column_last_seen, sorter = numericSorter; else if (!strcmp(sortColumn, "column_first_seen")) retriever->sorter = column_first_seen, sorter = numericSorter; else if (!strcmp(sortColumn, "column_client_rtt")) retriever->sorter = column_client_rtt, sorter = numericSorter; else if (!strcmp(sortColumn, "column_server_rtt")) retriever->sorter = column_server_rtt, sorter = numericSorter; else if ((!strcmp(sortColumn, "column_bytes")) || (!strcmp(sortColumn, "column_") /* default */)) retriever->sorter = column_bytes, sorter = numericSorter; else if (!strcmp(sortColumn, "column_info")) retriever->sorter = column_info, sorter = stringSorter; else if (!strcmp(sortColumn, "column_device_ip")) retriever->sorter = column_device_ip, sorter = numericSorter; else if (!strcmp(sortColumn, "column_in_index")) retriever->sorter = column_in_index, sorter = numericSorter; else if (!strcmp(sortColumn, "column_out_index")) retriever->sorter = column_out_index, sorter = numericSorter; else if (!strcmp(sortColumn, "column_key")) retriever->sorter = column_key, sorter = numericSorter; else if (!strcmp(sortColumn, "column_qoe")) retriever->sorter = column_qoe, sorter = numericSorter; else { ntop->getTrace()->traceEvent(TRACE_WARNING, "Unknown sort column %s", sortColumn); retriever->sorter = column_bytes, sorter = numericSorter; } if (false) { u_int32_t deviceIP = 0; u_int32_t inIndex = 0, outIndex = 0; char buf[32]; p->deviceIpFilter(&deviceIP), p->inIndexFilter(&inIndex), p->outIndexFilter(&outIndex); ntop->getTrace()->traceEvent(TRACE_NORMAL, "[Device IP] %s / [In Idx] %u / [Out Idx] %u", Utils::intoaV4(deviceIP, buf, sizeof(buf)), inIndex, outIndex); } // make sure the caller has disabled the purge!! walker(begin_slot, walk_all, walker_flows, flow_search_walker, (void *)retriever); qsort(retriever->elems, retriever->actNumEntries, sizeof(struct flowHostRetrieveList), sorter); return (retriever->actNumEntries); } /* **************************************************** */ static bool flow_sum_stats(GenericHashEntry *flow, void *user_data, bool *matched) { flowHostRetriever *retriever = (flowHostRetriever *)user_data; nDPIStats *ndpi_stats = retriever->ndpi_stats; FlowStats *stats = retriever->stats; Flow *f = (Flow *)flow; if (flow_matches(f, retriever)) { if(retriever->host) { /* Add this info only in case some filter host is requested */ stats->updateTalkingHosts(f); } if(f->getWLANSSID()) { stats->updateWLANSSID(f); } retriever->totBytesSent += f->get_bytes_cli2srv(); retriever->totBytesRcvd += f->get_bytes_srv2cli(); retriever->totThpt += f->get_bytes_thpt(); #ifdef NTOPNG_PRO retriever->qoe->incQoEStats(f->getQoEType()); #endif if (!retriever->only_traffic_stats) f->sumStats(ndpi_stats, stats); *matched = true; } return (false); /* false = keep on walking */ } /* **************************************************** */ void NetworkInterface::getActiveFlowsStats(nDPIStats *ndpi_stats, FlowStats *stats, AddressTree *allowed_hosts, Host *h, Host *talking_with_host, Host *client, Host *server, char *flow_info, Paginator *p, lua_State *vm, bool only_traffic_stats) { flowHostRetriever retriever; u_int32_t begin_slot = 0; bool walk_all = true; #ifdef NTOPNG_PRO QoEStats qoe; #endif memset(&retriever, 0, sizeof(retriever)); retriever.pag = p; retriever.host = h; retriever.talking_with_host = talking_with_host; retriever.client = client; retriever.server = server; retriever.location = location_all; retriever.ndpi_proto = -1; #ifdef NTOPNG_PRO retriever.qoe = &qoe; #endif retriever.actNumEntries = 0; retriever.maxNumEntries = getFlowsHashSize(); retriever.allowed_hosts = allowed_hosts; retriever.ndpi_stats = ndpi_stats; retriever.stats = stats; retriever.totBytesSent = 0, retriever.totBytesRcvd = 0, retriever.totThpt = 0; retriever.only_traffic_stats = only_traffic_stats; retriever.observationPointId = getLuaVMUservalue(vm, observationPointId); retriever.flow_info = flow_info; walker(&begin_slot, walk_all, walker_flows, flow_sum_stats, &retriever); lua_newtable(vm); /* Overview stats */ lua_push_uint64_table_entry(vm, "numFlows", retriever.actNumEntries); lua_push_uint64_table_entry(vm, "totBytesSent", retriever.totBytesSent); lua_push_uint64_table_entry(vm, "totBytesRcvd", retriever.totBytesRcvd); #ifdef NTOPNG_PRO /* Handle the QoE*/ retriever.qoe->lua_qoe_stats(vm); #endif if (!only_traffic_stats) { /* DPI stats */ ndpi_stats->lua(this, vm, true); stats->lua(vm); } retriever.stats->resetTalkingHosts(); } /* **************************************************** */ int NetworkInterface::getFlows(lua_State *vm, u_int32_t *begin_slot, bool walk_all, AddressTree *allowed_hosts, Host *host, Host *talking_with_host, Host *client, Host *server, char *flow_info, Paginator *p) { struct flowHostRetriever retriever; char sortColumn[32]; DetailsLevel highDetails; if (p == NULL) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Unable to return results with a NULL paginator"); return (-1); } LocationPolicy client_mode = location_all; LocationPolicy server_mode = location_all; p->clientMode(&client_mode); p->serverMode(&server_mode); bool local_hosts = ((client_mode == location_local_only) && (server_mode == location_local_only)); snprintf(sortColumn, sizeof(sortColumn), "%s", p->sortColumn()); if (!p->getDetailsLevel(&highDetails)) highDetails = p->detailedResults() ? details_high : (local_hosts || (p && p->maxHits() != CONST_MAX_NUM_HITS)) ? details_high : details_normal; retriever.observationPointId = getLuaVMUservalue(vm, observationPointId); retriever.talking_with_host = talking_with_host; if (sortFlows(begin_slot, walk_all, &retriever, allowed_hosts, host, client, server, flow_info, p, sortColumn) < 0) return (-1); lua_newtable(vm); lua_push_uint64_table_entry(vm, "numFlows", retriever.actNumEntries); lua_push_uint64_table_entry(vm, "nextSlot", *begin_slot); lua_newtable(vm); if (p->a2zSortOrder()) { for (int i = p->toSkip(), num = 0; i < (int)retriever.actNumEntries; i++) { lua_newtable(vm); retriever.elems[i].flow->lua(vm, allowed_hosts, highDetails, true); lua_pushinteger(vm, num + 1); lua_insert(vm, -2); lua_settable(vm, -3); if (++num >= (int)p->maxHits()) break; } } else { for (int i = (retriever.actNumEntries - 1 - p->toSkip()), num = 0; i >= 0; i--) { lua_newtable(vm); retriever.elems[i].flow->lua(vm, allowed_hosts, highDetails, true); lua_pushinteger(vm, num + 1); lua_insert(vm, -2); lua_settable(vm, -3); if (++num >= (int)p->maxHits()) break; } } lua_pushstring(vm, "flows"); lua_insert(vm, -2); lua_settable(vm, -3); if (retriever.elems) free(retriever.elems); return (retriever.actNumEntries); } /* **************************************************** */ int NetworkInterface::getFlowsGroup(lua_State *vm, AddressTree *allowed_hosts, Paginator *p, const char *groupColumn) { struct flowHostRetriever retriever; FlowGrouper *gper; u_int32_t begin_slot = 0; bool walk_all = true; if (p == NULL) { ntop->getTrace()->traceEvent( TRACE_WARNING, "Unable to return results with a NULL paginator"); return (-1); } retriever.observationPointId = getLuaVMUservalue(vm, observationPointId); if (sortFlows(&begin_slot, walk_all, &retriever, allowed_hosts, NULL, NULL, NULL, NULL, p, groupColumn) < 0) { return (-1); } // build a new grouper that will help in aggregating stats if ((gper = new (std::nothrow) FlowGrouper(retriever.sorter)) == NULL) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to allocate memory for a Grouper."); return (-1); } lua_newtable(vm); for (int i = 0; i < (int)retriever.actNumEntries; i++) { Flow *flow = retriever.elems[i].flow; if (flow) { if (gper->inGroup(flow) == false) { if (gper->getNumEntries() > 0) gper->lua(vm); gper->newGroup(flow); } gper->incStats(flow); } } if (gper->getNumEntries() > 0) gper->lua(vm); delete gper; if (retriever.elems) free(retriever.elems); return (retriever.actNumEntries); } /* **************************************************** */ static bool flow_drop_walker(GenericHashEntry *h, void *user_data, bool *matched) { struct flowHostRetriever *retriever = (struct flowHostRetriever *)user_data; Flow *f = (Flow *)h; if (flow_matches(f, retriever)) { f->setDropVerdict(DROP_REASON_USER_ACTION); *matched = true; } return (false); /* Keep on walking */ } /* **************************************************** */ int NetworkInterface::dropFlowsTraffic(AddressTree *allowed_hosts, Paginator *p) { struct flowHostRetriever retriever; u_int32_t begin_slot = 0; bool walk_all = true; #ifdef NTOPNG_PRO QoEStats qoe; #endif memset(&retriever, 0, sizeof(retriever)); retriever.allowed_hosts = allowed_hosts; retriever.pag = p; #ifdef NTOPNG_PRO retriever.qoe = &qoe; #endif walker(&begin_slot, walk_all, walker_flows, flow_drop_walker, (void *)&retriever); return (0); } /* **************************************************** */ int NetworkInterface::sortHosts(u_int32_t *begin_slot, bool walk_all, struct flowHostRetriever *retriever, u_int8_t bridge_iface_idx, AddressTree *allowed_hosts, bool host_details, LocationPolicy location, char *countryFilter, char *mac_filter, u_int16_t vlan_id, ndpi_os osFilter, u_int32_t asnFilter, int32_t networkFilter, u_int16_t pool_filter, bool filtered_hosts, bool blacklisted_hosts, bool anomalousOnly, bool dhcpOnly, const AddressTree *const cidr_filter, u_int8_t ipver_filter, int proto_filter, TrafficType traffic_type_filter, u_int32_t device_ip, char *sortColumn) { u_int8_t macAddr[6]; int (*sorter)(const void *_a, const void *_b); if (retriever == NULL) return (-1); if (mac_filter) { Utils::parseMac(macAddr, mac_filter); retriever->mac = macAddr; } else { retriever->mac = NULL; } retriever->allowed_hosts = allowed_hosts, retriever->location = location, retriever->country = countryFilter, retriever->vlan_id = vlan_id, retriever->osFilter = osFilter, retriever->asnFilter = asnFilter, retriever->networkFilter = networkFilter, retriever->actNumEntries = 0, retriever->poolFilter = pool_filter, retriever->bridge_iface_idx = 0, retriever->ipVersionFilter = ipver_filter, retriever->filteredHosts = filtered_hosts, retriever->blacklistedHosts = blacklisted_hosts, retriever->anomalousOnly = anomalousOnly, retriever->dhcpOnly = dhcpOnly, retriever->cidr_filter = cidr_filter, retriever->ndpi_proto = proto_filter, retriever->traffic_type = traffic_type_filter, retriever->device_ip = device_ip, retriever->maxNumEntries = getHostsHashSize(); retriever->elems = (struct flowHostRetrieveList *)calloc(sizeof(struct flowHostRetrieveList), retriever->maxNumEntries); if (retriever->elems == NULL) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Out of memory :-("); return (-1); } if ((!strcmp(sortColumn, "column_ip")) || (!strcmp(sortColumn, "column_"))) retriever->sorter = column_ip, sorter = (isViewed() || isView()) ? ipSorter : hostSorter; else if (!strcmp(sortColumn, "column_vlan")) retriever->sorter = column_vlan, sorter = numericSorter; else if (!strcmp(sortColumn, "column_alerts")) retriever->sorter = column_alerts, sorter = numericSorter; else if (!strcmp(sortColumn, "column_name")) retriever->sorter = column_name, sorter = stringSorter; else if (!strcmp(sortColumn, "column_country")) retriever->sorter = column_country, sorter = stringSorter; else if (!strcmp(sortColumn, "column_os")) retriever->sorter = column_os, sorter = numericSorter; else if (!strcmp(sortColumn, "column_since")) retriever->sorter = column_since, sorter = numericSorter; else if (!strcmp(sortColumn, "column_asn")) retriever->sorter = column_asn, sorter = numericSorter; else if (!strcmp(sortColumn, "column_thpt")) retriever->sorter = column_thpt, sorter = numericSorter; else if (!strcmp(sortColumn, "column_num_flows")) retriever->sorter = column_num_flows, sorter = numericSorter; else if (!strcmp(sortColumn, "column_num_dropped_flows")) retriever->sorter = column_num_dropped_flows, sorter = numericSorter; else if (!strcmp(sortColumn, "column_traffic")) retriever->sorter = column_traffic, sorter = numericSorter; else if (!strcmp(sortColumn, "column_local_network_id")) retriever->sorter = column_local_network_id, sorter = numericSorter; else if (!strcmp(sortColumn, "column_local_network")) retriever->sorter = column_local_network, sorter = ipNetworkSorter; else if (!strcmp(sortColumn, "column_mac")) retriever->sorter = column_mac, sorter = numericSorter; /* criteria (datatype sortField in ntop_typedefs.h / see also * host_search_walker:NetworkInterface.cpp) */ else if (!strcmp(sortColumn, "column_traffic_sent")) retriever->sorter = column_traffic_sent, sorter = numericSorter; else if (!strcmp(sortColumn, "column_traffic_rcvd")) retriever->sorter = column_traffic_rcvd, sorter = numericSorter; else if (!strcmp(sortColumn, "column_traffic_unknown")) retriever->sorter = column_traffic_unknown, sorter = numericSorter; else if (!strcmp(sortColumn, "column_num_flows_as_client")) retriever->sorter = column_num_flows_as_client, sorter = numericSorter; else if (!strcmp(sortColumn, "column_num_flows_as_server")) retriever->sorter = column_num_flows_as_server, sorter = numericSorter; else if (!strcmp(sortColumn, "column_total_num_alerted_flows_as_client")) retriever->sorter = column_total_num_alerted_flows_as_client, sorter = numericSorter; else if (!strcmp(sortColumn, "column_total_num_alerted_flows_as_server")) retriever->sorter = column_total_num_alerted_flows_as_server, sorter = numericSorter; else if (!strcmp(sortColumn, "column_total_num_unreachable_flows_as_client")) retriever->sorter = column_total_num_unreachable_flows_as_client, sorter = numericSorter; else if (!strcmp(sortColumn, "column_total_num_unreachable_flows_as_server")) retriever->sorter = column_total_num_unreachable_flows_as_server, sorter = numericSorter; else if (!strcmp(sortColumn, "column_total_num_retx_sent")) retriever->sorter = column_total_num_retx_sent, sorter = numericSorter; else if (!strcmp(sortColumn, "column_total_num_retx_rcvd")) retriever->sorter = column_total_num_retx_rcvd, sorter = numericSorter; else if (!strcmp(sortColumn, "column_total_alerts")) retriever->sorter = column_total_alerts, sorter = numericSorter; else if (!strcmp(sortColumn, "column_score")) retriever->sorter = column_score, sorter = numericSorter; else if (!strcmp(sortColumn, "column_score_as_client")) retriever->sorter = column_score_as_client, sorter = numericSorter; else if (!strcmp(sortColumn, "column_score_as_server")) retriever->sorter = column_score_as_server, sorter = numericSorter; else if (!strcmp(sortColumn, "column_pool_id")) retriever->sorter = column_pool_id, sorter = numericSorter; else if (!strcmp(sortColumn, "column_tcp_unresp_as_client")) retriever->sorter = column_tcp_udp_unresp_as_client, sorter = numericSorter; else if (!strcmp(sortColumn, "column_tcp_unresp_as_server")) retriever->sorter = column_tcp_udp_unresp_as_server, sorter = numericSorter; else { ntop->getTrace()->traceEvent(TRACE_WARNING, "Unknown sort column %s", sortColumn); retriever->sorter = column_traffic, sorter = numericSorter; } // make sure the caller has disabled the purge!! walker(begin_slot, walk_all, walker_hosts, host_search_walker, (void *)retriever); qsort(retriever->elems, retriever->actNumEntries, sizeof(struct flowHostRetrieveList), sorter); return (retriever->actNumEntries); } /* **************************************************** */ int NetworkInterface::sortMacs(u_int32_t *begin_slot, bool walk_all, struct flowHostRetriever *retriever, u_int8_t bridge_iface_idx, bool sourceMacsOnly, const char *manufacturer, char *sortColumn, u_int16_t pool_filter, u_int8_t devtype_filter, u_int8_t location_filter, time_t min_first_seen) { int (*sorter)(const void *_a, const void *_b); if (retriever == NULL) return (-1); retriever->sourceMacsOnly = sourceMacsOnly, retriever->actNumEntries = 0, retriever->poolFilter = pool_filter, retriever->manufacturer = (char *)manufacturer, retriever->maxNumEntries = getMacsHashSize(); retriever->devtypeFilter = devtype_filter, retriever->locationFilter = location_filter, retriever->min_first_seen = min_first_seen, retriever->ndpi_proto = -1, retriever->elems = (struct flowHostRetrieveList *)calloc(sizeof(struct flowHostRetrieveList), retriever->maxNumEntries); if (retriever->elems == NULL) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Out of memory :-("); return (-1); } if ((!strcmp(sortColumn, "column_mac")) || (!strcmp(sortColumn, "column_"))) retriever->sorter = column_mac, sorter = numericSorter; else if (!strcmp(sortColumn, "column_since")) retriever->sorter = column_since, sorter = numericSorter; else if (!strcmp(sortColumn, "column_thpt")) retriever->sorter = column_thpt, sorter = numericSorter; else if (!strcmp(sortColumn, "column_traffic")) retriever->sorter = column_traffic, sorter = numericSorter; else if (!strcmp(sortColumn, "column_hosts")) retriever->sorter = column_num_hosts, sorter = numericSorter; else if (!strcmp(sortColumn, "column_manufacturer")) retriever->sorter = column_manufacturer, sorter = stringSorter; else if (!strcmp(sortColumn, "column_device_type")) retriever->sorter = column_device_type, sorter = numericSorter; else if (!strcmp(sortColumn, "column_arp_total")) retriever->sorter = column_arp_total, sorter = numericSorter; else if (!strcmp(sortColumn, "column_arp_sent")) retriever->sorter = column_arp_sent, sorter = numericSorter; else if (!strcmp(sortColumn, "column_arp_rcvd")) retriever->sorter = column_arp_rcvd, sorter = numericSorter; else ntop->getTrace()->traceEvent(TRACE_WARNING, "Unknown sort column %s", sortColumn), sorter = numericSorter; // make sure the caller has disabled the purge!! walker(begin_slot, walk_all, walker_macs, mac_search_walker, (void *)retriever); qsort(retriever->elems, retriever->actNumEntries, sizeof(struct flowHostRetrieveList), sorter); return (retriever->actNumEntries); } /* **************************************************** */ int NetworkInterface::sortASes(struct flowHostRetriever *retriever, char *sortColumn) { int (*sorter)(const void *_a, const void *_b); u_int32_t begin_slot = 0; bool walk_all = true; if (retriever == NULL) return (-1); retriever->actNumEntries = 0, retriever->maxNumEntries = getASesHashSize(); retriever->elems = (struct flowHostRetrieveList *)calloc(sizeof(struct flowHostRetrieveList), retriever->maxNumEntries); if (retriever->elems == NULL) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Out of memory :-("); return (-1); } if ((!strcmp(sortColumn, "column_asn")) || (!strcmp(sortColumn, "column_"))) retriever->sorter = column_asn, sorter = numericSorter; else if (!strcmp(sortColumn, "column_asname")) retriever->sorter = column_asname, sorter = stringSorter; else if (!strcmp(sortColumn, "column_score")) retriever->sorter = column_score, sorter = numericSorter; else if (!strcmp(sortColumn, "column_alerted_flows")) retriever->sorter = column_alerted_flows, sorter = numericSorter; else if (!strcmp(sortColumn, "column_since")) retriever->sorter = column_since, sorter = numericSorter; else if (!strcmp(sortColumn, "column_thpt")) retriever->sorter = column_thpt, sorter = numericSorter; else if (!strcmp(sortColumn, "column_traffic")) retriever->sorter = column_traffic, sorter = numericSorter; else if (!strcmp(sortColumn, "column_hosts")) retriever->sorter = column_num_hosts, sorter = numericSorter; else ntop->getTrace()->traceEvent(TRACE_WARNING, "Unknown sort column %s", sortColumn), sorter = numericSorter; // make sure the caller has disabled the purge!! walker(&begin_slot, walk_all, walker_ases, as_search_walker, (void *)retriever); qsort(retriever->elems, retriever->actNumEntries, sizeof(struct flowHostRetrieveList), sorter); return (retriever->actNumEntries); } /* **************************************************** */ int NetworkInterface::sortObsPoints(struct flowHostRetriever *retriever, char *sortColumn) { int (*sorter)(const void *_a, const void *_b); u_int32_t begin_slot = 0; bool walk_all = true; if (retriever == NULL) return (-1); retriever->actNumEntries = 0, retriever->maxNumEntries = getObsHashSize(); retriever->elems = (struct flowHostRetrieveList *)calloc(sizeof(struct flowHostRetrieveList), retriever->maxNumEntries); if (retriever->elems == NULL) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Out of memory :-("); return (-1); } if ((!strcmp(sortColumn, "column_obs_point")) || (!strcmp(sortColumn, "column_"))) retriever->sorter = column_obs_point, sorter = numericSorter; else if (!strcmp(sortColumn, "column_score")) retriever->sorter = column_score, sorter = numericSorter; else if (!strcmp(sortColumn, "column_since")) retriever->sorter = column_since, sorter = numericSorter; else if (!strcmp(sortColumn, "column_thpt")) retriever->sorter = column_thpt, sorter = numericSorter; else if (!strcmp(sortColumn, "column_traffic")) retriever->sorter = column_traffic, sorter = numericSorter; else if (!strcmp(sortColumn, "column_hosts")) retriever->sorter = column_num_hosts, sorter = numericSorter; else ntop->getTrace()->traceEvent(TRACE_WARNING, "Unknown sort column %s", sortColumn), sorter = numericSorter; // make sure the caller has disabled the purge!! walker(&begin_slot, walk_all, walker_obs, obs_point_search_walker, (void *)retriever); qsort(retriever->elems, retriever->actNumEntries, sizeof(struct flowHostRetrieveList), sorter); return (retriever->actNumEntries); } /* **************************************************** */ int NetworkInterface::sortCountries(struct flowHostRetriever *retriever, char *sortColumn) { int (*sorter)(const void *_a, const void *_b); u_int32_t begin_slot = 0; bool walk_all = true; if (retriever == NULL) return (-1); retriever->actNumEntries = 0, retriever->maxNumEntries = getCountriesHashSize(); retriever->elems = (struct flowHostRetrieveList *)calloc(sizeof(struct flowHostRetrieveList), retriever->maxNumEntries); if (retriever->elems == NULL) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Out of memory :-("); return (-1); } if ((!strcmp(sortColumn, "column_country")) || (!strcmp(sortColumn, "column_id")) || (!strcmp(sortColumn, "column_"))) retriever->sorter = column_country, sorter = stringSorter; else if (!strcmp(sortColumn, "column_since")) retriever->sorter = column_since, sorter = numericSorter; else if (!strcmp(sortColumn, "column_score")) retriever->sorter = column_score, sorter = numericSorter; else if (!strcmp(sortColumn, "column_hosts")) retriever->sorter = column_num_hosts, sorter = numericSorter; else if (!strcmp(sortColumn, "column_thpt")) retriever->sorter = column_thpt, sorter = numericSorter; else if (!strcmp(sortColumn, "column_traffic")) retriever->sorter = column_traffic, sorter = numericSorter; else ntop->getTrace()->traceEvent(TRACE_WARNING, "Unknown sort column %s", sortColumn), sorter = numericSorter; // make sure the caller has disabled the purge!! walker(&begin_slot, walk_all, walker_countries, country_search_walker, (void *)retriever); qsort(retriever->elems, retriever->actNumEntries, sizeof(struct flowHostRetrieveList), sorter); return (retriever->actNumEntries); } /* **************************************************** */ int NetworkInterface::sortVLANs(struct flowHostRetriever *retriever, char *sortColumn) { int (*sorter)(const void *_a, const void *_b); u_int32_t begin_slot = 0; bool walk_all = true; if (retriever == NULL) return (-1); retriever->actNumEntries = 0, retriever->maxNumEntries = getVLANsHashSize(); retriever->elems = (struct flowHostRetrieveList *)calloc(sizeof(struct flowHostRetrieveList), retriever->maxNumEntries); if (retriever->elems == NULL) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Out of memory :-("); return (-1); } if ((!strcmp(sortColumn, "column_vlan")) || (!strcmp(sortColumn, "column_"))) retriever->sorter = column_vlan, sorter = numericSorter; else if (!strcmp(sortColumn, "column_since")) retriever->sorter = column_since, sorter = numericSorter; else if (!strcmp(sortColumn, "column_thpt")) retriever->sorter = column_thpt, sorter = numericSorter; else if (!strcmp(sortColumn, "column_score")) retriever->sorter = column_score, sorter = numericSorter; else if (!strcmp(sortColumn, "column_traffic")) retriever->sorter = column_traffic, sorter = numericSorter; else if (!strcmp(sortColumn, "column_hosts")) retriever->sorter = column_num_hosts, sorter = numericSorter; else ntop->getTrace()->traceEvent(TRACE_WARNING, "Unknown sort column %s", sortColumn), sorter = numericSorter; // make sure the caller has disabled the purge!! walker(&begin_slot, walk_all, walker_vlans, vlan_search_walker, (void *)retriever); qsort(retriever->elems, retriever->actNumEntries, sizeof(struct flowHostRetrieveList), sorter); return (retriever->actNumEntries); } /* **************************************************** */ int NetworkInterface::getActiveHostsList(lua_State *vm, u_int32_t *begin_slot, bool walk_all, u_int8_t bridge_iface_idx, AddressTree *allowed_hosts, bool host_details, LocationPolicy location, char *countryFilter, char *mac_filter, u_int16_t vlan_id, ndpi_os osFilter, u_int32_t asnFilter, int32_t networkFilter, u_int16_t pool_filter, bool filtered_hosts, bool blacklisted_hosts, u_int8_t ipver_filter, int proto_filter, TrafficType traffic_type_filter, u_int32_t device_ip, bool tsLua, bool anomalousOnly, bool dhcpOnly, const AddressTree *const cidr_filter, char *sortColumn, u_int32_t maxHits, u_int32_t toSkip, bool a2zSortOrder, bool useArrayFormat) { struct flowHostRetriever retriever; int count = 1; #ifdef NTOPNG_PRO QoEStats qoe; #endif #if DEBUG if (!walk_all) ntop->getTrace()->traceEvent(TRACE_NORMAL, "[BEGIN] %s(begin_slot=%u, walk_all=%u)", __FUNCTION__, *begin_slot, walk_all); #endif memset(&retriever, 0, sizeof(struct flowHostRetriever)); retriever.observationPointId = getLuaVMUservalue(vm, observationPointId); #ifdef NTOPNG_PRO retriever.qoe = &qoe; #endif if (sortHosts(begin_slot, walk_all, &retriever, bridge_iface_idx, allowed_hosts, host_details, location, countryFilter, mac_filter, vlan_id, osFilter, asnFilter, networkFilter, pool_filter, filtered_hosts, blacklisted_hosts, anomalousOnly, dhcpOnly, cidr_filter, ipver_filter, proto_filter, traffic_type_filter, device_ip, sortColumn) < 0) { return (-1); } #if DEBUG if (!walk_all) ntop->getTrace()->traceEvent( TRACE_NORMAL, "[END] %s(end_slot=%u, numHosts=%u)", __FUNCTION__, *begin_slot, retriever.actNumEntries); #endif lua_newtable(vm); lua_push_uint64_table_entry(vm, "numHosts", retriever.actNumEntries); lua_push_uint64_table_entry(vm, "nextSlot", *begin_slot); lua_newtable(vm); if (a2zSortOrder) { for (int i = toSkip, num = 0; i < (int)retriever.actNumEntries && num < (int)maxHits; i++, num++) { Host *h = retriever.elems[i].hostValue; if (h != NULL) { if (!tsLua) { if(useArrayFormat) { h->lua(vm, NULL /* Already checked */, host_details, false, false, false); lua_pushinteger(vm, count++); lua_insert(vm, -2); lua_settable(vm, -3); } else { h->lua(vm, NULL /* Already checked */, host_details, false, false, true); } } else h->lua_get_timeseries(vm); } } } else { for (int i = (retriever.actNumEntries - 1 - toSkip), num = 0; i >= 0 && num < (int)maxHits; i--, num++) { Host *h = retriever.elems[i].hostValue; if(h != NULL) { if (!tsLua) { if(useArrayFormat) { h->lua(vm, NULL /* Already checked */, host_details, false, false, false); lua_pushinteger(vm, count++); lua_insert(vm, -2); lua_settable(vm, -3); } else { h->lua(vm, NULL /* Already checked */, host_details, false, false, true); } } else h->lua_get_timeseries(vm); } } } lua_pushstring(vm, "hosts"); lua_insert(vm, -2); lua_settable(vm, -3); for (u_int i = 0; i < retriever.actNumEntries; i++) { if (retriever.elems[i].hostValue) retriever.elems[i].hostValue->decUses(); /* See (***) */ } // it's up to us to clean sorted data // make sure first to free elements in case a string sorter has been used if (retriever.sorter == column_name || retriever.sorter == column_country || retriever.sorter == column_os) { for (u_int i = 0; i < retriever.maxNumEntries; i++) if (retriever.elems[i].stringValue) free((char *)retriever.elems[i].stringValue); } else if (retriever.sorter == column_local_network) for (u_int i = 0; i < retriever.maxNumEntries; i++) if (retriever.elems[i].ipValue) delete retriever.elems[i].ipValue; // finally free the elements regardless of the sorted kind if (retriever.elems) free(retriever.elems); return (retriever.actNumEntries); } /* **************************************************** */ struct hosts_get_macs_retriever { lua_State *vm; int idx; }; static bool hosts_get_macs(GenericHashEntry *he, void *user_data, bool *matched) { struct hosts_get_macs_retriever *r = (struct hosts_get_macs_retriever *)user_data; Host *host = (Host *)he; Mac *mac = host->getMac(); char mac_buf[32], *mac_ptr; char ip_buf[64]; if (mac && !mac->isSpecialMac() && host->get_ip()) { mac_ptr = Utils::formatMac(mac->get_mac(), mac_buf, sizeof(mac_buf)); lua_getfield(r->vm, r->idx, mac_ptr); if (lua_type(r->vm, -1) == LUA_TTABLE) { lua_getfield(r->vm, -1, "ip"); if (lua_type(r->vm, -1) == LUA_TNIL) { /* First assignment - create table */ lua_pop(r->vm, 1); lua_pushstring(r->vm, "ip"); lua_newtable(r->vm); lua_settable(r->vm, -3); lua_getfield(r->vm, -1, "ip"); } if (lua_type(r->vm, -1) == LUA_TTABLE) { /* Add the ip address to the table */ lua_push_uint64_table_entry(r->vm, host->get_hostkey(ip_buf, sizeof(ip_buf)), host->get_ip()->isIPv4() ? 4 : 6); } lua_pop(r->vm, 1); } lua_pop(r->vm, 1); *matched = true; } /* keep on iterating */ return (false); } /* **************************************************** */ int NetworkInterface::getMacsIpAddresses(lua_State *vm, int idx) { struct hosts_get_macs_retriever retriever; u_int32_t begin_slot = 0; bool walk_all = true; retriever.vm = vm; retriever.idx = idx; walker(&begin_slot, walk_all, walker_hosts, hosts_get_macs, (void *)&retriever); return 0; } /* **************************************************** */ static bool flow_stats_walker(GenericHashEntry *h, void *user_data, bool *matched) { struct active_flow_stats *stats = (struct active_flow_stats *)user_data; Flow *flow = (Flow *)h; NetworkInterface *iface = flow->getInterface(); if(iface) { u_int32_t proto_id = ndpi_map_user_proto_id_to_ndpi_id( iface->get_ndpi_struct(), flow->get_detected_protocol().proto.app_protocol); stats->num_flows++, stats->ndpi_bytes[proto_id] += (u_int32_t)flow->get_bytes(), stats->breeds_bytes[flow->get_protocol_breed()] += (u_int32_t)flow->get_bytes(); *matched = true; } return (false); /* false = keep on walking */ } /* **************************************************** */ void NetworkInterface::getFlowsStats(lua_State *vm) { struct active_flow_stats stats; u_int32_t begin_slot = 0; bool walk_all = true; memset(&stats, 0, sizeof(stats)); walker(&begin_slot, walk_all, walker_flows, flow_stats_walker, (void *)&stats); lua_newtable(vm); lua_push_uint64_table_entry(vm, "num_flows", stats.num_flows); lua_newtable(vm); for (int i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS + NDPI_MAX_NUM_CUSTOM_PROTOCOLS; i++) { if (stats.ndpi_bytes[i] > 0) lua_push_uint64_table_entry(vm, ndpi_get_proto_name(get_ndpi_struct(), i), stats.ndpi_bytes[i]); } lua_pushstring(vm, "protos"); lua_insert(vm, -2); lua_settable(vm, -3); lua_newtable(vm); for (int i = 0; i < NUM_BREEDS; i++) { if (stats.breeds_bytes[i] > 0) lua_push_uint64_table_entry( vm, ndpi_get_proto_breed_name((ndpi_protocol_breed_t)i), stats.breeds_bytes[i]); } lua_pushstring(vm, "breeds"); lua_insert(vm, -2); lua_settable(vm, -3); } /* **************************************************** */ void NetworkInterface::getNetworkStats(lua_State *vm, u_int32_t network_id, AddressTree *allowed_hosts, bool diff) const { NetworkStats *network_stats; if ((network_stats = getNetworkStats(network_id)) && network_stats->trafficSeen() && network_stats->match(allowed_hosts)) { lua_newtable(vm); network_stats->lua(vm, diff); lua_push_int32_table_entry(vm, "network_id", network_id); lua_pushstring(vm, ntop->getLocalNetworkName(network_id)); lua_insert(vm, -2); lua_settable(vm, -3); } } /* **************************************************** */ void NetworkInterface::getNetworksStats(lua_State *vm, AddressTree *allowed_hosts, bool diff) const { u_int32_t num_local_networks = ntop->getNumLocalNetworks(); lua_newtable(vm); for (u_int32_t network_id = 0; network_id < num_local_networks; network_id++) getNetworkStats(vm, network_id, allowed_hosts, diff); } /* **************************************************** */ /* Used to give the interface a new check loader to be used */ void NetworkInterface::reloadFlowChecks(FlowChecksLoader *fcbl) { /* Reload of the checks for this interface (e.g., interface type matters) */ FlowChecksExecutor *fce = new (std::nothrow) FlowChecksExecutor(fcbl, this); if (fce == NULL) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to reload checks on interface %s", ifname); return; } if (prev_flow_checks_executor) delete prev_flow_checks_executor; prev_flow_checks_executor = flow_checks_executor; flow_checks_executor = fce; } /* **************************************************** */ /* Used to give the interface a new check loader to be used */ void NetworkInterface::reloadHostChecks(HostChecksLoader *hcbl) { /* Reload of the checks for this interface (e.g., interface type matters) */ HostChecksExecutor *hce = new (std::nothrow) HostChecksExecutor(hcbl, this); if (hce == NULL) { ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to reload checks on interface %s", ifname); return; } if (prev_host_checks_executor) delete prev_host_checks_executor; prev_host_checks_executor = host_checks_executor; host_checks_executor = hce; } /* **************************************************** */ u_int NetworkInterface::purgeIdleFlows(bool force_idle, bool full_scan) { u_int n = 0; time_t last_packet_time = getTimeLastPktRcvd(); pollQueuedeCompanionEvents(); if (!force_idle && !full_scan && last_packet_time < next_idle_flow_purge) { return (0); /* Too early */ } else { /* Time to purge flows */ const struct timeval tv = periodicUpdateInitTime(); #if 0 ntop->getTrace()->traceEvent(TRACE_NORMAL, "Purging idle flows [ifname: %s] [ifid: %i] [current size: %i] [full scan: %s]", ifname, id, flows_hash->getNumEntries(), full_scan ? "True" : "False"); #endif n = (flows_hash ? flows_hash->purgeIdle(&tv, force_idle, full_scan) : 0); #ifdef NTOPNG_PRO ntop->getPro()->purgeIdleFlows(force_idle); #endif next_idle_flow_purge = last_packet_time + FLOW_PURGE_FREQUENCY; return (n); } } /* **************************************************** */ u_int64_t NetworkInterface::getNumPackets() { return (ethStats.getNumPackets()); }; /* **************************************************** */ u_int64_t NetworkInterface::getNumBytes() { return (ethStats.getNumBytes()); } /* **************************************************** */ u_int64_t NetworkInterface::getNumDroppedAlerts() { return num_host_dropped_alerts + num_flow_dropped_alerts + num_other_dropped_alerts; } /* **************************************************** */ u_int32_t NetworkInterface::getNumPacketDrops() { return (!isSubInterface() ? getNumDroppedPackets() : 0); }; /* **************************************************** */ u_int64_t NetworkInterface::getNumNewFlows() { return (num_new_flows); }; /* **************************************************** */ u_int NetworkInterface::getNumFlows() { return (flows_hash ? flows_hash->getNumEntries() : 0); }; /* **************************************************** */ u_int NetworkInterface::getNumHTTPHosts() { return (hosts_hash ? hosts_hash->getNumHTTPEntries() : 0); }; /* **************************************************** */ u_int NetworkInterface::getNumMacs() { return (macs_hash ? macs_hash->getNumEntries() : 0); }; /* **************************************************** */ u_int NetworkInterface::purgeIdleHosts(bool force_idle, bool full_scan) { time_t last_packet_time = getTimeLastPktRcvd(); if (!force_idle && !full_scan && last_packet_time < next_idle_host_purge) return (0); /* Too early */ else { /* Time to purge hosts */ const struct timeval tv = periodicUpdateInitTime(); u_int n; /* If the interface is no longer running it is safe to force all entries as * idle */ #if 0 ntop->getTrace()->traceEvent(TRACE_NORMAL, "Purging idle hosts [ifname: %s] [ifid: %i] [current size: %i]", ifname, id, hosts_hash->getNumEntries()); #endif // ntop->getTrace()->traceEvent(TRACE_INFO, "Purging idle hosts"); n = (hosts_hash ? hosts_hash->purgeIdle(&tv, force_idle, full_scan) : 0); next_idle_host_purge = last_packet_time + HOST_PURGE_FREQUENCY; return (n); } } /* **************************************************** */ u_int NetworkInterface::purgeIdleMacsASesCountriesVLANs(bool force_idle, bool full_scan) { time_t last_packet_time = getTimeLastPktRcvd(); if (!force_idle && !full_scan && last_packet_time < next_idle_other_purge) return (0); /* Too early */ else { /* Time to purge */ const struct timeval tv = periodicUpdateInitTime(); u_int n; /* If the interface is no longer running it is safe to force all entries as * idle */ n = (macs_hash ? macs_hash->purgeIdle(&tv, force_idle, full_scan) : 0) + (ases_hash ? ases_hash->purgeIdle(&tv, force_idle, full_scan) : 0) + (countries_hash ? countries_hash->purgeIdle(&tv, force_idle, full_scan) : 0) + (vlans_hash ? vlans_hash->purgeIdle(&tv, force_idle, full_scan) : 0) + (obs_hash ? obs_hash->purgeIdle(&tv, force_idle, full_scan) : 0); next_idle_other_purge = last_packet_time + OTHER_PURGE_FREQUENCY; return (n); } } /* **************************************************** */ void NetworkInterface::getnDPIProtocols(lua_State *vm, ndpi_protocol_category_t filter, bool skip_critical) { int i; u_int num_supported_protocols = ndpi_get_ndpi_num_supported_protocols(get_ndpi_struct()); ndpi_proto_defaults_t *proto_defaults = ndpi_get_proto_defaults(get_ndpi_struct()); lua_newtable(vm); for (i = 0; i < (int)num_supported_protocols; i++) { char buf[16]; if (((filter == NDPI_PROTOCOL_ANY_CATEGORY) || proto_defaults[i].protoCategory == filter) && (!skip_critical || !Utils::isCriticalNetworkProtocol(i))) { snprintf(buf, sizeof(buf) - 1, "%d", ndpi_map_ndpi_id_to_user_proto_id(get_ndpi_struct(), i)); if (!proto_defaults[i].protoName) ntop->getTrace()->traceEvent(TRACE_NORMAL, "NULL protoname for index %d!!", i); else lua_push_str_table_entry(vm, proto_defaults[i].protoName, buf); } } } /* **************************************************** */ #define NUM_TCP_STATES 4 /* 0 = RST 1 = SYN 2 = Established 3 = FIN */ static bool num_flows_state_walker(GenericHashEntry *node, void *user_data, bool *matched) { Flow *flow = (Flow *)node; u_int32_t *num_flows = (u_int32_t *)user_data; if (flow->get_protocol() == IPPROTO_TCP) { if (flow->isTCPEstablished()) num_flows[2]++; else if (flow->isTCPConnecting()) num_flows[1]++; else if (flow->isTCPReset()) num_flows[0]++; else if (flow->isTCPClosed()) num_flows[3]++; } *matched = true; return (false /* keep walking */); } /* *************************************** */ static bool num_flows_walker(GenericHashEntry *node, void *user_data, bool *matched) { Flow *flow = (Flow *)node; u_int32_t *num_flows = (u_int32_t *)user_data; NetworkInterface *iface = flow->getInterface(); if(iface) { u_int32_t proto_id = ndpi_map_user_proto_id_to_ndpi_id( iface->get_ndpi_struct(), flow->get_detected_protocol().proto.app_protocol); /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "Increasing usage of protocol %d, converted from %d", proto_id, flow->get_detected_protocol().proto.app_protocol); */ num_flows[proto_id]++; *matched = true; } return (false /* keep walking */); } /* *************************************** */ void NetworkInterface::getFlowsStatus(lua_State *vm) { u_int32_t num_flows[NUM_TCP_STATES] = {0}; u_int32_t begin_slot = 0; bool walk_all = true; walker(&begin_slot, walk_all, walker_flows, num_flows_state_walker, num_flows); lua_push_uint64_table_entry(vm, "RST", num_flows[0]); lua_push_uint64_table_entry(vm, "SYN", num_flows[1]); lua_push_uint64_table_entry(vm, "Established", num_flows[2]); lua_push_uint64_table_entry(vm, "FIN", num_flows[3]); } /* *************************************** */ void NetworkInterface::getnDPIFlowsCount(lua_State *vm) { u_int32_t *num_flows; u_int32_t begin_slot = 0; bool walk_all = true; u_int num_supported_protocols = ndpi_get_ndpi_num_supported_protocols(get_ndpi_struct()); ndpi_proto_defaults_t *proto_defaults = ndpi_get_proto_defaults(get_ndpi_struct()); /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "Allocating space for %d protocols", num_supported_protocols); */ num_flows = (u_int32_t *)calloc(num_supported_protocols, sizeof(u_int32_t)); if (num_flows) { walker(&begin_slot, walk_all, walker_flows, num_flows_walker, num_flows); for (int i = 0; i < (int)num_supported_protocols; i++) { if (num_flows[i] > 0) lua_push_uint64_table_entry(vm, proto_defaults[i].protoName, num_flows[i]); } free(num_flows); } } /* *************************************** */ void NetworkInterface::sumStats(TcpFlowStats *_tcpFlowStats, EthStats *_ethStats, LocalTrafficStats *_localStats, nDPIStats *_ndpiStats, PacketStats *_pktStats, TcpPacketStats *_tcpPacketStats, DSCPStats *_dscpStats, SyslogStats *_syslogStats, RoundTripStats *_downloadStats, RoundTripStats *_uploadStats) const { if (_tcpFlowStats) tcpFlowStats.sum(_tcpFlowStats); if (_ethStats) ethStats.sum(_ethStats); if (_localStats) localStats.sum(_localStats); if (_pktStats) pktStats.sum(_pktStats); if (_tcpPacketStats) tcpPacketStats.sum(_tcpPacketStats); if (_syslogStats) syslogStats.sum(_syslogStats); if (ndpiStats && _ndpiStats) ndpiStats->sum(_ndpiStats); if (dscpStats && _dscpStats) dscpStats->sum(_dscpStats); if (download_stats && _downloadStats) download_stats->sum(_downloadStats); if (upload_stats && _uploadStats) upload_stats->sum(_uploadStats); } /* *************************************** */ void NetworkInterface::lua(lua_State *vm, bool fullStats) { char buf[32]; TcpFlowStats _tcpFlowStats; EthStats _ethStats; RoundTripStats _downloadStats; RoundTripStats _uploadStats; LocalTrafficStats _localStats; nDPIStats _ndpiStats; PacketStats _pktStats; TcpPacketStats _tcpPacketStats; DSCPStats _dscpStats; SyslogStats _syslogStats; lua_newtable(vm); lua_push_str_table_entry(vm, "name", get_name()); lua_push_str_table_entry(vm, "description", get_description()); lua_push_uint64_table_entry(vm, "scalingFactor", getScalingFactor()); lua_push_int32_table_entry(vm, "id", id); lua_push_str_table_entry(vm, "mac", Utils::formatMac(ifMac, buf, sizeof(buf))); if (customIftype) lua_push_str_table_entry(vm, "customIftype", (char *)customIftype); lua_push_bool_table_entry(vm, "isView", isView()); /* View interface */ lua_push_bool_table_entry(vm, "isViewed", isViewed()); /* Viewed interface */ lua_push_bool_table_entry(vm, "isSampledTraffic", isSampledTraffic()); /* Whether this interface has sampled traffic */ lua_push_bool_table_entry(vm, "isLoading", isLoading()); if (isSubInterface()) luaSubInterface(vm); #ifdef NTOPNG_PRO #ifndef HAVE_NEDGE lua_push_bool_table_entry( vm, "hasSubInterfaces", (sub_interfaces && sub_interfaces->getNumSubInterfaces()) || (flowHashingMode != flowhashing_none)); #endif #endif lua_push_bool_table_entry(vm, "isFlowDumpDisabled", isFlowDumpDisabled()); lua_push_bool_table_entry(vm, "isFlowDumpRunning", db != NULL); lua_push_uint64_table_entry(vm, "seen.last", getTimeLastPktRcvd()); lua_push_bool_table_entry(vm, "inline", get_inline_interface()); lua_push_bool_table_entry(vm, "vlan", hasSeenVLANTaggedPackets()); lua_push_bool_table_entry(vm, "has_macs", hasSeenMacAddresses()); lua_push_bool_table_entry(vm, "has_seen_dhcp_addresses", hasSeenDHCPAddresses()); /* Note: source MAC is now used to get traffic direction when not * areTrafficDirectionsSupported() */ lua_push_bool_table_entry( vm, "has_traffic_directions", (areTrafficDirectionsSupported() || (!Utils::isEmptyMac(ifMac))) && (!isLoopback()) /* && (!isTrafficMirrored() || isGwMacConfigured())*/); if(fullStats) { lua_push_bool_table_entry(vm, "has_seen_pods", hasSeenPods()); lua_push_bool_table_entry(vm, "has_seen_containers", hasSeenContainers()); lua_push_bool_table_entry(vm, "has_seen_external_alerts", hasSeenExternalAlerts()); lua_push_bool_table_entry(vm, "has_seen_ebpf_events", hasSeenEBPFEvents()); luaNumEngagedAlerts(vm); luaAlertedFlows(vm); lua_push_uint64_table_entry(vm, "num_dropped_alerts", getNumDroppedAlertsSinceReset()); /* Those counters are absolute, i.e., they are not subject to reset */ lua_push_uint64_table_entry(vm, "num_host_dropped_alerts", num_host_dropped_alerts); lua_push_uint64_table_entry(vm, "num_flow_dropped_alerts", num_flow_dropped_alerts); lua_push_uint64_table_entry(vm, "num_other_dropped_alerts", num_other_dropped_alerts); lua_push_uint64_table_entry(vm, "periodic_stats_update_frequency_secs", periodicStatsUpdateFrequency()); } /* .stats */ lua_newtable(vm); lua_push_uint64_table_entry(vm, "packets", getNumPackets()); lua_push_uint64_table_entry(vm, "bytes", getNumBytes()); lua_push_uint64_table_entry(vm, "flows", getNumFlows()); lua_push_uint64_table_entry(vm, "hosts", getNumHosts()); lua_push_uint64_table_entry(vm, "hosts_rcvd_only", getNumRxOnlyHosts()); lua_push_uint64_table_entry(vm, "local_hosts", getNumLocalHosts()); lua_push_uint64_table_entry(vm, "local_rcvd_only_hosts", getNumLocalRxOnlyHosts()); lua_push_uint64_table_entry(vm, "http_hosts", getNumHTTPHosts()); lua_push_uint64_table_entry(vm, "drops", getNumPacketDrops()); lua_push_uint64_table_entry(vm, "new_flows", getNumNewFlows()); lua_push_uint64_table_entry(vm, "devices", getNumL2Devices()); lua_push_uint64_table_entry(vm, "current_macs", getNumMacs()); lua_push_uint64_table_entry(vm, "num_live_captures", num_live_captures); lua_push_float_table_entry(vm, "throughput_bps", bytes_thpt.getThpt()); lua_push_uint64_table_entry(vm, "throughput_trend_bps", bytes_thpt.getTrend()); lua_push_float_table_entry(vm, "throughput_pps", pkts_thpt.getThpt()); lua_push_uint64_table_entry(vm, "throughput_trend_pps", pkts_thpt.getTrend()); l4Stats.luaStats(vm); if(fullStats) { if (db) db->lua(vm, false /* Overall */); usedPorts.lua(vm, this); } lua_pushstring(vm, "stats"); lua_insert(vm, -2); lua_settable(vm, -3); lua_newtable(vm); lua_push_uint64_table_entry(vm, "packets", getNumPacketsSinceReset()); lua_push_uint64_table_entry(vm, "bytes", getNumBytesSinceReset()); lua_push_uint64_table_entry(vm, "drops", getNumPacketDropsSinceReset()); if (db) db->lua(vm, true /* Since last checkpoint */); lua_pushstring(vm, "stats_since_reset"); lua_insert(vm, -2); lua_settable(vm, -3); lua_push_uint64_table_entry(vm, "remote_pps", last_remote_pps); lua_push_uint64_table_entry(vm, "remote_bps", last_remote_bps); lua_push_uint64_table_entry(vm, "remote_update", last_remote_update); icmp_v4.lua(true, vm); icmp_v6.lua(false, vm); lua_push_uint64_table_entry(vm, "arp.requests", arp_requests); lua_push_uint64_table_entry(vm, "arp.replies", arp_replies); lua_push_str_table_entry(vm, "type", get_type()); lua_push_uint64_table_entry(vm, "speed", ifSpeed); lua_push_uint64_table_entry(vm, "mtu", ifMTU); lua_push_str_table_entry(vm, "ip_addresses", (char *)getLocalIPAddresses()); if(fullStats) { bcast_domains->lua(vm); if (top_sites && ntop->getPrefs()->are_top_talkers_enabled()) top_sites->lua(vm, (char *)"sites", (char *)"sites.old"); luaAnomalies(vm); luaScore(vm); } sumStats(&_tcpFlowStats, &_ethStats, &_localStats, &_ndpiStats, &_pktStats, &_tcpPacketStats, &_dscpStats, &_syslogStats, &_downloadStats, &_uploadStats); if(fullStats) { _downloadStats.luaRTStats(vm, "download_stats"); _uploadStats.luaRTStats(vm, "upload_stats"); } _tcpFlowStats.lua(vm, "tcpFlowStats"); _ethStats.lua(vm); _localStats.lua(vm); _ndpiStats.lua(this, vm, true, false); lua_push_uint64_table_entry(vm, "traffic_sent_since_reset", _ethStats.getNumEgressBytes() - getCheckPointNumTrafficSent()); lua_push_uint64_table_entry(vm, "traffic_rcvd_since_reset", _ethStats.getNumIngressBytes() - getCheckPointNumTrafficRcvd()); lua_push_uint64_table_entry(vm, "packets_sent_since_reset", _ethStats.getNumEgressPackets() - getCheckPointNumPacketsSent()); lua_push_uint64_table_entry(vm, "packets_rcvd_since_reset", _ethStats.getNumIngressPackets() - getCheckPointNumPacketsRcvd()); #ifdef NTOPNG_PRO qoe_stats.lua_qoe_stats(vm); #endif if(fullStats) { _pktStats.lua(vm, "pktSizeDistribution"); _tcpPacketStats.lua(vm, "tcpPacketStats"); _dscpStats.lua(this, vm); _syslogStats.lua(vm); } if (!isView()) { #ifdef NTOPNG_PRO #ifndef HAVE_NEDGE if (flow_profiles) flow_profiles->lua(vm); #endif #endif } if(fullStats) { if (host_pools) host_pools->lua(vm); #ifdef NTOPNG_PRO traffic_rx_behavior->luaBehavior(vm, "traffic_rx_behavior"); traffic_tx_behavior->luaBehavior(vm, "traffic_tx_behavior"); score_behavior->luaBehavior(vm, "score_behavior"); if (custom_app_stats) custom_app_stats->lua(vm); if (pMap) pMap->lua(vm, "periodicity_map"); if (sMap) sMap->lua(vm, "service_map"); #endif } } /* *************************************** */ void NetworkInterface::luaSubInterface(lua_State *vm) { char buf[64]; switch (dynamic_interface_mode) { case flowhashing_probe_ip: lua_push_str_table_entry( vm, "dynamic_interface_probe_ip", Utils::intoaV4(dynamic_interface_criteria, buf, sizeof(buf))); break; case flowhashing_probe_ip_and_ingress_iface_idx: lua_push_str_table_entry( vm, "dynamic_interface_probe_ip", Utils::intoaV4(dynamic_interface_criteria >> 32, buf, sizeof(buf))); lua_push_uint64_table_entry(vm, "dynamic_interface_inifidx", dynamic_interface_criteria & 0xFFFFFFFF); break; default: break; } } /* *************************************** */ void NetworkInterface::luaServiceMapStatus(lua_State *vm) { #if defined(NTOPNG_PRO) lua_newtable(vm); if (sMap) lua_push_bool_table_entry(vm, "service_map_learning_status", sMap->isLearning()); #endif } /* *************************************** */ void NetworkInterface::lua_hash_tables_stats(lua_State *vm) { /* Hash tables stats */ GenericHash *gh[] = {flows_hash, hosts_hash, macs_hash, vlans_hash, ases_hash, countries_hash, obs_hash}; lua_newtable(vm); for (u_int i = 0; i < sizeof(gh) / sizeof(gh[0]); i++) { if (gh[i]) gh[i]->lua(vm); } } /* *************************************** */ void NetworkInterface::lua_periodic_activities_stats(lua_State *vm) { lua_newtable(vm); /* Periodic activities stats */ ntop->lua_periodic_activities_stats(this, vm); } /* **************************************************** */ void NetworkInterface::lua_queues_stats(lua_State *vm) { if (idleFlowsToDump) idleFlowsToDump->lua(vm); if (activeFlowsToDump) activeFlowsToDump->lua(vm); if (flowAlertsQueue) flowAlertsQueue->lua(vm); } /* **************************************************** */ void NetworkInterface::runPeriodicHousekeepingTasks() { periodicStatsUpdate(); } /* **************************************************** */ void NetworkInterface::runShutdownTasks() { /* NOTE NOTE NOTE * This task runs asynchronously with respect to the datapath * which has been already stopped */ /* Run the periodic stats update one last time so certain tasks can be properly finalized, e.g., all hosts and all flows can be marked as idle */ periodicStatsUpdate(); #ifdef NTOPNG_PRO flushFlowDump(); #endif } /* **************************************************** */ Mac *NetworkInterface::getMac(u_int8_t _mac[6], bool create_if_not_present, bool isInlineCall) { Mac *ret = NULL; if (!_mac || !macs_hash) return (NULL); ret = macs_hash->get(_mac, isInlineCall); if ((ret == NULL) && create_if_not_present) { if (!macs_hash->hasEmptyRoom()) return (NULL); try { if ((ret = new Mac(this, _mac)) != NULL) { if(!macs_hash->add(ret, !isInlineCall /* Lock only if not inline, if inline there's no need to lock as also the purgeIdle is done inline*/)) { /* Note: this should never happen as we are checking hasEmptyRoom() */ delete ret; return (NULL); } } } catch (std::bad_alloc &ba) { static bool oom_warning_sent = false; if (!oom_warning_sent) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Not enough memory"); oom_warning_sent = true; } return (NULL); } } return (ret); } /* **************************************************** */ VLAN *NetworkInterface::getVLAN(u_int16_t u_int16_t, bool create_if_not_present, bool isInlineCall) { VLAN *ret = NULL; if (!vlans_hash) return (NULL); ret = vlans_hash->get(u_int16_t, isInlineCall); if ((ret == NULL) && create_if_not_present) { if (!vlans_hash->hasEmptyRoom()) return (NULL); try { if ((ret = new VLAN(this, u_int16_t)) != NULL) { if(!vlans_hash->add(ret, !isInlineCall /* Lock only if not inline, if inline there is no need to lock as we are sequential with the purgeIdle */)) { /* Note: this should never happen as we are checking hasEmptyRoom() */ delete ret; return (NULL); } } } catch (std::bad_alloc &ba) { static bool oom_warning_sent = false; if (!oom_warning_sent) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Not enough memory"); oom_warning_sent = true; } return (NULL); } } return (ret); } /* **************************************************** */ AutonomousSystem *NetworkInterface::getAS(IpAddress *ipa, bool create_if_not_present, bool is_inline_call) { AutonomousSystem *ret = NULL; if ((!ipa) || (!ases_hash)) return (NULL); ret = ases_hash->get(ipa, is_inline_call); if ((ret == NULL) && create_if_not_present) { if (!ases_hash->hasEmptyRoom()) return (NULL); try { if ((ret = new AutonomousSystem(this, ipa)) != NULL) { if(!ases_hash->add(ret, !is_inline_call /* Lock only if not inline, if inline there is no need to lock as we are sequential with the purgeIdle */)) { /* Note: this should never happen as we are checking hasEmptyRoom() */ delete ret; return (NULL); } } } catch (std::bad_alloc &ba) { static bool oom_warning_sent = false; if (!oom_warning_sent) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Not enough memory"); oom_warning_sent = true; } return (NULL); } } return (ret); } /* **************************************************** */ ObservationPoint *NetworkInterface::getObsPoint(u_int16_t obs_point, bool create_if_not_present, bool is_inline_call) { ObservationPoint *ret = NULL; if ((!obs_point) || (obs_point == (u_int16_t)-1) || (!obs_hash)) return (NULL); ret = obs_hash->get(obs_point, is_inline_call); if ((ret == NULL) && create_if_not_present) { if (!obs_hash->hasEmptyRoom()) return (NULL); try { if ((ret = new ObservationPoint(this, obs_point)) != NULL) { if(!obs_hash->add(ret, !is_inline_call /* Lock only if not inline, if inline there is no need to lock as we are sequential with the purgeIdle */)) { /* Note: this should never happen as we are checking hasEmptyRoom() */ delete ret; return (NULL); } } } catch (std::bad_alloc &ba) { static bool oom_warning_sent = false; if (!oom_warning_sent) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Not enough memory"); oom_warning_sent = true; } return (NULL); } } last_obs_point_id = obs_point; return (ret); } /* **************************************************** */ bool NetworkInterface::deleteObsPoint(u_int16_t obs_point) { ObservationPoint *ret = NULL; /* Invalid args given */ if ((!obs_point) || (obs_point == (u_int16_t)-1) || (!obs_hash)) return (false); ret = obs_hash->get(obs_point, true); /* Observation Point found, delete it */ if (ret != NULL) ret->deleteObsStats(); return (true); } /* **************************************************** */ bool NetworkInterface::prepareDeleteObsPoint(u_int16_t obs_point) { ObservationPoint *ret = NULL; /* Invalid args given */ if ((!obs_point) || (obs_point == (u_int16_t)-1) || (!obs_hash)) return (false); ret = obs_hash->get(obs_point, true); /* Observation Point found, delete it */ if (ret != NULL) ret->setDeleteRequested(true); return (true); } /* **************************************************** */ Country *NetworkInterface::getCountry(const char *country_name, bool create_if_not_present, bool is_inline_call) { Country *ret = NULL; if ((!countries_hash) || (!country_name) || (!country_name[0])) return (NULL); ret = countries_hash->get(country_name, is_inline_call); if ((ret == NULL) && create_if_not_present) { if (!countries_hash->hasEmptyRoom()) return (NULL); try { if ((ret = new Country(this, country_name)) != NULL) { if(!countries_hash->add(ret, !is_inline_call /* Lock only if not inline, if inline there is no need to lock as we are sequential with the purgeIdle */)) { /* Note: this should never happen as we are checking hasEmptyRoom() */ delete ret; return (NULL); } } } catch (std::bad_alloc &ba) { static bool oom_warning_sent = false; if (!oom_warning_sent) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Not enough memory"); oom_warning_sent = true; } return (NULL); } } return (ret); } /* **************************************************** */ Flow *NetworkInterface::findFlowByKeyAndHashId(u_int32_t key, u_int hash_id, AddressTree *allowed_hosts) { Flow *f = NULL; if (!flows_hash) return (NULL); f = flows_hash->findByKeyAndHashId(key, hash_id); if (f && (!f->match(allowed_hosts))) f = NULL; return (f); } /* **************************************************** */ Flow *NetworkInterface::findFlowByTuple(u_int16_t vlan_id, u_int16_t observation_domain_id, u_int32_t private_flow_id, Mac *src_mac, Mac *dst_mac, IpAddress *src_ip, IpAddress *dst_ip, u_int16_t src_port, u_int16_t dst_port, u_int8_t l4_proto, AddressTree *allowed_hosts) const { bool src2dst; Flow *f, *unswapped_flow; if (!flows_hash) return (NULL); f = (Flow *)flows_hash->find(src_mac, dst_mac, src_ip, dst_ip, src_port, dst_port, vlan_id, observation_domain_id, private_flow_id, l4_proto, NULL, &src2dst, false /* Not an inline call */, &unswapped_flow); if (f && (!f->match(allowed_hosts))) f = NULL; return (f); } /* **************************************************** */ struct search_host_info { lua_State *vm; char *host_name_or_ip; u_int num_matches; AddressTree *allowed_hosts; }; /* **************************************************** */ static bool hosts_search_walker(GenericHashEntry *h, void *user_data, bool *matched) { Host *host = (Host *)h; struct search_host_info *info = (struct search_host_info *)user_data; if (host->addIfMatching(info->vm, info->allowed_hosts, info->host_name_or_ip)) { info->num_matches++; *matched = true; } /* Stop after CONST_MAX_NUM_FIND_HITS matches */ return ((info->num_matches > CONST_MAX_NUM_FIND_HITS) ? true /* stop */ : false /* keep walking */); } /* **************************************************** */ struct search_mac_info { lua_State *vm; u_int8_t *mac; u_int num_matches; }; /* **************************************************** */ struct search_bost_by_mac { u_int8_t *mac; Host *match; }; /* **************************************************** */ static bool macs_search_walker(GenericHashEntry *h, void *user_data, bool *matched) { Host *host = (Host *)h; struct search_mac_info *info = (struct search_mac_info *)user_data; if (host->addIfMatching(info->vm, info->mac)) { info->num_matches++; *matched = true; } /* Stop after CONST_MAX_NUM_FIND_HITS matches */ return ((info->num_matches > CONST_MAX_NUM_FIND_HITS) ? true /* stop */ : false /* keep walking */); } /* **************************************************** */ static bool first_mac_search_walker(GenericHashEntry *h, void *user_data, bool *matched) { Host *host = (Host *)h; struct search_bost_by_mac *info = (struct search_bost_by_mac *)user_data; if (memcmp(info->mac, host->get_mac(), 6) == 0) { info->match = host; return (true /* stop */); } else return (false); } /* *************************************** */ bool NetworkInterface::findHostsByMac(lua_State *vm, u_int8_t *mac) { struct search_mac_info info; u_int32_t begin_slot = 0; bool walk_all = true; info.vm = vm, info.mac = mac, info.num_matches = 0; lua_newtable(vm); walker(&begin_slot, walk_all, walker_hosts, macs_search_walker, (void *)&info); return (info.num_matches > 0); } /* **************************************************** */ Host *NetworkInterface::findHostByMac(u_int8_t *mac) { struct search_bost_by_mac info; u_int32_t begin_slot = 0; bool walk_all = true; info.mac = mac, info.match = NULL; walker(&begin_slot, walk_all, walker_hosts, first_mac_search_walker, (void *)&info); #if 0 if(info.match) { char buf[64], buf1[64]; ntop->getTrace()->traceEvent(TRACE_NORMAL, "*** %s %s ***", info.match->get_visual_name(buf, sizeof(buf)), info.match->printMask(buf1, sizeof(buf1))); } #endif return (info.match); } /* **************************************************** */ bool NetworkInterface::findHostsByName(lua_State *vm, AddressTree *allowed_hosts, char *key) { struct search_host_info info; u_int32_t begin_slot = 0; bool walk_all = true; info.vm = vm, info.host_name_or_ip = key, info.num_matches = 0, info.allowed_hosts = allowed_hosts; lua_newtable(vm); walker(&begin_slot, walk_all, walker_hosts, hosts_search_walker, (void *)&info); return (info.num_matches > 0); } /* **************************************************** */ u_int NetworkInterface::printAvailableInterfaces(bool printHelp, int idx, char *ifname, u_int ifname_len) { int numInterfaces = 0; ntop_if_t *devpointer, *cur; if (printHelp && help_printed) return (0); if (Utils::ntop_findalldevs(&devpointer)) { ; } else { if (ifname == NULL) { if (printHelp) printf("Available interfaces (-i ):\n"); else if (!help_printed) ntop->getTrace()->traceEvent( TRACE_NORMAL, "Available interfaces (-i ):"); } for (cur = devpointer; cur; cur = cur->next) { if (Utils::validInterface(cur)) { numInterfaces++; if (ifname == NULL) { if (printHelp) { #ifdef WIN32 printf( " %d. %s\n" "\t%s\n", numInterfaces, cur->description ? cur->description : "", cur->name); #else printf(" %d. %s\n", numInterfaces, cur->name); #endif } else if (!help_printed) ntop->getTrace()->traceEvent( TRACE_NORMAL, "%d. %s (%s)\n", numInterfaces, cur->name, cur->description ? cur->description : cur->name); } else if (numInterfaces == idx) { snprintf(ifname, ifname_len, "%s", cur->name); break; } } } /* for */ Utils::ntop_freealldevs(devpointer); } /* else */ if (numInterfaces == 0) { #ifdef WIN32 ntop->getTrace()->traceEvent( TRACE_WARNING, "No interfaces available! This application cannot work"); ntop->getTrace()->traceEvent( TRACE_WARNING, "Make sure that https://nmap.org/npcap/ is installed,"); ntop->getTrace()->traceEvent(TRACE_WARNING, "that you have administrative rights,"); ntop->getTrace()->traceEvent( TRACE_WARNING, "and that you have network interfaces installed."); #else ntop->getTrace()->traceEvent(TRACE_WARNING, "No interfaces available: are you superuser?"); #endif } help_printed = true; return (numInterfaces); } /* **************************************************** */ bool NetworkInterface::isNumber(const char *str) { while (*str) { if (!isdigit(*str)) return (false); str++; } return (true); } /* **************************************************** */ struct proc_name_flows { lua_State *vm; char *proc_name; }; static bool proc_name_finder_walker(GenericHashEntry *node, void *user_data, bool *matched) { Flow *f = (Flow *)node; struct proc_name_flows *info = (struct proc_name_flows *)user_data; char *name = f->get_proc_name(true); if (name && (strcmp(name, info->proc_name) == 0)) { f->lua(info->vm, NULL, details_normal /* Minimum details */, false); lua_pushinteger(info->vm, f->key()); // Key lua_insert(info->vm, -2); lua_settable(info->vm, -3); } else { name = f->get_proc_name(false); if (name && (strcmp(name, info->proc_name) == 0)) { f->lua(info->vm, NULL, details_normal /* Minimum details */, false); lua_pushinteger(info->vm, f->key()); // Key lua_insert(info->vm, -2); lua_settable(info->vm, -3); } } *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ void NetworkInterface::findProcNameFlows(lua_State *vm, char *proc_name) { struct proc_name_flows u; u_int32_t begin_slot = 0; bool walk_all = true; u.vm = vm, u.proc_name = proc_name; lua_newtable(vm); walker(&begin_slot, walk_all, walker_flows, proc_name_finder_walker, &u); } /* **************************************************** */ struct pid_flows { lua_State *vm; u_int32_t pid; }; static bool pidfinder_walker(GenericHashEntry *node, void *pid_data, bool *matched) { Flow *f = (Flow *)node; struct pid_flows *info = (struct pid_flows *)pid_data; if ((f->getPid(true) == info->pid) || (f->getPid(false) == info->pid)) { f->lua(info->vm, NULL, details_normal /* Minimum details */, false); lua_pushinteger(info->vm, f->key()); // Key lua_insert(info->vm, -2); lua_settable(info->vm, -3); *matched = true; } return (false); /* false = keep on walking */ } /* **************************************** */ void NetworkInterface::findPidFlows(lua_State *vm, u_int32_t pid) { struct pid_flows u; u_int32_t begin_slot = 0; bool walk_all = true; u.vm = vm, u.pid = pid; lua_newtable(vm); walker(&begin_slot, walk_all, walker_flows, pidfinder_walker, &u); } /* **************************************** */ struct virtual_host_valk_info { lua_State *vm; char *key; u_int32_t num; }; /* **************************************** */ static bool virtual_http_hosts_walker(GenericHashEntry *node, void *data, bool *matched) { Host *h = (Host *)node; struct virtual_host_valk_info *info = (struct virtual_host_valk_info *)data; HTTPstats *s = h->getHTTPstats(); if (s) { info->num += s->luaVirtualHosts(info->vm, info->key, h); *matched = true; } return (false); /* false = keep on walking */ } /* **************************************** */ bool NetworkInterface::alert_store_query(lua_State *vm, const char *sql, bool limit_rows) { if (!alertStore) return false; return alertStore->query(vm, sql, limit_rows); } /* **************************************** */ void NetworkInterface::listHTTPHosts(lua_State *vm, char *key) { struct virtual_host_valk_info info; u_int32_t begin_slot = 0; bool walk_all = true; lua_newtable(vm); info.vm = vm, info.key = key, info.num = 0; walker(&begin_slot, walk_all, walker_hosts, virtual_http_hosts_walker, &info); } /* **************************************** */ void NetworkInterface::addAllAvailableInterfaces() { ntop_if_t *devpointer, *cur; if (!Utils::ntop_findalldevs(&devpointer)) { for (cur = devpointer; cur; cur = cur->next) { if (Utils::validInterface(cur) && (strncmp(cur->name, "virbr", 5) != 0) /* Ignore virtual interfaces */ && Utils::isInterfaceUp(cur->name)) { ntop->getPrefs()->add_network_interface(cur->name, cur->description); } else ntop->getTrace()->traceEvent( TRACE_INFO, "Interface [%s][%s] not valid or down: discarded", cur->name, cur->description); } /* for */ Utils::ntop_freealldevs(devpointer); } } /* **************************************** */ #ifdef NTOPNG_PRO void NetworkInterface::refreshL7Rules() { if (ntop->getPro()->has_valid_license() && policer) policer->refreshL7Rules(); } #endif /* **************************************** */ #ifdef NTOPNG_PRO void NetworkInterface::refreshShapers() { if (ntop->getPro()->has_valid_license() && policer) policer->refreshShapers(); } #endif /* **************************************** */ void NetworkInterface::addInterfaceAddress(char *const addr) { if (ip_addresses.size() == 0) ip_addresses = addr; else { string s = addr; ip_addresses = ip_addresses + "," + s; } } /* **************************************** */ void NetworkInterface::addInterfaceNetwork(char *const net, char *addr) { /* E.g. 192.168.1.0/24 -> 192.168.1.42 */ char *addr_cp = strdup(addr); char *sep = NULL; if (addr_cp && (sep = strchr(addr_cp, '/'))) *sep = '\0'; interface_networks.addAddressAndData(net, addr_cp); } /* **************************************** */ bool NetworkInterface::isInterfaceNetwork(IpAddress *ipa, int network_bits) { return interface_networks.match(ipa, network_bits); } /* **************************************** */ void NetworkInterface::FillObsHash() { /* Adding all observation points to the hash map*/ if (obs_hash) { char **keys; char pattern[64]; int rc = 0; snprintf(pattern, sizeof(pattern), "ntopng.serialized_as.ifid_%u_obs_point_*", get_id()); // ntop->getTrace()->traceEvent(TRACE_INFO, "Pattern: %s", pattern); /* Get all Observation Points keys */ rc = ntop->getRedis()->keys(pattern, &keys); if (rc > 0) { for (int i = 0; i < rc; i++) { if (keys[i]) { char symbol = '_'; /* Get last occurrence of _ , because the key is serialized like * ntopng.serialized_as.ifid_10_obs_point_1234 */ /* In this way it's possible to get all the ids of the Obs. Points */ char *obs_point = strrchr(keys[i], symbol); if (obs_point) { u_int16_t obs_point_id = atoi(&obs_point[1]); if (!obs_point_id) { ntop->getTrace()->traceEvent( TRACE_ERROR, "Failed to deserialize Observation Point stats: %u", obs_point_id); if (keys[i]) free(keys[i]); continue; } /* Found at least one element */ /* Create a new observation point with the id found to deserialize * stats */ ObservationPoint *tmp_obs_point = new ObservationPoint(this, obs_point_id); last_obs_point_id = obs_point_id; /* Add to the map */ if (obs_hash->add(tmp_obs_point, false /* Do lock */)) ntop->getTrace()->traceEvent(TRACE_NORMAL, "Found Observation Point: %u; Stats " "deserialization complete.", obs_point_id); else ntop->getTrace()->traceEvent( TRACE_ERROR, "Failed to deserialize Observation Point stats: %u", obs_point_id); } } if (keys[i]) free(keys[i]); } } if (keys) free(keys); } } /* **************************************** */ void NetworkInterface::allocateStructures(bool disable_dump) { u_int32_t numNetworks = ntop->getNumLocalNetworks(); char buf[16]; try { if (get_id() >= 0) { u_int32_t num_hashes = max_val(4096, ntop->getPrefs()->get_max_num_flows() / 4); flows_hash = new FlowHash(this, num_hashes, ntop->getPrefs()->get_max_num_flows()); if(!flowsOnlyInterface() /* Do not allocate HTs when the interface should only have flows */ && !isViewed() /* Do not allocate HTs when the interface is viewed, HTs are allocated in the corresponding ViewInterface */) { num_hashes = max_val(4096, ntop->getPrefs()->get_max_num_hosts() / 4); hosts_hash = new HostHash(this, num_hashes, ntop->getPrefs()->get_max_num_hosts()); /* The number of ASes cannot be greater than the number of hosts */ ases_hash = new AutonomousSystemHash(this, ndpi_min(num_hashes, 4096), 32768); if (!isPacketInterface()) obs_hash = new ObservationPointHash(this, ndpi_min(num_hashes, 4096), 32768); countries_hash = new CountriesHash(this, ndpi_min(num_hashes, 1024), 32768); vlans_hash = new VLANHash(this, 1024, 2048); macs_hash = new MacHash(this, ndpi_min(num_hashes, 8192), 32768); } } FillObsHash(); networkStats = new NetworkStats *[numNetworks]; statsManager = new StatsManager(id, STATS_MANAGER_STORE_NAME); ndpiStats = new nDPIStats(true /* Enable throughput calculation */, ntop->getPrefs()->isIfaceL7BehavourAnalysisEnabled()); dscpStats = new DSCPStats(); download_stats = new RoundTripStats(); upload_stats = new RoundTripStats(); gw_macs = new MacHash(this, 32, 64); top_sites = new (std::nothrow) MostVisitedList(HOST_SITES_TOP_NUMBER); if (disable_dump) flow_dump_disabled_by_backend = true; if (db == NULL && !disable_dump) { if (ntop->getPrefs()->do_dump_flows_on_clickhouse()) { #ifdef NTOPNG_PRO #if defined(HAVE_CLICKHOUSE) && defined(HAVE_MYSQL) /* Allocate only the DB connection, not any thread or queue for the export */ try { db = new ClickHouseFlowDB(this); } catch (const std::invalid_argument &e) { db = NULL; ntop->getTrace()->traceEvent( TRACE_WARNING, "Leaving due to failed ClickHouse initialization"); exit(-1); } #endif #endif } } if (!isViewed() && !disable_dump) { #if defined(NTOPNG_PRO) && defined(HAVE_CLICKHOUSE) && defined(HAVE_MYSQL) if (ntop->getPrefs()->do_dump_alerts_on_clickhouse()) alertStore = new ClickHouseAlertStore(this); #endif if (alertStore == NULL) alertStore = new SQLiteAlertStore(id, ALERTS_STORE_DB_FILE_NAME); alertsQueue = new AlertsQueue(this); } for (u_int32_t i = 0; i < numNetworks; i++) networkStats[i] = new NetworkStats(this, i); } catch (std::bad_alloc &ba) { static bool oom_warning_sent = false; if (!oom_warning_sent) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Not enough memory"); oom_warning_sent = true; } } #if defined(NTOPNG_PRO) if (ntop->getPrefs() && ntop->getPro()->has_valid_license() && ntop->getPrefs()->isBehavourAnalysisEnabled() && (ntop->getPrefs()->is_enterprise_l_edition() || ntop->getPrefs()->is_nedge_enterprise_edition()) && ifname && strcmp(ifname, SYSTEM_INTERFACE_NAME) && !isViewed() /* Skip for viewed interface, only store service maps in the view to save memory */ ) { if(!ntop->getPrefs()->limitResourcesUsage()) pMap = new (std::nothrow) PeriodicityMap(this, ntop->getPrefs()->get_max_num_flows() / 8, 3600 /* 1h idleness */); sMap = new (std::nothrow) ServiceMap(this, ntop->getPrefs()->get_max_num_flows() / 8, 86400 /* 1d idleness */); } else pMap = NULL, sMap = NULL; acl_flow = new (std::nothrow) ACLFlow(); #ifndef HAVE_NEDGE updateFlowProfiles(); #endif #endif // Keep format in sync with alerts_api.interfaceAlertEntity(ifid) snprintf(buf, sizeof(buf), "%d", get_id()); setEntityValue(buf); reloadGwMacs(); removeRedisSitesKey(); } /* **************************************** */ AlertsQueue *NetworkInterface::getAlertsQueue() const { #ifdef NTOPNG_PRO if (isViewed()) return viewedBy()->getAlertsQueue(); else #endif return alertsQueue; } /* **************************************** */ NetworkStats *NetworkInterface::getNetworkStats(u_int32_t networkId) const { if ((networkStats == NULL) || (networkId >= ntop->getNumLocalNetworks())) return (NULL); else return (networkStats[networkId]); } /* **************************************** */ void NetworkInterface::checkPointCounters(bool drops_only) { if (!drops_only) { checkpointPktCount = getNumPackets(), checkpointBytesCount = getNumBytes(); } checkpointDroppedAlertsCount = getNumDroppedAlerts(); checkpointPktDropCount = getNumPacketDrops(); checkpointTrafficSent = getStats() ? getStats()->getNumEgressBytes() : 0; checkpointTrafficRcvd = getStats() ? getStats()->getNumIngressBytes() : 0; checkpointPacketsSent = getStats() ? getStats()->getNumEgressPackets() : 0; checkpointPacketsRcvd = getStats() ? getStats()->getNumIngressPackets() : 0; if (db) db->checkPointCounters(drops_only); } /* **************************************************** */ u_int64_t NetworkInterface::getCheckPointNumPackets() { return (checkpointPktCount); }; /* **************************************************** */ u_int64_t NetworkInterface::getCheckPointDroppedAlerts() { return (checkpointDroppedAlertsCount); }; /* **************************************************** */ u_int64_t NetworkInterface::getCheckPointNumBytes() { return (checkpointBytesCount); } /* **************************************************** */ u_int32_t NetworkInterface::getCheckPointNumPacketDrops() { return (checkpointPktDropCount); }; /* **************************************************** */ u_int64_t NetworkInterface::getCheckPointNumTrafficSent() const { return checkpointTrafficSent; } /* **************************************************** */ u_int64_t NetworkInterface::getCheckPointNumTrafficRcvd() const { return checkpointTrafficRcvd; } /* **************************************************** */ u_int64_t NetworkInterface::getCheckPointNumPacketsSent() const { return checkpointPacketsSent; } /* **************************************************** */ u_int64_t NetworkInterface::getCheckPointNumPacketsRcvd() const { return checkpointPacketsRcvd; } /* **************************************** */ void NetworkInterface::processInterfaceStats(sFlowInterfaceStats *stats) { if (interfaceStats == NULL) interfaceStats = new (std::nothrow) InterfaceStatsHash(NUM_IFACE_STATS_HASH); if (interfaceStats) { char a[64]; ntop->getTrace()->traceEvent(TRACE_INFO, "[%s][ifIndex=%u]", Utils::intoaV4(stats->deviceIP, a, sizeof(a)), stats->ifIndex); interfaceStats->set(stats); } } /* **************************************** */ void NetworkInterface::reloadGwMacs() { char kname[64]; char **macs = NULL; if (!ntop->getRedis()) return; gw_macs->cleanup(); snprintf(kname, sizeof(kname), CONST_IFACE_GW_MACS_PREFS, id); int num_macs = ntop->getRedis()->smembers(kname, &macs); for (int i = 0; i < num_macs; i++) { Mac *m = NULL; u_int8_t addr[6]; char *mac = macs[i]; if (!mac) continue; // ntop->getTrace()->traceEvent(TRACE_NORMAL, "Loading Gw MAC %s", mac); Utils::parseMac(addr, mac); try { if ((m = new Mac(this, addr)) != NULL) { if (!gw_macs->add(m, true)) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Not enough root in GW macs hash"); delete m; } } } catch (std::bad_alloc &ba) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Not enough memory"); } free(mac); } if (macs) free(macs); gw_macs_reload_requested = false; } /* **************************************** */ int NetworkInterface::getActiveMacList( lua_State *vm, u_int32_t *begin_slot, bool walk_all, u_int8_t bridge_iface_idx, bool sourceMacsOnly, const char *manufacturer, char *sortColumn, u_int32_t maxHits, u_int32_t toSkip, bool a2zSortOrder, u_int16_t pool_filter, u_int8_t devtype_filter, u_int8_t location_filter, time_t min_first_seen) { struct flowHostRetriever retriever; bool show_details = true; retriever.observationPointId = getLuaVMUservalue(vm, observationPointId); if (sortMacs(begin_slot, walk_all, &retriever, bridge_iface_idx, sourceMacsOnly, manufacturer, sortColumn, pool_filter, devtype_filter, location_filter, min_first_seen) < 0) { return (-1); } lua_newtable(vm); lua_push_uint64_table_entry(vm, "numMacs", retriever.actNumEntries); lua_push_uint64_table_entry(vm, "nextSlot", *begin_slot); lua_newtable(vm); if (a2zSortOrder) { for (int i = toSkip, num = 0; i < (int)retriever.actNumEntries && num < (int)maxHits; i++, num++) { Mac *m = retriever.elems[i].macValue; m->lua(vm, show_details, false); lua_rawseti(vm, -2, num + 1); /* Must use integer keys to preserve and iterate inorder with ipairs */ } } else { for (int i = (retriever.actNumEntries - 1 - toSkip), num = 0; i >= 0 && num < (int)maxHits; i--, num++) { Mac *m = retriever.elems[i].macValue; m->lua(vm, show_details, false); lua_rawseti(vm, -2, num + 1); } } lua_pushstring(vm, "macs"); lua_insert(vm, -2); lua_settable(vm, -3); // finally free the elements regardless of the sorted kind if (retriever.elems) free(retriever.elems); return (retriever.actNumEntries); } /* **************************************** */ int NetworkInterface::getActiveASList(lua_State *vm, const Paginator *p, bool diff) { struct flowHostRetriever retriever; DetailsLevel details_level; if (!p) return (-1); if (sortASes(&retriever, p->sortColumn()) < 0) { return (-1); } if (!p->getDetailsLevel(&details_level)) details_level = details_normal; lua_newtable(vm); lua_push_uint64_table_entry(vm, "numASes", retriever.actNumEntries); lua_newtable(vm); if (p->a2zSortOrder()) { for (int i = p->toSkip(), num = 0; i < (int)retriever.actNumEntries && num < (int)p->maxHits(); i++, num++) { AutonomousSystem *as = retriever.elems[i].asValue; as->lua(vm, details_level, false, diff); lua_rawseti(vm, -2, num + 1); /* Must use integer keys to preserve and iterate inorder with ipairs */ } } else { for (int i = (retriever.actNumEntries - 1 - p->toSkip()), num = 0; i >= 0 && num < (int)p->maxHits(); i--, num++) { AutonomousSystem *as = retriever.elems[i].asValue; as->lua(vm, details_level, false, diff); lua_rawseti(vm, -2, num + 1); } } lua_pushstring(vm, "ASes"); lua_insert(vm, -2); lua_settable(vm, -3); // finally free the elements regardless of the sorted kind if (retriever.elems) free(retriever.elems); return (retriever.actNumEntries); } /* **************************************** */ int NetworkInterface::getActiveObsPointsList(lua_State *vm, const Paginator *p) { struct flowHostRetriever retriever; DetailsLevel details_level; if (!p) return (-1); if (sortObsPoints(&retriever, p->sortColumn()) < 0) { return (-1); } if (!p->getDetailsLevel(&details_level)) details_level = details_normal; lua_newtable(vm); lua_push_uint64_table_entry(vm, "numObsPoints", retriever.actNumEntries); lua_newtable(vm); if (p->a2zSortOrder()) { for (int i = p->toSkip(), num = 0; i < (int)retriever.actNumEntries && num < (int)p->maxHits(); i++, num++) { ObservationPoint *obs_point = retriever.elems[i].obsPointValue; obs_point->lua(vm, details_level, false); lua_rawseti(vm, -2, num + 1); /* Must use integer keys to preserve and iterate inorder with ipairs */ } } else { for (int i = (retriever.actNumEntries - 1 - p->toSkip()), num = 0; i >= 0 && num < (int)p->maxHits(); i--, num++) { ObservationPoint *obs_point = retriever.elems[i].obsPointValue; obs_point->lua(vm, details_level, false); lua_rawseti(vm, -2, num + 1); } } lua_pushstring(vm, "ObsPoints"); lua_insert(vm, -2); lua_settable(vm, -3); // finally free the elements regardless of the sorted kind if (retriever.elems) free(retriever.elems); return (retriever.actNumEntries); } /* **************************************** */ int NetworkInterface::getActiveCountriesList(lua_State *vm, const Paginator *p) { struct flowHostRetriever retriever; DetailsLevel details_level; if (!p) return (-1); if (sortCountries(&retriever, p->sortColumn()) < 0) { return (-1); } if (!p->getDetailsLevel(&details_level)) details_level = details_normal; lua_newtable(vm); lua_push_uint64_table_entry(vm, "numCountries", retriever.actNumEntries); lua_newtable(vm); if (p->a2zSortOrder()) { for (int i = p->toSkip(), num = 0; i < (int)retriever.actNumEntries && num < (int)p->maxHits(); i++, num++) { Country *country = retriever.elems[i].countryVal; country->lua(vm, details_level, false); lua_rawseti(vm, -2, num + 1); /* Must use integer keys to preserve and iterate inorder with ipairs */ } } else { for (int i = (retriever.actNumEntries - 1 - p->toSkip()), num = 0; i >= 0 && num < (int)p->maxHits(); i--, num++) { Country *country = retriever.elems[i].countryVal; country->lua(vm, details_level, false); lua_rawseti(vm, -2, num + 1); } } lua_pushstring(vm, "Countries"); lua_insert(vm, -2); lua_settable(vm, -3); // finally free the elements regardless of the sorted kind if (retriever.elems) free(retriever.elems); return (retriever.actNumEntries); } /* **************************************** */ int NetworkInterface::getActiveVLANList(lua_State *vm, char *sortColumn, u_int32_t maxHits, u_int32_t toSkip, bool a2zSortOrder, DetailsLevel details_level) { struct flowHostRetriever retriever; if (!hasSeenVLANTaggedPackets()) { /* VLAN statistics are calculated only if VLAN tagged traffic has been seen */ lua_pushnil(vm); return 0; } if (sortVLANs(&retriever, sortColumn) < 0) { return (-1); } lua_newtable(vm); lua_push_uint64_table_entry(vm, "numVLANs", retriever.actNumEntries); lua_newtable(vm); if (a2zSortOrder) { for (int i = toSkip, num = 0; i < (int)retriever.actNumEntries && num < (int)maxHits; i++, num++) { VLAN *vl = retriever.elems[i].vlanValue; vl->lua(vm, details_level, false); lua_rawseti(vm, -2, num + 1); /* Must use integer keys to preserve and iterate inorder with ipairs */ } } else { for (int i = (retriever.actNumEntries - 1 - toSkip), num = 0; i >= 0 && num < (int)maxHits; i--, num++) { VLAN *vl = retriever.elems[i].vlanValue; vl->lua(vm, details_level, false); lua_rawseti(vm, -2, num + 1); } } lua_pushstring(vm, "VLANs"); lua_insert(vm, -2); lua_settable(vm, -3); // finally free the elements regardless of the sorted kind if (retriever.elems) free(retriever.elems); return (retriever.actNumEntries); } /* **************************************** */ int NetworkInterface::getActiveMacManufacturers( lua_State *vm, u_int8_t bridge_iface_idx, bool sourceMacsOnly, u_int32_t maxHits, u_int8_t devtype_filter, u_int8_t location_filter) { struct flowHostRetriever retriever; u_int32_t begin_slot = 0; bool walk_all = true; if (sortMacs(&begin_slot, walk_all, &retriever, bridge_iface_idx, sourceMacsOnly, NULL, (char *)"column_manufacturer", (u_int16_t)-1, devtype_filter, location_filter, 0) < 0) { return (-1); } lua_newtable(vm); const char *cur_manuf = NULL; u_int32_t cur_count = 0; int k = 0; for (int i = 0; i < (int)retriever.actNumEntries && k < (int)maxHits; i++) { Mac *m = retriever.elems[i].macValue; const char *manufacturer = m->get_manufacturer(); if (manufacturer != NULL) { if (!cur_manuf || (strcmp(cur_manuf, manufacturer) != 0)) { if (cur_manuf != NULL) lua_push_int32_table_entry(vm, cur_manuf, cur_count); cur_manuf = manufacturer; cur_count = 1; k++; } else { cur_count++; } } } if (cur_manuf != NULL) lua_push_int32_table_entry(vm, cur_manuf, cur_count); // finally free the elements regardless of the sorted kind if (retriever.elems) free(retriever.elems); return (retriever.actNumEntries); } /* **************************************** */ static bool find_mac_hosts(GenericHashEntry *h, void *user_data, bool *matched) { struct mac_find_info *info = (struct mac_find_info *)user_data; Host *host = (Host *)h; if (host->getMac() == info->m) host->lua(info->vm, NULL /* Already checked */, false, false, false, true); return false; /* false = keep on walking */ } /* **************************************** */ bool NetworkInterface::getActiveMacHosts(lua_State *vm, const char *mac) { struct mac_find_info info; bool res = false; u_int32_t begin_slot = 0; if (!macs_hash) return res; memset(&info, 0, sizeof(info)); Utils::parseMac(info.mac, mac); info.vm = vm; info.m = macs_hash->get(info.mac, false /* Not an inline call */); if(!info.m || !info.m->getNumHosts() /* Avoid walking the hosts hash table when there are no hosts associated */) return res; walker(&begin_slot, true /* walk_all */, walker_hosts, find_mac_hosts, &info); return res; } /* **************************************** */ int NetworkInterface::getActiveDeviceTypes( lua_State *vm, u_int8_t bridge_iface_idx, bool sourceMacsOnly, u_int32_t maxHits, const char *manufacturer, u_int8_t location_filter) { struct flowHostRetriever retriever; u_int32_t begin_slot = 0; bool walk_all = true; if (sortMacs(&begin_slot, walk_all, &retriever, bridge_iface_idx, sourceMacsOnly, manufacturer, (char *)"column_device_type", (u_int16_t)-1, (u_int8_t)-1, location_filter, 0) < 0) { return (-1); } lua_newtable(vm); u_int8_t cur_devtype = 0; u_int32_t cur_count = 0; int k = 0; for (int i = 0; i < (int)retriever.actNumEntries && k < (int)maxHits; i++) { Mac *m = retriever.elems[i].macValue; if (m->getDeviceType() != cur_devtype) { if (cur_count) { lua_pushinteger(vm, cur_devtype); lua_pushinteger(vm, cur_count); lua_settable(vm, -3); } cur_devtype = m->getDeviceType(); cur_count = 1; k++; } else { cur_count++; } } if (cur_count) { lua_pushinteger(vm, cur_devtype); lua_pushinteger(vm, cur_count); lua_settable(vm, -3); } // finally free the elements regardless of the sorted kind if (retriever.elems) free(retriever.elems); return (retriever.actNumEntries); } /* **************************************** */ bool NetworkInterface::isMacActive(char *mac) { struct mac_find_info info; if (!macs_hash) return false; memset(&info, 0, sizeof(info)); Utils::parseMac(info.mac, mac); return !!macs_hash->get(info.mac, false /* Not an inline call */); } /* **************************************** */ bool NetworkInterface::getMacInfo(lua_State *vm, char *mac) { struct mac_find_info info; bool ret = false; if (!macs_hash) return ret; memset(&info, 0, sizeof(info)); Utils::parseMac(info.mac, mac); info.m = macs_hash->get(info.mac, false /* Not an inline call */); if (info.m) { info.m->lua(vm, true, false); ret = true; } return ret; } /* **************************************** */ bool NetworkInterface::resetMacStats(lua_State *vm, char *mac, bool delete_data) { struct mac_find_info info; bool ret = false; if (!macs_hash) return ret; memset(&info, 0, sizeof(info)); Utils::parseMac(info.mac, mac); info.m = macs_hash->get(info.mac, false /* Not an inline call */); if (info.m) { if (delete_data) info.m->requestDataReset(); else info.m->requestStatsReset(); ret = true; } return ret; } /* **************************************** */ bool NetworkInterface::setMacDeviceType(char *strmac, DeviceType dtype, bool alwaysOverwrite) { u_int8_t mac[6]; Mac *m; DeviceType oldtype; Utils::parseMac(mac, strmac); ntop->getTrace()->traceEvent(TRACE_INFO, "setMacDeviceType(%s) = %d", strmac, (int)dtype); if ((m = getMac(mac, false /* Don't create if missing */, false /* Not an inline call */))) { oldtype = m->getDeviceType(); if (alwaysOverwrite || (oldtype == device_unknown)) { m->forceDeviceType(dtype); if (alwaysOverwrite && (oldtype != device_unknown) && (oldtype != dtype)) ntop->getTrace()->traceEvent(TRACE_INFO, "Device %s type changed from %d to %d\n", strmac, oldtype, dtype); } return (true); } else return (false); } /* **************************************** */ bool NetworkInterface::getASInfo(lua_State *vm, u_int32_t asn) { struct as_find_info info; bool ret; u_int32_t begin_slot = 0; bool walk_all = true; memset(&info, 0, sizeof(info)); info.asn = asn; walker(&begin_slot, walk_all, walker_ases, find_as_by_asn, (void *)&info); if (info.as) { info.as->lua(vm, details_higher, false); ret = true; } else ret = false; return ret; } /* **************************************** */ bool NetworkInterface::getObsPointInfo(lua_State *vm, u_int16_t obs_point) { struct obs_point_find_info info; bool ret; u_int32_t begin_slot = 0; bool walk_all = true; memset(&info, 0, sizeof(info)); info.obs_point_id = obs_point; walker(&begin_slot, walk_all, walker_obs, find_obs_point_by_id, (void *)&info); if (info.obs_point) { info.obs_point->lua(vm, details_higher, false); ret = true; } else ret = false; return ret; } /* **************************************** */ bool NetworkInterface::getCountryInfo(lua_State *vm, const char *country) { struct country_find_info info; bool ret; u_int32_t begin_slot = 0; bool walk_all = true; memset(&info, 0, sizeof(info)); info.country_id = country; walker(&begin_slot, walk_all, walker_countries, find_country, (void *)&info); if (info.country) { info.country->lua(vm, details_higher, false); ret = true; } else ret = false; return ret; } /* **************************************** */ bool NetworkInterface::getVLANInfo(lua_State *vm, u_int16_t vlan_id) { struct vlan_find_info info; bool ret; u_int32_t begin_slot = 0; bool walk_all = true; memset(&info, 0, sizeof(info)); info.vlan_id = vlan_id; walker(&begin_slot, walk_all, walker_vlans, find_vlan_by_vlan_id, (void *)&info); if (info.vl) { info.vl->lua(vm, details_higher, false); ret = true; } else ret = false; return ret; } /* **************************************** */ int NetworkInterface::updateHostTrafficPolicy(AddressTree *allowed_networks, char *host_ip, u_int16_t host_vlan) { Host *h; int rv; if ((h = findHostByIP(allowed_networks, host_ip, host_vlan, 0 /* any observation point */)) != NULL) { h->updateHostTrafficPolicy(host_ip); rv = CONST_LUA_OK; } else rv = CONST_LUA_ERROR; return rv; } /* *************************************** */ TimeseriesExporter *NetworkInterface::getInfluxDBTSExporter() { if (!influxdb_ts_exporter) influxdb_ts_exporter = new (nothrow) InfluxDBTimeseriesExporter(this); return (influxdb_ts_exporter); } /* *************************************** */ TimeseriesExporter *NetworkInterface::getRRDTSExporter() { if (!rrd_ts_exporter) rrd_ts_exporter = new (nothrow) RRDTimeseriesExporter(this); return (rrd_ts_exporter); } /* *************************************** */ void NetworkInterface::checkMacIPAssociation(bool triggerEvent, u_char *_mac, u_int32_t ipv4, Mac *host_mac) { if (!are_ip_reassignment_alerts_enabled()) return; u_int64_t mac = Utils::mac2int(_mac); if ((ipv4 != 0) && (mac != 0) && (mac != 0xFFFFFFFFFFFF)) { std::map::iterator it; if (!triggerEvent) ip_mac[ipv4] = mac; else { if ((it = ip_mac.find(ipv4)) != ip_mac.end()) { /* Found entry */ if (it->second != mac) { u_char tmp[16]; Utils::int2mac(it->second, tmp); AlertsQueue *q = getAlertsQueue(); if (q) q->pushMacIpAssociationChangedAlert(ntohl(ipv4), tmp, _mac, host_mac); ip_mac[ipv4] = mac; } } else ip_mac[ipv4] = mac; } } } /* *************************************** */ void NetworkInterface::checkDhcpIPRange(Mac *sender_mac, struct dhcp_packet *dhcp_reply, u_int16_t vlan_id) { if (!hasConfiguredDhcpRanges()) return; u_char *_mac = dhcp_reply->chaddr; u_int64_t mac = Utils::mac2int(_mac); u_int32_t ipv4 = dhcp_reply->yiaddr; if ((ipv4 != 0) && (mac != 0) && (mac != 0xFFFFFFFFFFFF)) { IpAddress ip; ip.set(ipv4); if (!isInDhcpRange(&ip)) { AlertsQueue *q = getAlertsQueue(); if (q) q->pushOutsideDhcpRangeAlert(_mac, sender_mac, ntohl(ipv4), ntohl(dhcp_reply->siaddr), vlan_id); } } } /* *************************************** */ /* Updates interface broadcast domains, inferring them from the range between `src` and `dst` IP addresses. This function must be called when `src` and `dst` are (reasonably) assumed to be in the same broadcast domain. For example, this function must be called when: - `src` and `dst` are read inside ARP sender protocol address (spa) and target protocol address (tpa) - `src` and `dst` are read from an IP packet with broadcast destination MAC (FF:FF:FF:FF:FF:FF) */ void NetworkInterface::updateBroadcastDomains(u_int16_t vlan_id, const u_int8_t *src_mac, const u_int8_t *dst_mac, u_int32_t src, u_int32_t dst) { u_int32_t net = src & dst; u_int32_t diff; IpAddress cur_bcast_domain; IpAddress search_ip; Host *h; search_ip.set(htonl(src)); h = getHostByIP(&search_ip, vlan_id, 0 /* observation_point_id */, true /* isInlineCall */); if(h != NULL) h->setMACmeaningful(); search_ip.set(htonl(dst)); h = getHostByIP(&search_ip, vlan_id, 0 /* observation_point_id */, true /* isInlineCall */); if(h != NULL) h->setMACmeaningful(); /* Smaller address in src */ if (src > dst) { u_int32_t r = src; src = dst; dst = r; } /* Range between dst and src */ diff = dst - src; /* Following is an heuristic which tries to detect the broadcast domain with its size and network-part of the address. Detection is done by checking source and target protocol addresses found in arp. Link-local addresses are excluded, as well as arp Probes with a zero source IP. ARP Probes are defined in RFC 5227: In this document, the term 'ARP Probe' is used to refer to an ARP Request packet, broadcast on the local link, with an all-zero 'sender IP address'. [...] The 'target IP address' field MUST be set to the address being probed. An ARP Probe conveys both a question ("Is anyone using this address?") and an implied statement ("This is the address I hope to use."). */ if (diff && src /* Not a zero source IP (ARP Probe) */ && (src & 0xFFFF0000) != 0xA9FE0000 /* Not a link-local IP */ && (dst & 0xFFFF0000) != 0xA9FE0000 /* Not a link-local IP */) { u_int32_t cur_mask; u_int8_t cur_cidr; for (cur_mask = 0xFFFFFFF0, cur_cidr = 28; cur_mask > 0x00000000; cur_mask <<= 1, cur_cidr--) { if ((diff & cur_mask) == 0) { /* diff < cur_mask */ net &= cur_mask; if ((src & cur_mask) != (dst & cur_mask)) { cur_mask <<= 1, cur_cidr -= 1; net = src & cur_mask; } cur_bcast_domain.set(htonl(net)); if (cur_mask > 0xFFFF0000 /* /16 */) { /* NOTE: call this also for existing domains in order to update the * hits */ bcast_domains->addAddress(&cur_bcast_domain, cur_cidr); #ifdef BROADCAST_DOMAINS_DEBUG char buf1[32], buf2[32], buf3[32]; ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s <-> %s [%s - %u]", Utils::intoaV4(src, buf1, sizeof(buf1)), Utils::intoaV4(dst, buf2, sizeof(buf2)), Utils::intoaV4(net, buf3, sizeof(buf3)), cur_cidr); #endif } else { if (ntop->getPrefs()->isBroadcastDomainTooLargeEnabled()) { AlertsQueue *q = getAlertsQueue(); if (q) q->pushBroadcastDomainTooLargeAlert(src_mac, dst_mac, src, dst, vlan_id); } } break; } } } } /* *************************************** */ /* Start the thread for the execution of flow user script hooks */ bool NetworkInterface::initFlowChecksLoop() { pthread_create(&flowChecksLoop, NULL, ::flowChecksLoop, (void *)this); flowAlertsDequeueLoopCreated = true; return true; } /* *************************************** */ /* Start the thread for the execution of host user script hooks */ bool NetworkInterface::initHostChecksLoop() { pthread_create(&hostChecksLoop, NULL, ::hostChecksLoop, (void *)this); hostAlertsDequeueLoopCreated = true; return true; } /* *************************************** */ /* Put here all the code that is executed when the NIC initialization is successful */ bool NetworkInterface::initFlowDump(u_int8_t num_dump_interfaces) { startFlowDumping(); /* Flows are dumped by the view only */ if (isViewed()) /* No need to allocate databases on view interfaces */ return (true); if (db == NULL) { try { if (ntop->getPrefs()->do_dump_flows_on_clickhouse()) { #if defined(NTOPNG_PRO) && defined(HAVE_CLICKHOUSE) && defined(HAVE_MYSQL) db = new (std::nothrow) ClickHouseFlowDB(this); if ((db == NULL) || (db->isDbCreated() == false)) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Leaving due to failed ClickHouse initialization"); exit(-1); } #endif } #ifdef HAVE_MYSQL else if (ntop->getPrefs()->do_dump_flows_on_mysql()) { db = new (std::nothrow) MySQLDB(this); if (!db) throw "Not enough memory"; } #endif #ifndef HAVE_NEDGE else if (ntop->getPrefs()->do_dump_flows_on_es()) { db = new (std::nothrow) ElasticSearch(this); } #if defined(HAVE_KAFKA) && defined(NTOPNG_PRO) else if ((ntop->getPrefs()->getKakfaBrokersList() != NULL)) { kafka = new (std::nothrow) KafkaProducer(this, ntop->getPrefs()->getKakfaBrokersList(), ntop->getPrefs()->getKafkaTopic(), ntop->getPrefs()->getKafkaOptions()); db = kafka; } #endif #if !defined(WIN32) && !defined(__APPLE__) else if (ntop->getPrefs()->do_dump_flows_on_syslog()) { db = new (std::nothrow) SyslogDump(this); } #endif #endif } catch (...) { ; } } return (db != NULL); } /* *************************************** */ bool NetworkInterface::registerLiveCapture(NtopngLuaContext *const luactx, int *id) { bool ret = false; *id = -1; active_captures_lock.lock(__FILE__, __LINE__); if (num_live_captures < MAX_NUM_PCAP_CAPTURES) { for (int i = 0; i < MAX_NUM_PCAP_CAPTURES; i++) { if (live_captures[i] == NULL) { live_captures[i] = luactx, num_live_captures++; ret = true, *id = i; break; } } } active_captures_lock.unlock(__FILE__, __LINE__); return (ret); } /* *************************************** */ bool NetworkInterface::deregisterLiveCapture( NtopngLuaContext *const luactx) { bool ret = false; active_captures_lock.lock(__FILE__, __LINE__); for (int i = 0; i < MAX_NUM_PCAP_CAPTURES; i++) { if (live_captures[i] == luactx) { NtopngLuaContext *c = (NtopngLuaContext *)live_captures[i]; c->live_capture.stopped = true; live_captures[i] = NULL, num_live_captures--; ret = true; break; } } active_captures_lock.unlock(__FILE__, __LINE__); return (ret); } /* *************************************** */ bool NetworkInterface::matchLiveCapture(NtopngLuaContext *const luactx, const struct pcap_pkthdr *const h, const u_char *const packet, Flow *const f) { if (!luactx->live_capture.matching_host /* No host filter set */ || (f && (luactx->live_capture.matching_host == f->get_cli_host() || luactx->live_capture.matching_host == f->get_srv_host()))) { if (luactx->live_capture.bpfFilterSet) { if (!bpf_filter(luactx->live_capture.fcode.bf_insns, (const u_char *)packet, h->caplen, h->caplen)) { return (false); } } return (true); } return false; } /* *************************************** */ void NetworkInterface::deliverLiveCapture(const struct pcap_pkthdr *const h, const u_char *const packet, Flow *const f) { int res; for (u_int i = 0, num_found = 0; (i < MAX_NUM_PCAP_CAPTURES) && (num_found < num_live_captures); i++) { if (live_captures[i] != NULL) { NtopngLuaContext *c = (NtopngLuaContext *)live_captures[i]; bool http_client_disconnected = false; int disconnect_stage = 0; num_found++; if ((c->live_capture.capture_until < (u_int32_t)h->ts.tv_sec) || c->live_capture.stopped) http_client_disconnected = true, disconnect_stage = 1; #ifdef TRACE_DOWNLOAD ntop->getTrace()->traceEvent( TRACE_NORMAL, "%d seconds left", c->live_capture.capture_until - h->ts.tv_sec); #endif /* The header is always sent even when there is never a match with matchLiveCapture, as otherwise some browsers may end up in hanging. Hanging has been verified with Safari Version 12.0 (13606.2.11) but not with Chrome Version 68.0.3440.106 (Official Build) (64-bit) */ if (!http_client_disconnected && c->conn && !c->live_capture.pcaphdr_sent) { struct pcap_file_header pcaphdr; Utils::init_pcap_header(&pcaphdr, get_datalink(), ntop->getGlobals()->getSnaplen(get_name())); if ((res = mg_write_async(c->conn, &pcaphdr, sizeof(pcaphdr))) < (int)sizeof(pcaphdr)) http_client_disconnected = true, disconnect_stage = 2; c->live_capture.pcaphdr_sent = true; } if (!http_client_disconnected && c->conn && matchLiveCapture(c, h, packet, f)) { struct pcap_disk_pkthdr *pkthdr; /* Cannot use h as the format on disk differs */ if (c->live_capture.data_not_yet_sent_len > 0) { /* We have some leftover from the previous send, so let's try to send this data first */ #ifdef TRACE_DOWNLOAD ntop->getTrace()->traceEvent(TRACE_NORMAL, "Resending data [%u bytes left]", c->live_capture.data_not_yet_sent_len); #endif res = mg_write_async(c->conn, c->live_capture.send_buffer, c->live_capture.data_not_yet_sent_len); if (res > 0) { if (res == (int)c->live_capture.data_not_yet_sent_len) { c->live_capture.data_not_yet_sent_len = 0; /* We've sent everything that was in queue */ } else { u_int leftover = c->live_capture.data_not_yet_sent_len - res; memmove(c->live_capture.send_buffer, &c->live_capture.send_buffer[res], leftover); c->live_capture.data_not_yet_sent_len = leftover; #ifdef TRACE_DOWNLOAD ntop->getTrace()->traceEvent( TRACE_NORMAL, "Partial send [%u bytes left]", c->live_capture.data_not_yet_sent_len); #endif return; /* The current packet is dropped */ } } } /* If we're here all the previous data has been sent out */ pkthdr = (struct pcap_disk_pkthdr *)c->live_capture.send_buffer; pkthdr->ts.tv_sec = h->ts.tv_sec, pkthdr->ts.tv_usec = h->ts.tv_usec, pkthdr->caplen = ndpi_min(h->caplen, sizeof(c->live_capture.send_buffer) - sizeof(struct pcap_disk_pkthdr)), pkthdr->len = h->len; memcpy(&c->live_capture.send_buffer[sizeof(struct pcap_disk_pkthdr)], packet, pkthdr->caplen); c->live_capture.data_not_yet_sent_len = pkthdr->caplen + sizeof(struct pcap_disk_pkthdr); /* Now send data */ #ifdef TRACE_DOWNLOAD ntop->getTrace()->traceEvent( TRACE_NORMAL, "About to send %u bytes [%d sec left]", c->live_capture.data_not_yet_sent_len, c->live_capture.capture_until - h->ts.tv_sec); #endif res = mg_write_async(c->conn, &c->live_capture.send_buffer, c->live_capture.data_not_yet_sent_len); #ifdef TRACE_DOWNLOAD ntop->getTrace()->traceEvent(TRACE_NORMAL, "Sent %d / %u bytes", res, c->live_capture.data_not_yet_sent_len); #endif if (res == (int)c->live_capture.data_not_yet_sent_len) { c->live_capture.data_not_yet_sent_len = 0; /* All sent */ c->live_capture.num_captured_packets++; if ((c->live_capture.capture_max_pkts != 0) && (c->live_capture.num_captured_packets == c->live_capture.capture_max_pkts)) { http_client_disconnected = true, disconnect_stage = 4; } } else if (res >= 0) { #ifdef TRACE_DOWNLOAD ntop->getTrace()->traceEvent(TRACE_NORMAL, "Partial send: %u bytes sent", res); #endif if (res > 0) { /* Some data has been sent */ u_int leftover = c->live_capture.data_not_yet_sent_len - res; memmove(c->live_capture.send_buffer, &c->live_capture.send_buffer[res], leftover); c->live_capture.data_not_yet_sent_len = leftover; #ifdef TRACE_DOWNLOAD ntop->getTrace()->traceEvent(TRACE_NORMAL, "Partial send [%u bytes left]", c->live_capture.data_not_yet_sent_len); #endif } return; /* The current packet is dropped */ } else { /* An error occurred */ http_client_disconnected = true, disconnect_stage = 3; } } if (http_client_disconnected) { ntop->getTrace()->traceEvent(TRACE_INFO, "Client disconnected or socket for live " "capture is busy, stopping capture (%d)", disconnect_stage); deregisterLiveCapture(c); /* (*) */ } } } } /* *************************************** */ void NetworkInterface::dumpLiveCaptures(lua_State *vm) { /* Administrative privileges checked by the caller */ active_captures_lock.lock(__FILE__, __LINE__); lua_newtable(vm); for (int i = 0, capture_id = 0; i < MAX_NUM_PCAP_CAPTURES; i++) { if (live_captures[i] != NULL && !live_captures[i]->live_capture.stopped) { lua_newtable(vm); lua_push_uint64_table_entry(vm, "id", i); lua_push_uint64_table_entry(vm, "capture_until", live_captures[i]->live_capture.capture_until); lua_push_uint64_table_entry( vm, "capture_max_pkts", live_captures[i]->live_capture.capture_max_pkts); lua_push_uint64_table_entry( vm, "num_captured_packets", live_captures[i]->live_capture.num_captured_packets); if (live_captures[i]->live_capture.matching_host != NULL) { Host *h = (Host *)live_captures[i]->live_capture.matching_host; char buf[64]; lua_push_str_table_entry(vm, "host", h->get_ip()->print(buf, sizeof(buf))); } lua_pushinteger(vm, ++capture_id); lua_insert(vm, -2); lua_settable(vm, -3); } } active_captures_lock.unlock(__FILE__, __LINE__); } /* *************************************** */ bool NetworkInterface::stopLiveCapture(int capture_id) { bool rc = false; /* Administrative privileges checked by the caller */ if ((capture_id >= 0) && (capture_id < MAX_NUM_PCAP_CAPTURES)) { active_captures_lock.lock(__FILE__, __LINE__); if (live_captures[capture_id] != NULL) { NtopngLuaContext *c = (NtopngLuaContext *)live_captures[capture_id]; c->live_capture.stopped = true, rc = true; if (c->live_capture.bpfFilterSet) pcap_freecode(&c->live_capture.fcode); /* live_captures[capture_id] = NULL; */ /* <-- not necessary as mongoose will clean it */ } active_captures_lock.unlock(__FILE__, __LINE__); } return (rc); } /* *************************************** */ static bool host_reload_blacklist(GenericHashEntry *host, void *user_data, bool *matched) { Host *h = (Host *)host; h->reloadHostBlacklist(); *matched = true; return (false); /* false = keep on walking */ } /* *************************************** */ void NetworkInterface::reloadHostsBlacklist() { u_int32_t begin_slot = 0; bool walk_all = true; /* Update the hosts */ walker(&begin_slot, walk_all, walker_hosts, host_reload_blacklist, NULL); } /* *************************************** */ static bool host_reload_dhcp_host(GenericHashEntry *host, void *user_data, bool *matched) { Host *h = (Host *)host; h->reloadDhcpHost(); *matched = true; return (false); /* false = keep on walking */ } /* *************************************** */ void NetworkInterface::reloadDhcpRanges() { char redis_key[CONST_MAX_LEN_REDIS_KEY], *rsp = NULL; dhcp_range *new_ranges = NULL; u_int num_ranges = 0; u_int len; if (!ntop->getRedis()) return; snprintf(redis_key, sizeof(redis_key), IFACE_DHCP_RANGE_KEY, get_id()); if ((rsp = (char *)malloc(CONST_MAX_LEN_REDIS_VALUE)) && !ntop->getRedis()->get(redis_key, rsp, CONST_MAX_LEN_REDIS_VALUE) && (len = strlen(rsp))) { u_int i; num_ranges = 1; for (i = 0; i < len; i++) { if (rsp[i] == ',') num_ranges++; } // +1 for final zero IP, which is used to indicate array termination new_ranges = new (std::nothrow) dhcp_range[num_ranges + 1]; if (new_ranges) { char *cur_pos = rsp; /* E.g. 192.168.1.2-192.168.1.150,10.0.0.50-10.0.0.60 */ for (i = 0; i < num_ranges; i++) { char *end = strchr(cur_pos, ','); char *delim = strchr(cur_pos, '-'); if (!end) end = cur_pos + strlen(cur_pos); if (delim) { *delim = 0; *end = 0; new_ranges[i].first_ip.set(cur_pos); new_ranges[i].last_ip.set(delim + 1); } cur_pos = end + 1; } } } if (dhcp_ranges_shadow) delete[] (dhcp_ranges_shadow); dhcp_ranges_shadow = dhcp_ranges; dhcp_ranges = new_ranges; if (rsp) free(rsp); /* Reload existing hosts */ u_int32_t begin_slot = 0; bool walk_all = true; walker(&begin_slot, walk_all, walker_hosts, host_reload_dhcp_host, NULL); } /* *************************************** */ bool NetworkInterface::isInDhcpRange(IpAddress *ip) { // Important: cache it as it may change dhcp_range *ranges = dhcp_ranges; if (!ranges) return (false); while (!ranges->last_ip.isEmpty()) { if ((ranges->first_ip.compare(ip) <= 0) && (ranges->last_ip.compare(ip) >= 0)) return true; ranges++; } return false; } /* *************************************** */ bool NetworkInterface::isLocalBroadcastDomainHost(Host *const h, bool is_inline_call) { bool res = bcast_domains->isLocalBroadcastDomainHost(h, is_inline_call); return res; } /* *************************************** */ typedef std::map PodsMap; static bool flow_get_pods_stats(GenericHashEntry *entry, void *user_data, bool *matched) { PodsMap *pods_stats = (PodsMap *)user_data; Flow *flow = (Flow *)entry; const ContainerInfo *cli_cont, *srv_cont; const char *cli_pod = NULL, *srv_pod = NULL; if ((cli_cont = flow->getClientContainerInfo()) && cli_cont->data_type == container_info_data_type_k8s) cli_pod = cli_cont->data.k8s.pod; if ((srv_cont = flow->getServerContainerInfo()) && srv_cont->data_type == container_info_data_type_k8s) srv_pod = srv_cont->data.k8s.pod; if (cli_pod) { ContainerStats stats = (*pods_stats)[cli_pod]; /* get existing or create new */ const TcpInfo *client_tcp = flow->getClientTcpInfo(); stats.incNumFlowsAsClient(); stats.accountLatency(client_tcp ? client_tcp->rtt : 0, client_tcp ? client_tcp->rtt_var : 0, true /* as_client */); if (cli_cont->id) stats.addContainer(cli_cont->id); /* Update */ (*pods_stats)[cli_pod] = stats; } if (srv_pod) { ContainerStats stats = (*pods_stats)[srv_pod]; /* get existing or create new */ const TcpInfo *server_tcp = flow->getServerTcpInfo(); stats.incNumFlowsAsServer(); stats.accountLatency(server_tcp ? server_tcp->rtt : 0, server_tcp ? server_tcp->rtt_var : 0, false /* as server */); if (srv_cont->id) stats.addContainer(srv_cont->id); /* Update */ (*pods_stats)[srv_pod] = stats; } return (false /* keep walking */); } /* *************************************** */ void NetworkInterface::getPodsStats(lua_State *vm) { PodsMap pods_stats; u_int32_t begin_slot = 0; bool walk_all = true; PodsMap::iterator it; walker(&begin_slot, walk_all, walker_flows, flow_get_pods_stats, (void *)&pods_stats); lua_newtable(vm); for (it = pods_stats.begin(); it != pods_stats.end(); ++it) { it->second.lua(vm); lua_pushstring(vm, it->first.c_str()); lua_insert(vm, -2); lua_settable(vm, -3); } } /* *************************************** */ typedef struct { const ContainerInfo *info; ContainerStats stats; } ContainerData; typedef std::map ContainersMap; struct containerRetrieverData { ContainersMap containers; const char *pod_filter; }; static bool flow_get_container_stats(GenericHashEntry *entry, void *user_data, bool *matched) { ContainersMap *containers_data = &((containerRetrieverData *)user_data)->containers; const char *pod_filter = ((containerRetrieverData *)user_data)->pod_filter; Flow *flow = (Flow *)entry; const ContainerInfo *cli_cont, *srv_cont; const char *cli_cont_id = NULL, *srv_cont_id = NULL; const char *cli_pod = NULL, *srv_pod = NULL; if ((cli_cont = flow->getClientContainerInfo())) { cli_cont_id = cli_cont->id; if (cli_cont->data_type == container_info_data_type_k8s) cli_pod = cli_cont->data.k8s.pod; } if ((srv_cont = flow->getServerContainerInfo())) { srv_cont_id = srv_cont->id; if (srv_cont->data_type == container_info_data_type_k8s) srv_pod = srv_cont->data.k8s.pod; } if (cli_cont_id && ((!pod_filter) || (cli_pod && !strcmp(pod_filter, cli_pod)))) { ContainerData data = (*containers_data)[cli_cont_id]; /* get existing or create new */ const TcpInfo *client_tcp = flow->getClientTcpInfo(); data.stats.incNumFlowsAsClient(); data.stats.accountLatency(client_tcp ? client_tcp->rtt : 0, client_tcp ? client_tcp->rtt_var : 0, true /* as_client */); data.info = cli_cont; /* Update */ (*containers_data)[cli_cont_id] = data; } if (srv_cont_id && ((!pod_filter) || (srv_pod && !strcmp(pod_filter, srv_pod)))) { ContainerData data = (*containers_data)[srv_cont_id]; /* get existing or create new */ const TcpInfo *server_tcp = flow->getServerTcpInfo(); data.stats.incNumFlowsAsServer(); data.stats.accountLatency(server_tcp ? server_tcp->rtt : 0, server_tcp ? server_tcp->rtt_var : 0, false /* as server */); data.info = srv_cont; /* Update */ (*containers_data)[srv_cont_id] = data; } return (false /* keep walking */); } /* *************************************** */ void NetworkInterface::getContainersStats(lua_State *vm, const char *pod_filter) { containerRetrieverData user_data; u_int32_t begin_slot = 0; bool walk_all = true; ContainersMap::iterator it; user_data.pod_filter = pod_filter; walker(&begin_slot, walk_all, walker_flows, flow_get_container_stats, (void *)&user_data); lua_newtable(vm); for (it = user_data.containers.begin(); it != user_data.containers.end(); ++it) { it->second.stats.lua(vm); if (it->second.info) { Utils::containerInfoLua(vm, it->second.info); lua_pushstring(vm, "info"); lua_insert(vm, -2); lua_settable(vm, -3); } lua_pushstring(vm, it->first.c_str()); lua_insert(vm, -2); lua_settable(vm, -3); } } /* *************************************** */ bool NetworkInterface::enqueueFlowToCompanion(ParsedFlow *const pf, bool skip_loopback_traffic) { if (skip_loopback_traffic && (pf->src_ip.isLoopbackAddress() || pf->dst_ip.isLoopbackAddress())) return false; if (companionQueue[next_compq_insert_idx]) return false; if ((companionQueue[next_compq_insert_idx] = new (std::nothrow) ParsedFlow(*pf))) { next_compq_insert_idx = (next_compq_insert_idx + 1) % COMPANION_QUEUE_LEN; return true; } return false; } /* *************************************** */ void NetworkInterface::incNumAlertedFlows(Flow *f, AlertLevel severity) { switch (Utils::mapAlertLevelToGroup(severity)) { case alert_level_group_notice_or_lower: num_active_alerted_flows_notice++; break; case alert_level_group_warning: num_active_alerted_flows_warning++; break; case alert_level_group_error: num_active_alerted_flows_error++; break; default: break; } } /* *************************************** */ void NetworkInterface::decNumAlertedFlows(Flow *f, AlertLevel severity) { switch (Utils::mapAlertLevelToGroup(severity)) { case alert_level_group_notice_or_lower: num_active_alerted_flows_notice--; break; case alert_level_group_warning: num_active_alerted_flows_warning--; break; case alert_level_group_error: num_active_alerted_flows_error--; break; default: break; } }; /* *************************************** */ u_int64_t NetworkInterface::getNumActiveAlertedFlows( AlertLevelGroup alert_level_group) const { switch (alert_level_group) { case alert_level_group_notice_or_lower: return num_active_alerted_flows_notice; case alert_level_group_warning: return num_active_alerted_flows_warning; case alert_level_group_error: return num_active_alerted_flows_error; default: return 0; } }; /* *************************************** */ u_int64_t NetworkInterface::getNumActiveAlertedFlows() const { return num_active_alerted_flows_notice + num_active_alerted_flows_warning + num_active_alerted_flows_error; }; /* *************************************** */ void NetworkInterface::incNumActiveProbes() { num_active_probes++; } /* *************************************** */ void NetworkInterface::decNumActiveProbes() { num_active_probes--; } /* *************************************** */ u_int64_t NetworkInterface::getNumActiveProbes() const { return num_active_probes; } /* *************************************** */ bool NetworkInterface::dequeueFlowFromCompanion(ParsedFlow **f) { if (!companionQueue[next_compq_remove_idx]) { *f = NULL; return false; } *f = companionQueue[next_compq_remove_idx]; companionQueue[next_compq_remove_idx] = NULL; next_compq_remove_idx = (next_compq_remove_idx + 1) % COMPANION_QUEUE_LEN; return true; } /* *************************************** */ struct alertable_walker_data { AddressTree *allowed_nets; alertable_callback *callback; void *user_data; }; static bool host_invoke_alertable_callback(GenericHashEntry *entity, void *user_data, bool *matched) { AlertableEntity *alertable = dynamic_cast(entity); struct alertable_walker_data *data = (struct alertable_walker_data *)user_data; if (alertable->matchesAllowedNetworks(data->allowed_nets)) { data->callback(alert_entity_host, alertable, data->user_data); *matched = true; } return (false); /* false = keep on walking */ } /* *************************************** */ /* Walks alertable entities on this interface. * The user provided callback is called with the alertable_walker_data.user_data * parameter set to the provided user_data. */ void NetworkInterface::walkAlertables(AlertEntity alert_entity, const char *entity_value, AddressTree *allowed_nets, alertable_callback *callback, void *user_data) { std::map, InterfaceMemberAlertableEntity *>::iterator it; /* Hosts */ if (((alert_entity == alert_entity_none) || (alert_entity == alert_entity_host))) { if (entity_value == NULL) { struct alertable_walker_data data; bool walk_all = true; u_int32_t begin_slot = 0; data.callback = callback; data.user_data = user_data; data.allowed_nets = allowed_nets; walker(&begin_slot, walk_all, walker_hosts, host_invoke_alertable_callback, &data); } else { /* Specific host */ char *host_ip = NULL; u_int16_t vlan_id = 0; char buf[64]; Host *host; get_host_vlan_info((char *)entity_value, &host_ip, &vlan_id, buf, sizeof(buf)); if (host_ip && (host = getHost(host_ip, vlan_id, 0 /* not sure it can be read by the VM */, false /* not inline */)) && host->matchesAllowedNetworks(allowed_nets)) callback(alert_entity_host, host, user_data); } } /* Interface */ if (((alert_entity == alert_entity_none) || (alert_entity == alert_entity_interface))) { if ((entity_value != NULL) && (getEntityValue().compare(entity_value) != 0)) ntop->getTrace()->traceEvent(TRACE_WARNING, "Interface filter does not " "correspond[type=%u]: expected %s, found %s", alert_entity, entity_value, getEntityValue().c_str()); if (matchesAllowedNetworks(allowed_nets)) callback(alert_entity_interface, this, user_data); } /* Networks */ if (((alert_entity == alert_entity_none) || (alert_entity == alert_entity_network))) { u_int32_t num_local_networks = ntop->getNumLocalNetworks(); for (u_int32_t network_id = 0; network_id < num_local_networks; network_id++) { NetworkStats *netstats = getNetworkStats(network_id); if ((entity_value == NULL) || (netstats->getEntityValue().compare(entity_value) == 0)) { if (netstats->matchesAllowedNetworks(allowed_nets)) callback(alert_entity_network, netstats, user_data); } } } /* External Alerts. * Must lock in order to avoid concurrency issues with insertions/updates */ external_alerts_lock.lock(__FILE__, __LINE__); for (it = external_alerts.begin(); it != external_alerts.end(); ++it) { if (((alert_entity == alert_entity_none) || (alert_entity == it->second->getEntityType()))) { if ((entity_value == NULL) || (it->second->getEntityValue().compare(entity_value) == 0)) { if (it->second->matchesAllowedNetworks(allowed_nets)) callback(alert_entity_other, it->second, user_data); } } } external_alerts_lock.unlock(__FILE__, __LINE__); } /* *************************************** */ void NetworkInterface::incNumAlertsEngaged(AlertEntity alert_entity, AlertLevel alert_severity) { u_int i = alert_entity; if (i < ALERT_ENTITY_MAX_NUM_ENTITIES) { switch (Utils::mapAlertLevelToGroup(alert_severity)) { case alert_level_group_notice_or_lower: num_alerts_engaged_notice[i]++; break; case alert_level_group_warning: num_alerts_engaged_warning[i]++; break; case alert_level_group_error: num_alerts_engaged_error[i]++; break; case alert_level_group_critical: num_alerts_engaged_critical[i]++; break; case alert_level_group_emergency: num_alerts_engaged_emergency[i]++; break; default: break; } } } /* *************************************** */ void NetworkInterface::decNumAlertsEngaged(AlertEntity alert_entity, AlertLevel alert_severity) { u_int i = alert_entity; if (i < ALERT_ENTITY_MAX_NUM_ENTITIES) { switch (Utils::mapAlertLevelToGroup(alert_severity)) { case alert_level_group_notice_or_lower: num_alerts_engaged_notice[i]--; break; case alert_level_group_warning: num_alerts_engaged_warning[i]--; break; case alert_level_group_error: num_alerts_engaged_error[i]--; break; case alert_level_group_critical: num_alerts_engaged_critical[i]--; break; case alert_level_group_emergency: num_alerts_engaged_emergency[i]--; break; default: break; } } } /* *************************************** */ u_int32_t NetworkInterface::getNumEngagedAlerts() const { u_int32_t tot_engaged_alerts = 0; for (int i = 0; i < ALERT_ENTITY_MAX_NUM_ENTITIES; i++) tot_engaged_alerts += num_alerts_engaged_notice[i] + num_alerts_engaged_warning[i] + num_alerts_engaged_error[i] + num_alerts_engaged_critical[i] + num_alerts_engaged_emergency[i]; return tot_engaged_alerts; }; /* *************************************** */ u_int32_t NetworkInterface::getNumEngagedAlerts( AlertLevelGroup alert_level_group) const { u_int32_t tot_engaged_alerts = 0; const u_int32_t(*num_alerts_engaged)[ALERT_ENTITY_MAX_NUM_ENTITIES] = NULL; switch (alert_level_group) { case alert_level_group_notice_or_lower: num_alerts_engaged = &num_alerts_engaged_notice; break; case alert_level_group_warning: num_alerts_engaged = &num_alerts_engaged_warning; break; case alert_level_group_error: num_alerts_engaged = &num_alerts_engaged_error; break; case alert_level_group_critical: num_alerts_engaged = &num_alerts_engaged_critical; break; case alert_level_group_emergency: num_alerts_engaged = &num_alerts_engaged_emergency; break; default: return tot_engaged_alerts; } for (int i = 0; i < ALERT_ENTITY_MAX_NUM_ENTITIES; i++) tot_engaged_alerts += (*num_alerts_engaged)[i]; return tot_engaged_alerts; }; /* *************************************** */ void NetworkInterface::luaNumEngagedAlerts(lua_State *vm) const { lua_push_int32_table_entry(vm, "num_alerts_engaged", getNumEngagedAlerts()); /* By Entity */ lua_newtable(vm); for (int i = 0; i < ALERT_ENTITY_MAX_NUM_ENTITIES; i++) { u_int32_t num_alerts = num_alerts_engaged_notice[i] + num_alerts_engaged_warning[i] + num_alerts_engaged_error[i] + num_alerts_engaged_critical[i] + num_alerts_engaged_emergency[i]; if (num_alerts) /* Use string keys for entity id to avoid confusing lua and treating the * table as an array */ lua_push_uint64_table_entry(vm, to_string(i).c_str(), num_alerts); } lua_pushstring(vm, "num_alerts_engaged_by_entity"); lua_insert(vm, -2); lua_settable(vm, -3); /* By severity */ lua_newtable(vm); lua_push_uint64_table_entry( vm, "notice", getNumEngagedAlerts(alert_level_group_notice_or_lower)); lua_push_uint64_table_entry(vm, "warning", getNumEngagedAlerts(alert_level_group_warning)); lua_push_uint64_table_entry(vm, "error", getNumEngagedAlerts(alert_level_group_error)); lua_push_uint64_table_entry(vm, "critical", getNumEngagedAlerts(alert_level_group_critical)); lua_push_uint64_table_entry(vm, "emergency", getNumEngagedAlerts(alert_level_group_emergency)); lua_pushstring(vm, "num_alerts_engaged_by_severity"); lua_insert(vm, -2); lua_settable(vm, -3); }; /* *************************************** */ void NetworkInterface::incNumDroppedAlerts(AlertEntity alert_entity) { switch (alert_entity) { case alert_entity_host: num_host_dropped_alerts++; break; case alert_entity_flow: num_flow_dropped_alerts++; break; default: num_other_dropped_alerts++; break; } } /* *************************************** */ struct get_engaged_alerts_userdata { lua_State *vm; AlertEntity alert_entity; AlertType alert_type; AlertLevel alert_severity; AlertRole role_filter; u_int idx; }; static void get_engaged_alerts_callback(AlertEntity alert_entity, AlertableEntity *alertable, void *user_data) { struct get_engaged_alerts_userdata *data = (struct get_engaged_alerts_userdata *)user_data; alertable->getAlerts(data->vm, no_periodicity, data->alert_type, data->alert_severity, data->role_filter, &data->idx); } /* *************************************** */ void NetworkInterface::getEngagedAlerts(lua_State *vm, AlertEntity alert_entity, const char *entity_value, AlertType alert_type, AlertLevel alert_severity, AlertRole role_filter, AddressTree *allowed_nets) { struct get_engaged_alerts_userdata data; data.vm = vm; data.idx = 0; data.alert_entity = alert_entity; data.alert_type = alert_type; data.alert_severity = alert_severity; data.role_filter = role_filter; lua_newtable(vm); walkAlertables(alert_entity, entity_value, allowed_nets, get_engaged_alerts_callback, &data); } /* *************************************** */ void NetworkInterface::processExternalAlertable(AlertEntity entity, const char *entity_val, lua_State *vm, u_int vm_argument_idx, bool do_store_alert) { std::map, InterfaceMemberAlertableEntity *>::iterator it; std::pair key(entity, std::string(entity_val)); InterfaceMemberAlertableEntity *alertable = NULL; external_alerts_lock.lock(__FILE__, __LINE__); /* Lookup */ if ((it = external_alerts.find(key)) != external_alerts.end()) alertable = it->second; if (alertable) { /* Already present */ if (do_store_alert) { /* Trigger, but already present */ /* Nothing to store - return */ external_alerts_lock.unlock(__FILE__, __LINE__); lua_pushnil(vm); return; } } else { /* Not present */ if (!do_store_alert) { /* Release, but not found */ /* Nothing to release - return */ external_alerts_lock.unlock(__FILE__, __LINE__); lua_pushnil(vm); return; } /* Trigger (create) */ alertable = new (std::nothrow) InterfaceMemberAlertableEntity(this, entity); if (alertable == NULL) { external_alerts_lock.unlock(__FILE__, __LINE__); ntop->getTrace()->traceEvent(TRACE_ERROR, "Not enough memory"); lua_pushnil(vm); return; } alertable->setEntityValue(entity_val); /* Add to the map */ external_alerts[key] = alertable; } if (do_store_alert) ntop_store_triggered_alert(vm, alertable, vm_argument_idx); else { ntop_release_triggered_alert(vm, alertable, vm_argument_idx); if (alertable->getNumEngagedAlerts() == 0) { external_alerts.erase(key); delete alertable; } } external_alerts_lock.unlock(__FILE__, __LINE__); } /* *************************************** */ void NetworkInterface::luaAlertedFlows(lua_State *vm) { /* Total */ lua_push_int32_table_entry(vm, "num_alerted_flows", getNumActiveAlertedFlows()); /* Breakdown */ lua_push_int32_table_entry( vm, "num_alerted_flows_notice", getNumActiveAlertedFlows(alert_level_group_notice_or_lower)); lua_push_int32_table_entry( vm, "num_alerted_flows_warning", getNumActiveAlertedFlows(alert_level_group_warning)); lua_push_int32_table_entry(vm, "num_alerted_flows_error", getNumActiveAlertedFlows(alert_level_group_error)); } /* *************************************** */ void NetworkInterface::luaPeriodicityFilteringMenu(lua_State *vm, MapsFilters *filters) { #if defined(NTOPNG_PRO) && !defined(HAVE_NEDGE) if (pMap) { pMap->luaFilteringMenu(vm, this, true, filters); return; } #endif lua_pushnil(vm); } /* *************************************** */ void NetworkInterface::luaServiceFilteringMenu(lua_State *vm, MapsFilters *filters) { #if defined(NTOPNG_PRO) if (sMap) { sMap->luaFilteringMenu(vm, this, false, filters); return; } #endif lua_pushnil(vm); } /* *************************************** */ void NetworkInterface::luaPeriodicityMap(lua_State *vm, MapsFilters *filters) { #if defined(NTOPNG_PRO) && !defined(HAVE_NEDGE) if (pMap) { pMap->lua(vm, filters); return; } #endif lua_pushnil(vm); } /* *************************************** */ void NetworkInterface::luaServiceMap(lua_State *vm, MapsFilters *filters) { #if defined(NTOPNG_PRO) if (sMap) { sMap->lua(vm, filters); return; } #endif lua_pushnil(vm); } /* *************************************** */ #if defined(NTOPNG_PRO) void NetworkInterface::updateFlowPeriodicity(Flow *f) { if (isViewed()) viewedBy()->updateFlowPeriodicity(f); else if (pMap) pMap->updateElement(f, f->get_first_seen()); } /* *************************************** */ void NetworkInterface::updateServiceMap(Flow *f) { if (isViewed()) viewedBy()->updateServiceMap(f); else if (sMap) sMap->update(f, f->get_first_seen()); } #endif /* *************************************** */ void NetworkInterface::updateSitesStats() { if (ntop->getPrefs()->are_top_talkers_enabled()) { /* String used to add extra info to the redis key */ std::string additional_key_info = ""; if (top_sites) top_sites->saveOldData(get_id(), (char *)additional_key_info.c_str(), (char *)HASHKEY_LOCAL_HOSTS_TOP_SITES_HOUR_KEYS_PUSHED); } } /* *************************************** */ void NetworkInterface::incrVisitedWebSite(char *hostname) { if (top_sites) top_sites->incrVisitedData(hostname, 1); } /* *************************************** */ HostCheck *NetworkInterface::getCheck(HostCheckID t) { return (host_checks_executor->getCheck(t)); } /* *************************************** */ void NetworkInterface::execFlowBeginChecks(Flow *f) { if (flow_checks_executor) { flow_checks_executor->execChecks(f, flow_check_flow_begin); } } /* *************************************** */ void NetworkInterface::execProtocolDetectedChecks(Flow *f) { if (flow_checks_executor) { flow_checks_executor->execChecks(f, flow_check_protocol_detected); } } /* *************************************** */ void NetworkInterface::execPeriodicUpdateChecks(Flow *f) { if (flow_checks_executor) { flow_checks_executor->execChecks(f, flow_check_periodic_update); } } /* *************************************** */ void NetworkInterface::execFlowEndChecks(Flow *f) { if (flow_checks_executor) { flow_checks_executor->execChecks(f, flow_check_flow_end); } } /* *************************************** */ void NetworkInterface::luaScore(lua_State *vm) { /* Score */ lua_newtable(vm); lua_push_uint64_table_entry(vm, "score_as_cli", score_as_cli); lua_push_uint64_table_entry(vm, "score_as_srv", score_as_srv); lua_pushstring(vm, "score"); lua_insert(vm, -2); lua_settable(vm, -3); } /* *************************************** */ void NetworkInterface::luaNdpiStats(lua_State *vm, bool diff) { /* nDPI stats */ if (ndpiStats) ndpiStats->lua(this, vm, true, diff); } /* *************************************** */ void NetworkInterface::luaAnomalies(lua_State *vm) { /* Anomalies */ lua_newtable(vm); if (has_too_many_flows) lua_push_bool_table_entry(vm, "too_many_flows", true); if (has_too_many_hosts) lua_push_bool_table_entry(vm, "too_many_hosts", true); lua_push_uint32_table_entry(vm, "num_local_hosts_anomalies", tot_num_anomalies.local_hosts); lua_push_uint32_table_entry(vm, "num_remote_hosts_anomalies", tot_num_anomalies.remote_hosts); lua_pushstring(vm, "anomalies"); lua_insert(vm, -2); lua_settable(vm, -3); } /* *************************************** */ void NetworkInterface::execHostChecks(Host *h) { if (host_checks_executor) host_checks_executor->execChecks(h); } /* *************************************** */ void NetworkInterface::incObservationPointIdFlows(u_int16_t pointId) { ObservationPoint *obs_point_stats = obs_hash ? obs_hash->get(pointId, true /* Lock */) : NULL; if (obs_point_stats) obs_point_stats->incFlows(); } /* *************************************** */ bool NetworkInterface::hasObservationPointId(u_int16_t pointId) { return (((obs_hash) && (obs_hash->get(pointId, false)) ? true : false)); } /* *************************************** */ bool NetworkInterface::haveObservationPointsDefined() { return (((!obs_hash) || (obs_hash->getNumEntries() == 0)) ? false : true); } /* *************************************** */ u_int16_t NetworkInterface::getFirstObservationPointId() { return (((!obs_hash) || (obs_hash->getNumEntries() == 0)) ? 0 : last_obs_point_id); } /* *************************************** */ void NetworkInterface::removeRedisSitesKey() { // System Interface, no Network sites for sure if (id == -1 || !top_sites) return; top_sites->serializeDeserialize( id, false, (char *)"", (char *)HASHKEY_TOP_SITES_SERIALIZATION_KEY, (char *)HASHKEY_LOCAL_HOSTS_TOP_SITES_HOUR_KEYS_PUSHED, (char *)HASHKEY_LOCAL_HOSTS_TOP_SITES_DAY_KEYS_PUSHED); } /* *************************************** */ void NetworkInterface::addRedisSitesKey() { // System Interface, no Network sites for sure if (id == -1 || !top_sites) return; top_sites->serializeDeserialize( id, true, (char *)"", (char *)HASHKEY_TOP_SITES_SERIALIZATION_KEY, (char *)HASHKEY_LOCAL_HOSTS_TOP_SITES_HOUR_KEYS_PUSHED, (char *)HASHKEY_LOCAL_HOSTS_TOP_SITES_DAY_KEYS_PUSHED); } /* *************************************** */ int NetworkInterface::exec_csv_query(const char *sql, bool dump_in_json_format, struct mg_connection *conn) { #if defined(NTOPNG_PRO) && defined(HAVE_MYSQL) && defined(HAVE_CLICKHOUSE) ((ClickHouseFlowDB *)db)->execCSVQuery(sql, dump_in_json_format, conn); return (0); #endif return (-1); }; /* *************************************** */ struct host_walker_metadata { std::vector info; int32_t networkIdFilter; HostWalkMode mode; bool localHostsOnly; }; static bool active_hosts_walker(GenericHashEntry *h, void *user_data, bool *matched) { Host *host = (Host *)h; struct host_walker_metadata *m = (struct host_walker_metadata *)user_data; bool isLocal; if ((host == NULL) || ((m->networkIdFilter != -1 /* -1 == all networks */) && (host->get_local_network_id() != m->networkIdFilter))) return (false); isLocal = host->isLocalHost() || host->isSystemHost(); if ((m->localHostsOnly && isLocal) || ((m->localHostsOnly == false) && (!isLocal))) host->visit(&m->info, m->mode); return (false); /* false = keep on walking */ } /* *************************************** */ static bool walkerSort(const ActiveHostWalkerInfo &a, const ActiveHostWalkerInfo &b) { return (a.getZ() > b.getZ()); } /* *************************************** */ int NetworkInterface::walkActiveHosts(lua_State *vm, HostWalkMode mode, u_int32_t maxHits, int32_t networkIdFilter, /* -1 = means any network */ bool localHostsOnly, bool treeMapMode) { u_int32_t begin_slot = 0; struct host_walker_metadata m; int rc; std::vector::iterator it; m.mode = mode, m.localHostsOnly = localHostsOnly, m.networkIdFilter = networkIdFilter; rc = walker(&begin_slot, true /* walk_all */, walker_hosts, active_hosts_walker, (void *)&m); if ((rc != 0) || (m.info.size() == 0)) { lua_pushnil(vm); return (-1); } else { u_int num = 0; std::sort(m.info.begin(), m.info.end(), walkerSort); lua_newtable(vm); for (it = m.info.begin(); (num < maxHits) && (it != m.info.end()); ++it, num++) { it->lua(vm, treeMapMode); // ntop->getTrace()->traceEvent(TRACE_WARNING, "[%u] %u", num+1, // it->getZ()); lua_rawseti(vm, -2, num + 1); /* Array */ } return (m.info.size()); } } /* *************************************** */ #ifdef NTOPNG_PRO void NetworkInterface::checkDHCPStorm(time_t when, u_int32_t num_pkts) { if (last_sec_epoch == (u_int32_t)when) dhcp_last_sec_pkts += num_pkts; else { if (dhcp_last_sec_pkts > DHCP_STORM_PPS_THSHOLD) { char value[32], key[32]; #ifdef DEBUG ntop->getTrace()->traceEvent( TRACE_NORMAL, "DHCP [iface: %d][when: %u][total: %u]", get_id(), last_sec_epoch, dhcp_last_sec_pkts); #endif /* Queue alert for the DHCP storm plugin */ snprintf(key, sizeof(key), DHCP_STORM_QUEUE_NAME, get_id()); snprintf(value, sizeof(value), "%u;%u", last_sec_epoch, dhcp_last_sec_pkts); ntop->getRedis()->rpush(key, value, 32 /* trim size */); } last_sec_epoch = when, dhcp_last_sec_pkts = num_pkts; /* Reset counter */ } } #endif /* *************************************** */ void NetworkInterface::incNumHosts(Host *host, bool rxOnlyHost) { bool local; if(!host->isUnicastHost()) return; /* Account only unicast hosts */ local = host->isLocalHost(); /* Do not increase nor decrease hosts in case ntopng is shutting down, it's useless */ if(isShuttingDown() || (!host->isUnicastHost())) return; //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Increasing number of %s %s hosts", local ? "Local" : "Remote", rxOnlyHost ? "RX Only" : "Bidirectional"); totalNumHosts++; if(local) numLocalHosts++; if(rxOnlyHost) { numTotalRxOnlyHosts++; if (local) numLocalRxOnlyHosts++; } #ifdef DEBUG ntop->getTrace()->traceEvent(TRACE_NORMAL, "****** %s() [totalNumHosts: %u][numTotalRxOnlyHosts: %u][numLocalRxOnlyHosts: %u]", __FUNCTION__, totalNumHosts.load(), numTotalRxOnlyHosts.load(), numLocalRxOnlyHosts.load()); #endif } /* *************************************** */ void NetworkInterface::decNumHosts(Host *host, bool rxOnlyHost) { bool local; if(!host->isUnicastHost()) return; /* Account only unicast hosts */ local = host->isLocalHost(); /* Do not increase nor decrease hosts in case ntopng is shutting down, it's useless */ if(isShuttingDown() || (!host->isUnicastHost())) return; //ntop->getTrace()->traceEvent(TRACE_NORMAL, "Decreasing number of %s %s hosts", local ? "Local" : "Remote", rxOnlyHost ? "RX Only" : "Bidirectional"); /* Decrease total number of hosts */ if (totalNumHosts.load() == 0) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Internal Error (%d) on interface %s: Counter overflow [totalNumHosts]", 1, ifname); } else { totalNumHosts--; } /* Decrease total number of local hosts */ if (local) { if (numLocalHosts.load() == 0) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Internal Error (%d) on interface %s: Counter overflow [numLocalHosts]", 2, ifname); } else { numLocalHosts--; } } if(rxOnlyHost) { /* Decrease total number of RX only hosts */ if (numTotalRxOnlyHosts.load() == 0) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Internal Error (%d) on interface %s: Counter overflow [numTotalRxOnlyHosts]", 3, ifname); } else { numTotalRxOnlyHosts--; } /* Decrease total number of RX only local hosts */ if(local) { if (numLocalRxOnlyHosts.load() == 0) { ntop->getTrace()->traceEvent(TRACE_WARNING, "Internal Error (%d) on interface %s: Counter overflow [numLocalRxOnlyHosts]", 4, ifname); } else { numLocalRxOnlyHosts--; } } } #ifdef DEBUG ntop->getTrace()->traceEvent(TRACE_NORMAL, "****** %s() [totalNumHosts: %u][numTotalRxOnlyHosts: %u][numLocalRxOnlyHosts: %u]", __FUNCTION__, totalNumHosts.load(), numTotalRxOnlyHosts.load(), numLocalRxOnlyHosts.load()); #endif }; /* **************************************************** */ bool NetworkInterface::resetHostTopSites(AddressTree *allowed_hosts, char *host_ip, u_int16_t vlan_id, u_int16_t observationPointId) { Host *h = findHostByIP(allowed_hosts, host_ip, vlan_id, observationPointId); if (h) return (h->resetHostTopSites()); else return (false); } /* **************************************************** */ bool NetworkInterface::compute_protocol_flow_stats(GenericHashEntry *node, void *user_data, bool *matched) { Flow *f = (Flow *)node; if (!f) return false; u_int64_t key = 0; u_int64_t vlan_id = f->get_vlan_id(); ndpi_protocol detected_protocol = f->get_detected_protocol(); std::unordered_map::iterator it; struct aggregated_stats *stats = (struct aggregated_stats*)user_data; bool is_not_guessed = f->isDPIDetectedFlow(); if(stats->ip_addr != NULL) { if(!f->matchFlowIP(stats->ip_addr, stats->vlan_id)) return(false); } if(stats->vlan_id != NO_VLAN /* -1 == any VLAN */) { if(!f->matchFlowVLAN(stats->vlan_id)) return(false); } if (stats->flow_device_ip != (u_int32_t)-1 /* -1 == any Flow Device IP */) { if(!f->matchFlowDeviceIP(stats->flow_device_ip)) return(false); } if (stats->in_if_index != NO_IN_IF_INDEX /* -1 == any Flow Device In Interface Index */) { if (!f->matchInIfIdx(stats->in_if_index)) return(false); } if (stats->out_if_index != NO_OUT_IF_INDEX /* -1 == any Flow Device In Interface Index */) { if (!f->matchOutIfIdx(stats->out_if_index)) return(false); } /* count.find(key); if (it == stats->count.end()) { AggregatedFlowsStats *fs = new (std::nothrow) AggregatedFlowsStats(f->get_cli_ip_addr(), f->get_srv_ip_addr(), f->get_protocol(), f->get_bytes_cli2srv(), f->get_bytes_srv2cli(), f->getScore()); if (fs) { fs->setProtoKey(key); fs->setFlowIPVLANDeviceIP(f); fs->setIsNotGuessed(is_not_guessed); stats->count[key] = fs; } } else { it->second->incFlowStats(f->get_cli_ip_addr(), f->get_srv_ip_addr(), f->get_bytes_cli2srv(), f->get_bytes_srv2cli(), f->getScore()); } *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ bool NetworkInterface::compute_client_flow_stats(GenericHashEntry *node, void *user_data, bool *matched) { Flow *f = (Flow *)node; if(!f) return false; u_int64_t vlan_id = f->get_vlan_id(); u_int64_t key = (((u_int64_t)f->get_cli_ip_addr()->key()) << 16) + ((u_int64_t)vlan_id); std::unordered_map::iterator it; struct aggregated_stats *stats = (struct aggregated_stats*)user_data; if(stats->ip_addr != NULL) { if(!f->matchFlowIP(stats->ip_addr, stats->vlan_id)) return(false); } if(stats->vlan_id != NO_VLAN /* -1 == any VLAN */) { if(!f->matchFlowVLAN(stats->vlan_id)) return(false); } if (stats->flow_device_ip != (u_int32_t)-1 /* -1 == any Flow Device IP */) { if(!f->matchFlowDeviceIP(stats->flow_device_ip)) return(false); } if (stats->in_if_index != NO_IN_IF_INDEX /* -1 == any Flow Device In Interface Index */) { if (!f->matchInIfIdx(stats->in_if_index)) return(false); } if (stats->out_if_index != NO_OUT_IF_INDEX /* -1 == any Flow Device In Interface Index */) { if (!f->matchOutIfIdx(stats->out_if_index)) return(false); } it = stats->count.find(key); if (it == stats->count.end()) { AggregatedFlowsStats *fs = new (std::nothrow) AggregatedFlowsStats(f->get_cli_ip_addr(), f->get_srv_ip_addr(), f->get_protocol(), f->get_bytes_cli2srv(), f->get_bytes_srv2cli(), f->getScore()); if (fs != NULL) { fs->setFlowIPVLANDeviceIP(f); fs->setKey(key); stats->count[key] = fs; } } else { it->second->incFlowStats(f->get_cli_ip_addr(), f->get_srv_ip_addr(), f->get_bytes_cli2srv(), f->get_bytes_srv2cli(), f->getScore()); } *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ bool NetworkInterface::compute_server_flow_stats(GenericHashEntry *node, void *user_data, bool *matched) { Flow *f = (Flow *)node; if(!f) return false; u_int64_t vlan_id = f->get_vlan_id(); u_int64_t key = (((u_int64_t)f->get_srv_ip_addr()->key()) << 16) + ((u_int64_t)vlan_id); std::unordered_map::iterator it; struct aggregated_stats *stats = (struct aggregated_stats*)user_data; if(stats->ip_addr != NULL) { if(!f->matchFlowIP(stats->ip_addr, stats->vlan_id)) return(false); } if(stats->vlan_id != NO_VLAN/* -1 == any VLAN */) { if(!f->matchFlowVLAN(stats->vlan_id)) return(false); } if (stats->flow_device_ip != (u_int32_t)-1 /* -1 == any Flow Device IP */) { if(!f->matchFlowDeviceIP(stats->flow_device_ip)) return(false); } if (stats->in_if_index != NO_IN_IF_INDEX /* -1 == any Flow Device In Interface Index */) { if (!f->matchInIfIdx(stats->in_if_index)) return(false); } if (stats->out_if_index != NO_OUT_IF_INDEX /* -1 == any Flow Device In Interface Index */) { if (!f->matchOutIfIdx(stats->out_if_index)) return(false); } it = stats->count.find(key); if (it == stats->count.end()) { AggregatedFlowsStats *fs = new (std::nothrow) AggregatedFlowsStats(f->get_cli_ip_addr(), f->get_srv_ip_addr(), f->get_protocol(), f->get_bytes_cli2srv(), f->get_bytes_srv2cli(), f->getScore()); if (fs != NULL) { fs->setFlowIPVLANDeviceIP(f); fs->setKey(key); stats->count[key] = fs; } } else { it->second->incFlowStats(f->get_cli_ip_addr(), f->get_srv_ip_addr(), f->get_bytes_cli2srv(), f->get_bytes_srv2cli(), f->getScore()); } *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ bool NetworkInterface::compute_client_server_srv_port_flow_stats(GenericHashEntry *node, void *user_data, bool *matched) { Flow *f = (Flow *)node; struct aggregated_stats *stats = (struct aggregated_stats*)user_data; if(stats->ip_addr != NULL) { if(!f->matchFlowIP(stats->ip_addr, stats->vlan_id)) return(false); } if(stats->vlan_id != NO_VLAN /* -1 == any VLAN */) { if(!f->matchFlowVLAN(stats->vlan_id)) return(false); } if (stats->flow_device_ip != (u_int32_t)-1 /* -1 == any Flow Device IP */) { if(!f->matchFlowDeviceIP(stats->flow_device_ip)) return(false); } if (stats->in_if_index != NO_IN_IF_INDEX /* -1 == any Flow Device In Interface Index */) { if (!f->matchInIfIdx(stats->in_if_index)) return(false); } if (stats->out_if_index != NO_OUT_IF_INDEX /* -1 == any Flow Device In Interface Index */) { if (!f->matchOutIfIdx(stats->out_if_index)) return(false); } u_int64_t vlan_id = f->get_vlan_id(); std::unordered_map::iterator it; u_int64_t key = (((u_int64_t)f->get_cli_ip_addr()->key()) << 16) + (((u_int64_t)f->get_srv_ip_addr()->key()) << 16) + (((u_int64_t)f->get_srv_port()) << 32) + ((u_int64_t)vlan_id << 32); it = stats->count.find(key); if (it == stats->count.end()) { AggregatedFlowsStats *fs = new (std::nothrow) AggregatedFlowsStats(f->get_cli_ip_addr(), f->get_srv_ip_addr(), f->get_protocol(), f->get_bytes_cli2srv(), f->get_bytes_srv2cli(), f->getScore()); if (fs != NULL) { fs->setFlowIPVLANDeviceIP(f); fs->setSrvPort(f->get_srv_port()); fs->setKey(key); stats->count[key] = fs; } } else it->second->incFlowStats(f->get_cli_ip_addr(), f->get_srv_ip_addr(), f->get_bytes_cli2srv(), f->get_bytes_srv2cli(), f->getScore()); *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ bool NetworkInterface::compute_client_server_srv_port_app_proto_flow_stats(GenericHashEntry *node, void *user_data, bool *matched) { Flow *f = (Flow *)node; struct aggregated_stats *stats = (struct aggregated_stats*)user_data; if(stats->ip_addr != NULL) { if(!f->matchFlowIP(stats->ip_addr, stats->vlan_id)) return(false); } if(stats->vlan_id != (u_int16_t)-1 /* -1 == any VLAN */) { if(!f->matchFlowVLAN(stats->vlan_id)) return(false); } if (stats->flow_device_ip != (u_int32_t)-1 /* -1 == any Flow Device IP */) { if(!f->matchFlowDeviceIP(stats->flow_device_ip)) return(false); } if (stats->in_if_index != NO_IN_IF_INDEX /* -1 == any Flow Device In Interface Index */) { if (!f->matchInIfIdx(stats->in_if_index)) return(false); } if (stats->out_if_index != NO_OUT_IF_INDEX /* -1 == any Flow Device In Interface Index */) { if (!f->matchOutIfIdx(stats->out_if_index)) return(false); } ndpi_protocol detected_protocol = f->get_detected_protocol(); u_int64_t vlan_id = f->get_vlan_id(); bool is_not_guessed = f->isDPIDetectedFlow(); std::unordered_map::iterator it; u_int64_t key = (((u_int64_t)f->get_cli_ip_addr()->key()) << 16) + (((u_int64_t)f->get_srv_ip_addr()->key()) << 16) + (((u_int64_t)f->get_srv_port()) << 32) + ((u_int64_t)vlan_id << 32) + (((u_int64_t)detected_protocol.proto.app_protocol) << 16) + (u_int64_t)detected_protocol.proto.master_protocol; u_int64_t proto_key = ((u_int64_t)vlan_id << 32) + (((u_int64_t)detected_protocol.proto.app_protocol) << 16) + (u_int64_t)detected_protocol.proto.master_protocol; it = stats->count.find(key); if (it == stats->count.end()) { AggregatedFlowsStats *fs = new (std::nothrow) AggregatedFlowsStats(f->get_cli_ip_addr(), f->get_srv_ip_addr(), f->get_protocol(), f->get_bytes_cli2srv(), f->get_bytes_srv2cli(), f->getScore()); if (fs != NULL) { fs->setFlowIPVLANDeviceIP(f); fs->setSrvPort(f->get_srv_port()); fs->setKey(key); fs->setProtoKey(proto_key); fs->setIsNotGuessed(is_not_guessed); stats->count[key] = fs; } } else it->second->incFlowStats(f->get_cli_ip_addr(), f->get_srv_ip_addr(), f->get_bytes_cli2srv(), f->get_bytes_srv2cli(), f->getScore()); *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ /* Sort aggregated live flows compare functions */ static bool asc_application_name_cmp(AggregatedFlowsStats *a, AggregatedFlowsStats *b) { return strcasecmp(a->getProtoName(), b->getProtoName()) < 0; } static bool asc_str_info_cmp(AggregatedFlowsStats *a, AggregatedFlowsStats *b) { return strcasecmp(a->getInfoKey(), b->getInfoKey()) < 0; } static bool asc_cli_ip_hex_cmp(AggregatedFlowsStats *a, AggregatedFlowsStats *b) { char a_buf[48], b_buf[48]; return strcmp(a->getCliIPHex(a_buf, sizeof(a_buf)), b->getCliIPHex(b_buf, sizeof(b_buf))) < 0; } static bool asc_srv_ip_hex_cmp(AggregatedFlowsStats *a, AggregatedFlowsStats *b) { char a_buf[48], b_buf[48]; return strcmp(a->getSrvIPHex(a_buf, sizeof(a_buf)), b->getSrvIPHex(b_buf, sizeof(b_buf))) < 0; } static bool asc_srv_cli_ip_hex_cmp(AggregatedFlowsStats *a, AggregatedFlowsStats *b) { char a_c_buf[48], a_s_buf[48], b_c_buf[48], b_s_buf[48]; char a_sc_buf[96], b_sc_buf[96]; snprintf(a_sc_buf, sizeof(a_sc_buf), "%s%s", a->getCliIPHex(a_c_buf, sizeof(a_c_buf)), a->getSrvIPHex(a_s_buf, sizeof(a_s_buf))); snprintf(b_sc_buf, sizeof(b_sc_buf), "%s%s", b->getCliIPHex(b_c_buf, sizeof(b_c_buf)), b->getSrvIPHex(b_s_buf, sizeof(b_s_buf))); return (strcmp(a_sc_buf, b_sc_buf) < 0); } static bool asc_flow_num_cmp(AggregatedFlowsStats *a, AggregatedFlowsStats *b) { return a->getNumFlows() < b->getNumFlows(); } static bool asc_total_score_cmp(AggregatedFlowsStats *a, AggregatedFlowsStats *b) { return a->getTotalScore() < b->getTotalScore(); } static bool asc_numclients_cmp(AggregatedFlowsStats *a, AggregatedFlowsStats *b) { return a->getNumClients() < b->getNumClients(); } static bool asc_vlan_cmp(AggregatedFlowsStats *a, AggregatedFlowsStats *b) { return a->getVlanId() < b->getVlanId(); } static bool asc_num_servers_cmp(AggregatedFlowsStats *a, AggregatedFlowsStats *b) { return a->getNumServers() < b->getNumServers(); } static bool asc_total_sent_cmp(AggregatedFlowsStats *a, AggregatedFlowsStats *b) { return a->getTotalSent() < b->getTotalSent(); } static bool asc_total_rcvd_cmp(AggregatedFlowsStats *a, AggregatedFlowsStats *b) { return a->getTotalRcvd() < b->getTotalRcvd(); } static bool asc_total_traffic_cmp(AggregatedFlowsStats *a, AggregatedFlowsStats *b) { return (a->getTotalRcvd() + a->getTotalSent()) < (b->getTotalRcvd() + b->getTotalSent()); } static bool asc_srv_port_cmp(AggregatedFlowsStats *a, AggregatedFlowsStats *b) { return a->getSrvPort() < b->getSrvPort(); } /* Sort host ports compare functions */ static bool host_details_asc_total_traffic_cmp(HostDetails *a, HostDetails *b) { return (a->get_total_traffic() < b->get_total_traffic()); } static bool host_details_asc_ip_cmp(HostDetails *a, HostDetails *b) { return strcmp(a->get_ip_hex(), b->get_ip_hex()) < 0; } static bool host_details_asc_mac_cmp(HostDetails *a, HostDetails *b) { return strcmp(a->get_mac_address(), b->get_mac_address()) < 0; } static bool host_details_asc_mac_manufacturer_cmp(HostDetails *a, HostDetails *b) { return strcmp(a->get_mac_manufacturer(), b->get_mac_manufacturer()) < 0; } static bool host_details_asc_score_cmp(HostDetails *a, HostDetails *b) { return (a->get_score() < b->get_score()); } static bool host_details_asc_flows_cmp(HostDetails *a, HostDetails *b) { return (a->get_active_flows_as_server() < b->get_active_flows_as_server()); } static bool host_details_asc_vlan_cmp(HostDetails *a, HostDetails *b) { return (a->get_vlan_id() < b->get_vlan_id()); } static bool host_details_asc_name_cmp(HostDetails *a, HostDetails *b) { return strcmp(a->get_name(), b->get_name()) < 0; } /* **************************************************** */ /* Analysis Flows Stats Lua response builders */ void NetworkInterface::build_lua_rsp(lua_State *vm, AggregatedFlowsStats *flow_stats, u_int filter_type, u_int32_t size, u_int *num, bool set_resp) { if (set_resp) { char buf[128]; u_int8_t add_client = false, add_server = false, add_app_proto = false, add_info = false, add_srv_port = false; lua_newtable(vm); switch (filter_type) { case 1: add_app_proto = true; break; case 3: add_server = true; break; case 4: add_client = add_server = true; break; case 5: add_app_proto = add_client = add_server = true; break; case 6: add_info = true; break; case 7: add_client = add_server = add_srv_port = true; break; case 8: add_client = add_server = add_srv_port = add_app_proto = true; break; default: add_client = true; break; } if (add_app_proto) { ndpi_protocol detected_protocol; char buf[64], proto[16]; u_int64_t key = flow_stats->getProtoKey(); detected_protocol.proto.master_protocol = (u_int16_t)(key & 0x00000000000FFFF); detected_protocol.proto.app_protocol = (u_int16_t)((key >> 16) & 0x000000000000FFFF); if (detected_protocol.proto.master_protocol == detected_protocol.proto.app_protocol) snprintf(proto, sizeof(proto), "%u", detected_protocol.proto.master_protocol); else if (detected_protocol.proto.app_protocol == NDPI_PROTOCOL_UNKNOWN) snprintf(proto, sizeof(proto), "%u", detected_protocol.proto.master_protocol); else if (detected_protocol.proto.master_protocol == NDPI_PROTOCOL_UNKNOWN) snprintf(proto, sizeof(proto), "%u", detected_protocol.proto.app_protocol); else snprintf(proto, sizeof(proto), "%u.%u", detected_protocol.proto.master_protocol, detected_protocol.proto.app_protocol); /* Currently it is not supported the possibily to add double filter on * master and app proto */ lua_push_str_table_entry(vm, "proto_id", proto); lua_push_str_table_entry(vm, "proto_name", get_ndpi_full_proto_name(detected_protocol, buf, sizeof(buf))); lua_push_str_table_entry(vm, "proto_name", get_ndpi_full_proto_name(detected_protocol, buf, sizeof(buf))); lua_push_bool_table_entry(vm, "is_not_guessed", flow_stats->isNotGuessed()); } if (add_client) { lua_push_uint64_table_entry(vm, "cli_vlan_id", (u_int64_t)flow_stats->getCliVLANId()); lua_push_str_table_entry(vm, "client_ip", flow_stats->getCliIP(buf, sizeof(buf))); lua_push_str_table_entry(vm, "client_name", flow_stats->getCliName(buf, sizeof(buf))); lua_push_bool_table_entry(vm, "is_cli_in_mem", flow_stats->isCliInMem()); } if (add_server) { lua_push_uint64_table_entry(vm, "srv_vlan_id", (u_int64_t)flow_stats->getSrvVLANId()); lua_push_str_table_entry(vm, "server_ip", flow_stats->getSrvIP(buf, sizeof(buf))); lua_push_str_table_entry(vm, "server_name", flow_stats->getSrvName(buf, sizeof(buf))); lua_push_bool_table_entry(vm, "is_srv_in_mem", flow_stats->isSrvInMem()); } if (add_srv_port) { lua_push_uint64_table_entry(vm, "srv_port", (u_int64_t)flow_stats->getSrvPort()); } if (add_info) { lua_push_str_table_entry(vm, "info", flow_stats->getInfoKey()); } lua_push_uint64_table_entry(vm, "vlan_id", (u_int64_t)flow_stats->getVlanId()); lua_push_str_table_entry(vm, "device_ip", flow_stats->getFlowDeviceIP(buf, sizeof(buf))); lua_push_uint32_table_entry(vm, "l4_proto", flow_stats->getL4Protocol()); lua_push_uint32_table_entry(vm, "num_clients", flow_stats->getNumClients()); lua_push_uint32_table_entry(vm, "num_servers", flow_stats->getNumServers()); lua_push_uint32_table_entry(vm, "num_flows", flow_stats->getNumFlows()); lua_push_uint64_table_entry(vm, "bytes_sent", flow_stats->getTotalSent()); lua_push_uint64_table_entry(vm, "bytes_rcvd", flow_stats->getTotalRcvd()); lua_push_uint64_table_entry(vm, "total_score", flow_stats->getTotalScore()); lua_push_uint32_table_entry(vm, "num_entries", size); lua_pushinteger(vm, ++(*num)); lua_insert(vm, -2); lua_settable(vm, -3); } else { lua_newtable(vm); lua_push_uint32_table_entry(vm, "num_entries", size); lua_pushinteger(vm, ++(*num)); lua_insert(vm, -2); lua_settable(vm, -3); } } /* **************************************************** */ /* Analysis Flows Stats sorter function */ void NetworkInterface::sort_and_filter_flow_stats(lua_State *vm, struct aggregated_stats *stats, AnalysisCriteria filter_type) { std::vector vector; std::vector::iterator vector_it; char *sortColumn = NULL, *sortOrder = NULL, *search_string = NULL; u_int32_t start = 0, max_num_rows = 0; bool (*sorter)(AggregatedFlowsStats *, AggregatedFlowsStats *) = &asc_total_sent_cmp; bool is_asc; if (lua_type(vm, 3) == LUA_TSTRING) sortColumn = (char *)lua_tostring(vm, 3); if (lua_type(vm, 4) == LUA_TSTRING) sortOrder = (char *)lua_tostring(vm, 4); if (lua_type(vm, 5) == LUA_TNUMBER) start = (u_int32_t)lua_tonumber(vm, 5); if (lua_type(vm, 6) == LUA_TNUMBER) max_num_rows = (u_int32_t)lua_tonumber(vm, 6); if (lua_type(vm, 7) == LUA_TSTRING) search_string = (char *)lua_tostring(vm, 7); if(max_num_rows == 0) max_num_rows = 999; /* Set an upperbound */ is_asc = sortOrder ? (!strcmp(sortOrder, "asc")) : true; switch (filter_type) { case AnalysisCriteria::application_criteria: { /* Sorting by application criteria */ std::unordered_map::iterator it; for (it = stats->count.begin(); it != stats->count.end(); ++it) { bool do_add_it = false; ndpi_protocol detected_protocol; char buf[64], *proto; /* Get from the key, the master and application protocol, * first 16 bit for the master, second for the application */ detected_protocol.proto.master_protocol = (u_int16_t)(it->first & 0x00000000000FFFF); detected_protocol.proto.app_protocol = (u_int16_t)((it->first >> 16) & 0x000000000000FFFF); proto = get_ndpi_full_proto_name(detected_protocol, buf, sizeof(buf)); it->second->setProtoName(proto); if(search_string == NULL) do_add_it = true; else { // check filters if(strcasestr(proto, search_string) != NULL) do_add_it = true; } if(do_add_it) vector.push_back(it->second); } } break; case AnalysisCriteria::info_criteria: { std::unordered_map::iterator it; for (it = stats->info_count.begin(); it != stats->info_count.end(); ++it) { // check filters if((search_string == NULL) || (strcasestr(it->second->getInfoKey(), search_string) != NULL)) vector.push_back(it->second); } } break; default: { /* Client / Server / Client-Server / Client-Server-SrvPort */ std::unordered_map::iterator it; for (it = stats->count.begin(); it != stats->count.end(); ++it) { ndpi_protocol detected_protocol; char buf[64], *proto; /* Get from the key, the master and application protocol, * first 16 bit for the master, second for the application */ detected_protocol.proto.master_protocol = (u_int16_t)(it->second->getProtoKey() & 0x00000000000FFFF); detected_protocol.proto.app_protocol = (u_int16_t)((it->second->getProtoKey() >> 16) & 0x000000000000FFFF); it->second->setProtoName(proto = get_ndpi_full_proto_name(detected_protocol, buf, sizeof(buf))); if((search_string == NULL) || (strcasestr(proto, search_string) != NULL)) vector.push_back(it->second); } } break; } /* switch */ /* Choose the right sorter */ if (sortColumn) { if (!strcmp(sortColumn, "client")) sorter = &asc_cli_ip_hex_cmp; else if (!strcmp(sortColumn, "server")) sorter = &asc_srv_ip_hex_cmp; else if (!strcmp(sortColumn, "client_and_server")) sorter = &asc_srv_cli_ip_hex_cmp; else if (!strcmp(sortColumn, "srv_port")) sorter = &asc_srv_port_cmp; else if((!strcmp(sortColumn, "application"))) sorter = &asc_application_name_cmp; else if (!strcmp(sortColumn, "info")) sorter = &asc_str_info_cmp; else if (!strcmp(sortColumn, "flows")) sorter = &asc_flow_num_cmp; else if (!strcmp(sortColumn, "tot_score")) sorter = &asc_total_score_cmp; else if (!strcmp(sortColumn, "num_clients")) sorter = &asc_numclients_cmp; else if (!strcmp(sortColumn, "num_servers")) sorter = &asc_num_servers_cmp; else if (!strcmp(sortColumn, "bytes_sent")) sorter = &asc_total_sent_cmp; else if (!strcmp(sortColumn, "bytes_rcvd")) sorter = &asc_total_rcvd_cmp; else if (!strcmp(sortColumn, "tot_traffic")) sorter = &asc_total_traffic_cmp; else if (!strcmp(sortColumn, "vlan_id")) sorter = &asc_vlan_cmp; } std::sort(vector.begin(), vector.end(), sorter); /* Reverse order sort */ if (!is_asc) std::reverse(vector.begin(), vector.end()); const u_int32_t vector_size = vector.size(); u_int num = 0; lua_newtable(vm); /* Build up the lua response */ if (start < vector_size) { for (vector_it = std::next(vector.begin(), start); vector_it != vector.end(); ++vector_it) { AggregatedFlowsStats *fs = *vector_it; if (fs) { build_lua_rsp(vm, fs, filter_type, vector_size, &num, true); if(num >= max_num_rows) break; } } } else { build_lua_rsp(vm, NULL, filter_type, vector_size, &num, false); } } /* **************************************************** */ void NetworkInterface::getFilteredLiveFlowsStats(lua_State *vm) { u_int32_t begin_slot = 0; struct aggregated_stats stats; AnalysisCriteria filter_type = (AnalysisCriteria)lua_tonumber(vm, 1); char *host_ip = NULL; char *flow_device_ip = NULL; u_int16_t vlan_id = NO_VLAN /* Any VLAN */; u_int32_t in_if_idx = NO_IN_IF_INDEX /* Any inIfIdx */; u_int32_t out_if_idx = NO_OUT_IF_INDEX /* Any outIfIdx */; /* NOTE: parsing of additional Lua parameters in NetworkInterface::sort_and_filter_flow_stats() */ if (lua_type(vm, 8) == LUA_TSTRING) host_ip = (char *)lua_tostring(vm, 8); if (lua_type(vm, 9) == LUA_TNUMBER) vlan_id = lua_tonumber(vm,9); if (lua_type(vm, 10) == LUA_TSTRING) flow_device_ip = (char*)lua_tostring(vm,10); if (lua_type(vm, 11) == LUA_TNUMBER) in_if_idx = lua_tonumber(vm,11); if (lua_type(vm, 12) == LUA_TNUMBER) out_if_idx = lua_tonumber(vm,12); stats.vlan_id = vlan_id; stats.ip_addr = host_ip ? Utils::parseHostString(host_ip, &stats.vlan_id) : NULL; stats.flow_device_ip = flow_device_ip ? ntohl(inet_addr(flow_device_ip)) : /* Any flow device */ (u_int32_t)-1; stats.in_if_index = in_if_idx; stats.out_if_index = out_if_idx; switch (filter_type) { case AnalysisCriteria::application_criteria: /* application protocol criteria flows stats case */ walker(&begin_slot, true /* walk_all */, walker_flows, compute_protocol_flow_stats, &stats); break; case AnalysisCriteria::client_criteria: /* client criteria flows stats case */ walker(&begin_slot, true /* walk_all */, walker_flows, compute_client_flow_stats, &stats); break; case AnalysisCriteria::server_criteria: /* server criteria flows stats case */ walker(&begin_slot, true /* walk_all */, walker_flows, compute_server_flow_stats, &stats); break; case AnalysisCriteria::client_server_srv_port: /* client server server port criteria flows stats case */ walker(&begin_slot, true /* walk_all */, walker_flows, compute_client_server_srv_port_flow_stats, &stats); break; case AnalysisCriteria::client_server_srv_port_app_proto: /* client server server port app_proto criteria flows stats case */ walker(&begin_slot, true /* walk_all */, walker_flows, compute_client_server_srv_port_app_proto_flow_stats, &stats); break; #if defined(NTOPNG_PRO) case AnalysisCriteria::client_server_criteria: /* client server criteria flows stats case */ if (ntop->getPrefs()->is_enterprise_m_edition()) walker(&begin_slot, true /* walk_all */, walker_flows, compute_client_server_flow_stats, &stats); break; case AnalysisCriteria::app_client_server_criteria: /* app client server criteria flows stats case */ if (ntop->getPrefs()->is_enterprise_m_edition()) walker(&begin_slot, true /* walk_all */, walker_flows, compute_app_client_server_flow_stats, &stats); break; case AnalysisCriteria::info_criteria: /* info criteria flows stats case */ if (ntop->getPrefs()->is_enterprise_m_edition()) walker(&begin_slot, true /* walk_all */, walker_flows, compute_info_flow_stats, &stats); break; #endif default: /* client criteria flows stats case */ walker(&begin_slot, true /* walk_all */, walker_flows, compute_client_flow_stats, &stats); break; } sort_and_filter_flow_stats(vm, &stats, filter_type); /* Free memory before leaving */ for (std::unordered_map::iterator it1 = stats.count.begin(); it1 != stats.count.end(); ++it1) delete it1->second; for (std::unordered_map::iterator it2 = stats.info_count.begin(); it2 != stats.info_count.end(); ++it2) delete it2->second; } /* **************************************************** */ void NetworkInterface::getHostsPorts(lua_State *vm) { u_int32_t begin_slot = 0; HostsPorts count; u_int8_t l4_protocol = 0; u_int16_t vlan_id = /* ANY VLAN */(u_int16_t)-1; if (lua_type(vm, 1) == LUA_TNUMBER) { l4_protocol = (u_int8_t)lua_tonumber(vm, 1); count.set_protocol(l4_protocol); } if (lua_type(vm, 2) == LUA_TNUMBER) { vlan_id = (u_int16_t)lua_tonumber(vm, 2); count.set_vlan_id(vlan_id); } walker(&begin_slot, true , walker_flows, get_host_ports, &count); lua_push_ports(vm, &count, l4_protocol); /* Free memory before leaving */ std::unordered_map server_ports = count.get_srv_ports(); std::unordered_map::iterator it; for ( it = server_ports.begin(); it != server_ports.end(); ++it) { delete it->second; } } /* **************************************************** */ bool NetworkInterface::get_host_ports(GenericHashEntry *node, void *user_data, bool *matched) { /* Retrieve HostsPorts instance */ HostsPorts* hostsPorts = static_cast< HostsPorts *>(user_data); if (!hostsPorts) return (false); /* false = keep on walking */ /* Retrieve flow instance */ Flow* f = (Flow *)node; if (!f) return (false); /* false = keep on walking */ /* Retrieve the server host instance */ IpAddress* server = f->get_srv_ip_addr(); /* Filters */ u_int16_t filter_vlan_id = hostsPorts->get_vlan_id(); u_int8_t filter_l4_proto = hostsPorts->get_protocol(); /* Flow's attributes */ u_int16_t flow_vlan_id = f->get_vlan_id(); u_int8_t flow_l4_proto = f->get_protocol(); u_int16_t srv_port = f->get_srv_port(); ndpi_protocol detected_protocol = f->get_detected_protocol(); /* check l4_proto filter */ if (flow_l4_proto != filter_l4_proto) return (false); /* false = keep on walking */ /* check vlan filter */ if (filter_vlan_id != NO_VLAN) { /* check flow vlan */ if (flow_vlan_id != filter_vlan_id) return (false); /* false = keep on walking */ } /* */ u_int64_t host_key = (((u_int64_t)server->key()) << 16) + ((u_int64_t)flow_vlan_id); /* Add the server port + the host on the hosts u_map*/ hostsPorts->add_srv_port(port_proto_key, host_key); *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ void NetworkInterface::lua_push_ports(lua_State *vm, HostsPorts *count, u_int16_t protocol) { std::unordered_map port_list = count->get_srv_ports(); std::unordered_map::iterator it; /* Build the lua response */ lua_newtable(vm); u_int16_t size = port_list.size(); u_int num = 0; for (it = port_list.begin(); it != port_list.end(); ++it) { char buf[64], proto[16]; lua_newtable(vm); u_int64_t key = it->first; ndpi_protocol detected_protocol; detected_protocol.proto.master_protocol = (u_int16_t)(key & 0x00000000000FFFF); detected_protocol.proto.app_protocol = (u_int16_t)((key >> 16) & 0x000000000000FFFF); u_int16_t srv_port = (u_int64_t)((key >> 32) & 0x000000000000FFFF); if (detected_protocol.proto.master_protocol == detected_protocol.proto.app_protocol) snprintf(proto, sizeof(proto), "%u", detected_protocol.proto.master_protocol); else if (detected_protocol.proto.app_protocol == NDPI_PROTOCOL_UNKNOWN) snprintf(proto, sizeof(proto), "%u", detected_protocol.proto.master_protocol); else if (detected_protocol.proto.master_protocol == NDPI_PROTOCOL_UNKNOWN) snprintf(proto, sizeof(proto), "%u", detected_protocol.proto.app_protocol); else snprintf(proto, sizeof(proto), "%u.%u", detected_protocol.proto.master_protocol, detected_protocol.proto.app_protocol); lua_push_str_table_entry(vm, "proto_id", proto); lua_push_str_table_entry(vm, "l7_proto_name", get_ndpi_full_proto_name(detected_protocol, buf, sizeof(buf))); lua_push_uint32_table_entry(vm, "srv_port", srv_port); lua_push_str_table_entry(vm, "l4_proto", Utils::l4proto2name(count->get_protocol())); lua_push_uint64_table_entry(vm, "n_hosts", it->second->get_size()); lua_push_uint64_table_entry(vm, "num_entries", (u_int64_t)size); lua_pushinteger(vm, ++num); lua_insert(vm, -2); lua_settable(vm, -3); } } /* **************************************************** */ bool NetworkInterface::get_hosts_by_port(GenericHashEntry *node, void *user_data, bool *matched) { /* Retrieve HostsPortsAnalysis instance */ HostsPortsAnalysis *hostsPortsAnalysis = static_cast< HostsPortsAnalysis *>(user_data); if ( hostsPortsAnalysis == NULL ) return(false); /* Retrieve flow instance */ Flow *f = (Flow *)node; if (!f) return(false); /* Retrieve the server host instance */ Host* h = f->get_srv_host(); if (!h) h = f->getViewSharedServer(); if (!h) return(false); /* Filters */ u_int8_t filter_l4_proto = hostsPortsAnalysis->get_l4_proto(); u_int16_t filter_srv_port = hostsPortsAnalysis->get_port(); int l7_app_protocol = hostsPortsAnalysis->get_l7_app_proto(); int l7_master_protocol = hostsPortsAnalysis->get_l7_proto(); /* Check filters values */ if ((filter_l4_proto == 0) || (filter_srv_port == 0)) return(false); /* Check filters with flow's values (L4 Protocol & Dst Port)*/ if ((filter_l4_proto != f->get_protocol()) || (filter_srv_port != f->get_srv_port())) return(false); /* Check L7 filters */ ndpi_protocol detected_protocol = f->get_detected_protocol(); bool check_both = false; if (l7_app_protocol != NDPI_PROTOCOL_UNKNOWN) check_both = true; if (detected_protocol.proto.master_protocol == detected_protocol.proto.app_protocol || detected_protocol.proto.app_protocol == NDPI_PROTOCOL_UNKNOWN){ /* CASE master == app OR app == UNKNOWN */ if (check_both) return false; /* there's a specific app protocol filter */ else if (!check_both && l7_master_protocol != detected_protocol.proto.master_protocol) return false; } else if (detected_protocol.proto.master_protocol == NDPI_PROTOCOL_UNKNOWN) { /* CASE master == UNKNOWN */ if (check_both) return false; /* there's a specific app protocol filter */ else if (!check_both && l7_master_protocol != detected_protocol.proto.app_protocol) return false; } else { /* CASE BOTH protocols in Flow */ if (check_both) { /* there's a specific app protocol filter */ if (l7_master_protocol == detected_protocol.proto.master_protocol && l7_app_protocol != detected_protocol.proto.app_protocol) { /* Same master protocols different app protocols */ return false; } } else { /* Just MASTER protocol filter but Flow has both */ return false; } } /* Retrieve server host VLAN and host_key */ u_int64_t vlan_id = (u_int64_t) f->get_vlan_id(); u_int64_t host_key = (((u_int64_t)h->key()) << 16) ; host_key += ((u_int64_t)vlan_id); /* Retrieve host_details hash */ std::unordered_map *host_details = hostsPortsAnalysis->get_hosts_details(); std::unordered_map::iterator it; if (host_details) { /* Search the server host in hash */ it = host_details->find(host_key); if(it == host_details->end()) { /* Host not found in hash */ char ip_buf[64]; char mac_buf[64]; char ip_hex_buf[64]; char name[64]; u_int32_t flows_count = 1; /* Build new host_info instance */ HostDetails *host_info = new (std::nothrow) HostDetails( f->get_srv_ip_addr()->print(ip_buf, sizeof(ip_buf)), h->getMac() ? h->getMac()->print(mac_buf, sizeof(mac_buf)) : (char*)"", h->getMac() ? (char*)h->getMac()->get_manufacturer() : (char*)"", f->get_bytes_cli2srv() + f->get_bytes_srv2cli(), f->get_srv_ip_addr()->get_ip_hex(ip_hex_buf, sizeof(ip_hex_buf)), vlan_id, h->getScore(), flows_count, h->get_name(name, sizeof(name), false), host_key); if(host_info) { host_details->insert({host_key, host_info}); } } else { /* Found host in hash -> update stats */ it->second->inc_stats(f->get_bytes_cli2srv() + f->get_bytes_srv2cli()); } } *matched = true; return (false); } /* **************************************************** */ void NetworkInterface::getHostsByPort(lua_State *vm) { u_int32_t begin_slot = 0; u_int8_t protocol = 0; u_int16_t port = 0; int app_protocol = 0; int master_protocol = 0; HostsPortsAnalysis count; if (lua_type(vm, 1) == LUA_TNUMBER) protocol = (u_int8_t)lua_tonumber(vm, 1); count.set_l4_proto(protocol); if (lua_type(vm, 2) == LUA_TNUMBER) { port = (u_int16_t)lua_tonumber(vm, 2); count.set_port(port); } if (lua_type(vm, 3) == LUA_TNUMBER) { app_protocol = lua_tonumber(vm, 3); count.set_l7_app_proto(app_protocol); } if (lua_type(vm, 8) == LUA_TNUMBER) { master_protocol = lua_tonumber(vm, 8); count.set_l7_proto(master_protocol); } walker(&begin_slot, true , walker_flows, get_hosts_by_port, &count); sort_hosts_details(vm, &count, protocol, false); } /* **************************************************** */ bool NetworkInterface::get_hosts_by_service(GenericHashEntry *node, void *user_data, bool *matched) { Host *h = (Host *)node; if ( (!h) || (!h->isLocalHost()) ) return (false); HostsPortsAnalysis *hostsPortsAnalysis = static_cast< HostsPortsAnalysis *>(user_data); int l7_proto = hostsPortsAnalysis->get_l7_proto(); u_int8_t l4_proto = hostsPortsAnalysis->get_l4_proto(); if ((l7_proto == 0) || (l4_proto == 0)) { return (false); } LocalHost *lh = (LocalHost*) h; std::unordered_map *ports_list = l4_proto == 6 ? lh->getTCPServerPorts() : lh->getUDPServerPorts(); std::unordered_map::iterator port_finder; u_int16_t port_found = 0; port_finder = ports_list->begin(); while (port_finder != ports_list->end() && port_found == 0) { if(port_finder->second.proto.app_protocol == l7_proto) { port_found = port_finder->first; } ++port_finder; } if (port_found != 0) { u_int64_t vlan_id = (u_int64_t) h->get_vlan_id(); u_int64_t host_key = (((u_int64_t)h->get_ip()->key()) << 16) + ((u_int64_t)vlan_id); std::unordered_map *host_details = hostsPortsAnalysis->get_hosts_details(); if (host_details && host_details->find(host_key) == host_details->end() ) { /* Not found in hash the host details */ char ip_buf[64]; char mac_buf[64]; char ip_hex_buf[64]; char name[64]; HostDetails *host_info = new (std::nothrow) HostDetails(h->get_ip()->print(ip_buf, sizeof(ip_buf)), h->getMac() ? h->getMac()->print(mac_buf, sizeof(mac_buf)) : (char*)"", h->getMac() ? (char*)h->getMac()->get_manufacturer() : (char*)"", h->getNumBytesUDPSent() + h->getNumBytesUDPRcvd(), h->get_ip()->get_ip_hex(ip_hex_buf, sizeof(ip_hex_buf)), h->get_vlan_id(), h->getScore(), h->getNumIncomingFlows(), h->get_name(name, sizeof(name), false), host_key); if(host_info) { host_info->set_port(port_found); host_details->insert({host_key, host_info}); } } } *matched = true; return (false); } /* **************************************************** */ void NetworkInterface::getHostsByService(lua_State *vm) { int l7_proto = 0; u_int8_t protocol = 0; u_int32_t begin_slot = 0; HostsPortsAnalysis count; if (lua_type(vm, 2) == LUA_TNUMBER) { l7_proto = (u_int32_t)lua_tonumber(vm, 2); count.set_l7_proto(l7_proto); } if (lua_type(vm,3) == LUA_TNUMBER) { protocol = (u_int32_t)lua_tonumber(vm, 3); count.set_l4_proto(protocol); } walker(&begin_slot, true , walker_hosts, get_hosts_by_service, &count); sort_hosts_details(vm, &count, protocol, true); } /* **************************************************** */ void NetworkInterface::sort_hosts_details(lua_State *vm, HostsPortsAnalysis *count, u_int16_t protocol, bool get_port) { std::vector vector; char *sortColumn = NULL, *sortOrder = NULL; u_int32_t start = 0, max_num_rows = 0; if (lua_type(vm, 4) == LUA_TSTRING) sortColumn = (char *)lua_tostring(vm, 4); if (lua_type(vm, 5) == LUA_TSTRING) sortOrder = (char *)lua_tostring(vm, 5); if (lua_type(vm, 6) == LUA_TNUMBER) start = (u_int32_t)lua_tonumber(vm, 6); if (lua_type(vm, 7) == LUA_TNUMBER) max_num_rows = (u_int32_t)lua_tonumber(vm, 7); bool is_asc = sortOrder ? (!strcmp(sortOrder, "asc")) : true; bool (*sorter)(HostDetails *, HostDetails *) = &host_details_asc_ip_cmp; if (sortColumn) { if (!strcmp(sortColumn, "flows")) { sorter = &host_details_asc_flows_cmp; } else if (!strcmp(sortColumn, "score")) { sorter = &host_details_asc_score_cmp; } else if (!strcmp(sortColumn, "mac")) { sorter = &host_details_asc_mac_cmp; } else if (!strcmp(sortColumn, "mac_manufacturer")) { sorter = &host_details_asc_mac_manufacturer_cmp; } else if (!strcmp(sortColumn, "ip")) { sorter = &host_details_asc_ip_cmp; } else if (!strcmp(sortColumn, "tot_traffic")) { sorter = &host_details_asc_total_traffic_cmp; } else if (!strcmp(sortColumn, "vlan_id")) { sorter = &host_details_asc_vlan_cmp; } else if (!strcmp(sortColumn, "name")) { sorter = &host_details_asc_name_cmp; } } std::unordered_map *hosts_map = count->get_hosts_details(); std::unordered_map::iterator it; for (it = hosts_map->begin(); it != hosts_map->end(); ++it) { vector.push_back(it->second); } std::sort(vector.begin(), vector.end(), sorter); if (!is_asc) std::reverse(vector.begin(), vector.end()); const u_int32_t vector_size = vector.size(); u_int num = 1; lua_newtable(vm); if (start < vector_size) { for (std::vector::iterator vector_it = std::next(vector.begin(), start); vector_it != vector.end(); ++vector_it) { HostDetails *hd = *vector_it; if (hd) { lua_newtable(vm); lua_push_str_table_entry(vm, "ip", hd->get_ip()); lua_push_str_table_entry(vm, "mac", hd->get_mac_address()); lua_push_str_table_entry(vm, "mac_manufacturer", hd->get_mac_manufacturer()); lua_push_str_table_entry(vm, "name", hd->get_name()); lua_push_uint64_table_entry(vm, "vlan_id", (u_int64_t)hd->get_vlan_id()); lua_push_uint64_table_entry(vm, "flows", (u_int64_t)hd->get_active_flows_as_server()); lua_push_uint64_table_entry(vm, "score", (u_int64_t)hd->get_score()); lua_push_uint64_table_entry(vm, "tot_traffic", (u_int64_t)hd->get_total_traffic()); lua_push_uint32_table_entry(vm, "num_entries", vector_size); if (get_port) { lua_push_uint32_table_entry(vm, "port", (u_int64_t)hd->get_port()); } lua_pushinteger(vm, num++); lua_insert(vm, -2); lua_settable(vm, -3); } if ( (!get_port) && ((num-1) > max_num_rows) ) break; } } else { lua_newtable(vm); lua_push_uint32_table_entry(vm, "num_entries", vector_size); lua_pushinteger(vm, num++); lua_insert(vm, -2); lua_settable(vm, -3); } } /* **************************************************** */ static bool compute_vlan_flow_stats(GenericHashEntry *node, void *user_data, bool *matched) { Flow *f = (Flow *)node; if (!f) return false; ndpi_protocol detected_protocol = f->get_detected_protocol(); u_int64_t vlan_id = f->get_vlan_id(); /* <0 (16 bit)>get_srv_port()) << 48) + (((u_int64_t)vlan_id) << 32) + (((u_int64_t)detected_protocol.proto.app_protocol) << 16) + (u_int64_t)detected_protocol.proto.master_protocol; std::unordered_map::iterator it; std::unordered_map *count = static_cast *>(user_data); it = count->find(key); if (it == count->end()) { AggregatedFlowsStats *fs = new (std::nothrow) AggregatedFlowsStats(f->get_cli_ip_addr(), f->get_srv_ip_addr(), f->get_protocol(), f->get_bytes_cli2srv(), f->get_bytes_srv2cli(), f->getScore()); if (fs != NULL) (*count)[key] = fs; } else { it->second->incFlowStats(f->get_cli_ip_addr(), f->get_srv_ip_addr(), f->get_bytes_cli2srv(), f->get_bytes_srv2cli(), f->getScore()); } *matched = true; return (false); /* false = keep on walking */ } /* **************************************************** */ void NetworkInterface::getVLANFlowsStats(lua_State *vm) { u_int32_t begin_slot = 0; std::unordered_map count; std::unordered_map::iterator it; u_int num = 0; walker(&begin_slot, true /* walk_all */, walker_flows, compute_vlan_flow_stats, &count); lua_newtable(vm); for (it = count.begin(); it != count.end(); ++it) { AggregatedFlowsStats *fs = it->second; ndpi_protocol detected_protocol; char buf[64], proto[16]; u_int16_t vlan_id, dst_port; if (fs) { detected_protocol.proto.master_protocol = (u_int16_t)(it->first & 0x00000000000FFFF); detected_protocol.proto.app_protocol = (u_int16_t)((it->first >> 16) & 0x000000000000FFFF); vlan_id = (u_int16_t)((it->first >> 32) & 0x000000000000FFFF); dst_port = (u_int16_t)((it->first >> 48) & 0x000000000000FFFF); lua_newtable(vm); if (detected_protocol.proto.master_protocol == detected_protocol.proto.app_protocol) snprintf(proto, sizeof(proto), "%u", detected_protocol.proto.master_protocol); else if (detected_protocol.proto.app_protocol == NDPI_PROTOCOL_UNKNOWN) snprintf(proto, sizeof(proto), "%u", detected_protocol.proto.master_protocol); else if (detected_protocol.proto.master_protocol == NDPI_PROTOCOL_UNKNOWN) snprintf(proto, sizeof(proto), "%u", detected_protocol.proto.app_protocol); else snprintf(proto, sizeof(proto), "%u.%u", detected_protocol.proto.master_protocol, detected_protocol.proto.app_protocol); lua_push_uint32_table_entry(vm, "vlan_id", vlan_id); lua_push_uint32_table_entry(vm, "dst_port", dst_port); lua_push_str_table_entry(vm, "proto_id", proto); lua_push_str_table_entry( vm, "proto_name", get_ndpi_full_proto_name(detected_protocol, buf, sizeof(buf))); lua_push_uint32_table_entry(vm, "l4_proto", fs->getL4Protocol()); lua_push_uint32_table_entry(vm, "num_clients", fs->getNumClients()); lua_push_uint32_table_entry(vm, "num_servers", fs->getNumServers()); lua_push_uint32_table_entry(vm, "num_flows", fs->getNumFlows()); lua_push_uint64_table_entry(vm, "bytes_sent", fs->getTotalSent()); lua_push_uint64_table_entry(vm, "bytes_rcvd", fs->getTotalRcvd()); lua_push_uint64_table_entry(vm, "total_score", fs->getTotalScore()); lua_pushinteger(vm, ++num); lua_insert(vm, -2); lua_settable(vm, -3); delete it->second; } } } /* **************************************************** */ struct walk_no_tx_hosts_info { lua_State *vm; bool local_host_rx_only, list_host_peers; std::unordered_map hosts; NetworkInterface *iface; }; /* **************************************************** */ static bool setHostNoTXInfo(lua_State *vm, Host *h) { u_int32_t a = h->getNumContactedPeersAsClientTCPUDPNoTX(); u_int32_t b = h->getNumContactsFromPeersAsServerTCPUDPNoTX(); if ((a > 0) || (b > 0)) { IpAddress *i = h->get_ip(); char buf[64], *label = i->print(buf, sizeof(buf)); lua_newtable(vm); lua_push_uint32_table_entry(vm, "tcp_num_contacted_peers_as_client", a); lua_push_uint32_table_entry(vm, "tcp_num_peers_contacts_rcvd_as_server", b); lua_push_uint32_table_entry(vm, "tcp_num_ports_contacted_as_client", h->getNumContactedTCPUDPServerPortsNoTX()); lua_pushstring(vm, label); lua_insert(vm, -2); lua_settable(vm, -3); return (true); } else return (false); } /* **************************************************** */ static bool walk_no_tx_hosts(GenericHashEntry *node, void *user_data, bool *matched) { Host *h = (Host *)node; if (!(h->isBroadcastHost() || h->isMulticastHost())) { struct walk_no_tx_hosts_info *hosts = static_cast(user_data); bool good = false; if (hosts->local_host_rx_only) { /* retrieve local host (server) with no TX traffic */ if (h->isLocalHost()) good = true; } else { /* retrieve remote host (server) with no TX traffic */ if (!h->isLocalHost()) good = true; } if (good) setHostNoTXInfo(hosts->vm, h); } return (false); /* false = keep on walking */ } /* **************************************************** */ static bool walk_no_tx_host_flows(GenericHashEntry *node, void *user_data, bool *matched) { Flow *f = (Flow *)node; struct walk_no_tx_hosts_info *hosts = static_cast(user_data); Host *c = f->get_cli_host(), *s = f->get_srv_host(); /* View interfaces need this extra step */ if (c == NULL) c = hosts->iface->getHostByIP((IpAddress *)f->get_cli_ip_addr(), f->get_vlan_id(), f->getFlowObservationPointId(), false); if (s == NULL) s = hosts->iface->getHostByIP((IpAddress *)f->get_srv_ip_addr(), f->get_vlan_id(), f->getFlowObservationPointId(), false); if (c && s && s->isRxOnlyHost()) { if (!(s->isBroadcastHost() || s->isMulticastHost())) { bool good = false; if (hosts->local_host_rx_only) { /* retrieve local host (server) with no TX traffic */ if (c->isLocalHost()) good = true; } else { /* retrieve remote host (server) with no TX traffic */ if (!c->isLocalHost()) good = true; } if (good) { IpAddress *i = c->get_ip(); char buf[64], *what = i->print(buf, sizeof(buf)); std::string name(what); /* ntop->getTrace()->traceEvent(TRACE_NORMAL, "Checking %s", * name.c_str()); */ if (hosts->hosts.find(name) == hosts->hosts.end()) { if (setHostNoTXInfo(hosts->vm, c)) hosts->hosts[name] = true; /* Used to avoid duplicates */ } } } } return (false); /* false = keep on walking */ } /* **************************************************** */ /* local_host_rx_only - true: retrieve local host (server) with no TX traffic - false: retrieve remote host (server) with no TX traffic list_host_peers - true: retrieve the peers talking with the hosts with no TX traffic - false: retrieve the host with no TX traffic */ void NetworkInterface::getRxOnlyHostsList(lua_State *vm, bool local_host_rx_only, bool list_host_peers) { u_int32_t begin_slot = 0; struct walk_no_tx_hosts_info hosts; lua_newtable(vm); hosts.iface = this; if (list_host_peers) { hosts.local_host_rx_only = local_host_rx_only, hosts.list_host_peers = list_host_peers, hosts.vm = vm; walker(&begin_slot, true /* walk_all */, walker_flows, walk_no_tx_host_flows, &hosts); } else { hosts.local_host_rx_only = local_host_rx_only, hosts.list_host_peers = list_host_peers, hosts.vm = vm; walker(&begin_slot, true /* walk_all */, walker_hosts, walk_no_tx_hosts, &hosts); } } /* **************************************************** */ struct LuaArrayEntryStats { lua_State *vm; u_int num; }; static bool active_mac_search_walker(GenericHashEntry *he, void *user_data, bool *matched) { struct LuaArrayEntryStats *s = (struct LuaArrayEntryStats *)user_data; Mac *m = (Mac *)he; char buf[32]; lua_pushstring(s->vm, m->print(buf, sizeof(buf))); lua_rawseti(s->vm, -2, s->num + 1); /* Array */ s->num++; return (false); /* false = keep on walking */ } /* **************************************************** */ void NetworkInterface::getActiveMacs(lua_State *vm) { struct LuaArrayEntryStats s; u_int32_t begin_slot = 0; lua_newtable(vm); s.vm = vm, s.num = 0; walker(&begin_slot, true /* walk_all */, walker_macs, active_mac_search_walker, (void *)&s); } /* **************************************************** */ void NetworkInterface::getSFlowDevices(lua_State *vm, bool add_table) { /* Add the devices list only if not empty */ if (interfaceStats) { if(add_table) lua_newtable(vm); interfaceStats->luaDeviceList(vm); lua_pushinteger(vm, get_id()); lua_insert(vm, -2); lua_settable(vm, -3); } }; /* **************************************************** */ void NetworkInterface::incnDPIStats(time_t when, u_int16_t ndpi_proto, ndpi_protocol_category_t ndpi_category, u_int32_t bytes_sent, u_int32_t bytes_rcvd, u_int32_t pkts_sent, u_int32_t pkts_rcvd) { ndpiStats->incStats(when, ndpi_proto, pkts_sent, bytes_sent, pkts_rcvd, bytes_rcvd); ndpiStats->incCategoryStats(when, ndpi_category, bytes_sent, bytes_rcvd); } /* **************************************************** */ void NetworkInterface::resetBroacastDomains() { if(bcast_domains != NULL) bcast_domains->reset(); }