diff --git a/include/Flow.h b/include/Flow.h index 6c813afa61..caa16bf217 100644 --- a/include/Flow.h +++ b/include/Flow.h @@ -294,7 +294,9 @@ class Flow : public GenericHashEntry { void setNormalToAlertedCounters(); /* Decreases scores on both client and server hosts when the flow is being destructed */ void decAllFlowScores(); - + void updateServerPortsStats(Host *server_host, ndpi_protocol *proto); + void updateClientContactedPorts(Host *client, ndpi_protocol *proto); + public: Flow(NetworkInterface *_iface, VLANid _vlanId, u_int16_t _observation_point_id, diff --git a/include/Host.h b/include/Host.h index 4f25a4fe4a..d4236aeba7 100644 --- a/include/Host.h +++ b/include/Host.h @@ -608,10 +608,10 @@ class Host : public GenericHashEntry, public HostAlertableEntity, public Score, inline void incUnidirectionalIngressFlows() { unidirectionalTCPFlows.numIngressFlows++; } inline void incUnidirectionalEgressFlows() { unidirectionalTCPFlows.numEgressFlows++; } - virtual void setServerPort(bool isTCP, u_int16_t port) { ; }; - virtual void setContactedPort(bool isTCP, u_int16_t port) { ; }; + virtual void setServerPort(bool isTCP, u_int16_t port, ndpi_protocol *proto) { ; }; + virtual void setContactedPort(bool isTCP, u_int16_t port, ndpi_protocol *proto) { ; }; virtual void luaUsedPorts(lua_State* vm) { ; } - virtual ndpi_bitmap* getServerPorts() { return(NULL); } + virtual std::unordered_map* getServerPorts(bool isTCP) { return(NULL); } }; #endif /* _HOST_H_ */ diff --git a/include/HostPorts.h b/include/HostPorts.h index 625ed0dcb2..13b7dce66c 100644 --- a/include/HostPorts.h +++ b/include/HostPorts.h @@ -27,21 +27,24 @@ class HostPorts { private: /* Used for both TCP and UDP */ - ndpi_bitmap *host_server_ports, *contacted_ports; + std::unordered_map udp_host_server_ports, tcp_host_server_ports; + std::unordered_map udp_client_contacted_ports, tcp_client_contacted_ports; + + void setLuaArray(lua_State *vm, NetworkInterface *iface, bool isTCP, + std::unordered_map *ports); - void setLuaArray(lua_State *vm, ndpi_bitmap *ports, const char *label); - public: HostPorts(); ~HostPorts(); void reset(); - - void lua(lua_State *vm); - void setServerPort(bool isTCP, u_int16_t port); - void setContactedPort(bool isTCP, u_int16_t port); - inline ndpi_bitmap* getServerPorts() { return(host_server_ports); } + void lua(lua_State *vm, NetworkInterface *iface); + + void setServerPort(bool isTCP, u_int16_t port, ndpi_protocol *proto); + void setContactedPort(bool isTCP, u_int16_t port, ndpi_protocol *proto); + + std::unordered_map* getServerPorts(bool isTCP) { return(isTCP ? &tcp_host_server_ports : &udp_host_server_ports); } }; #endif /* _HOST_PORTS_H_ */ diff --git a/include/LocalHost.h b/include/LocalHost.h index 23ecb597fc..80460c7da7 100644 --- a/include/LocalHost.h +++ b/include/LocalHost.h @@ -117,10 +117,10 @@ class LocalHost : public Host, public SerializableElement { void setRouterMac(Mac *gw); - inline void setServerPort(bool isTCP, u_int16_t port) { usedPorts.setServerPort(isTCP, port); }; - inline void setContactedPort(bool isTCP, u_int16_t port) { usedPorts.setContactedPort(isTCP, port); }; - virtual void luaUsedPorts(lua_State* vm) { usedPorts.lua(vm); } - virtual ndpi_bitmap* getServerPorts() { return(usedPorts.getServerPorts()); } + inline void setServerPort(bool isTCP, u_int16_t port, ndpi_protocol *proto) { usedPorts.setServerPort(isTCP, port, proto); }; + inline void setContactedPort(bool isTCP, u_int16_t port, ndpi_protocol *proto) { usedPorts.setContactedPort(isTCP, port, proto); }; + virtual void luaUsedPorts(lua_State* vm) { usedPorts.lua(vm, iface); }; + virtual std::unordered_map* getServerPorts(bool isTCP) { return(usedPorts.getServerPorts(isTCP)); }; }; #endif /* _LOCAL_HOST_H_ */ diff --git a/scripts/lua/host_details.lua b/scripts/lua/host_details.lua index 21f4d6cfa3..e484c134f3 100644 --- a/scripts/lua/host_details.lua +++ b/scripts/lua/host_details.lua @@ -112,7 +112,9 @@ local function printPorts(ports) print(i18n("none")) else for k,v in pairs(ports) do - print(''.. v .." ") + local res = split(k, ":") + + print(''.. k.. " (" .. v ..")" .." ") end end end diff --git a/src/Flow.cpp b/src/Flow.cpp index 22eb359d0b..ac26afb85a 100644 --- a/src/Flow.cpp +++ b/src/Flow.cpp @@ -141,15 +141,6 @@ Flow::Flow(NetworkInterface *_iface, lh->setRouterMac(_srv_mac); } } - - if(cli_host->isLocalHost()) { - switch(protocol) { - case IPPROTO_TCP: - case IPPROTO_UDP: - cli_host->setContactedPort((protocol == IPPROTO_TCP), ntohs(srv_port)); - break; - } - } } else { /* Client host has not been allocated, let's keep the info in an IpAddress */ @@ -1133,8 +1124,10 @@ void Flow::updateProtocol(ndpi_protocol proto_id) { /* *************************************** */ -/* Called to update the flow protocol and possibly advance the flow to - * the protocol_detected state. */ +/* + * Called to update the flow protocol and possibly advance the flow to + * the protocol_detected state. + */ void Flow::setProtocolDetectionCompleted(u_int8_t *payload, u_int16_t payload_len) { if(detection_completed) return; @@ -1149,6 +1142,15 @@ void Flow::setProtocolDetectionCompleted(u_int8_t *payload, u_int16_t payload_le detection_completed = 1; + if(cli_host && srv_host) { + /* + Hosts are NULL for view interfaces and thus they are updated + in Flow::hosts_periodic_stats_update() + */ + updateServerPortsStats(srv_host, &ndpiDetectedProtocol); + updateClientContactedPorts(cli_host, &ndpiDetectedProtocol); + } + #ifdef BLACKLISTED_FLOWS_DEBUG if(ndpiDetectedProtocol.category == CUSTOM_CATEGORY_MALWARE) { char buf[512]; @@ -1524,10 +1526,13 @@ void Flow::incFlowDroppedCounters() { * * const is *required* here as the flow must not be modified (as it could go in concuncurrency * with the subinterfaces). */ + + void Flow::hosts_periodic_stats_update(NetworkInterface *iface, Host *cli_host, Host *srv_host, PartializableFlowTrafficStats *partial, bool first_partial, const struct timeval *tv) { - update_pools_stats(iface, cli_host, srv_host, tv, partial->get_cli2srv_packets(), partial->get_cli2srv_bytes(), + update_pools_stats(iface, cli_host, srv_host, tv, + partial->get_cli2srv_packets(), partial->get_cli2srv_bytes(), partial->get_srv2cli_packets(), partial->get_srv2cli_bytes()); if(cli_host && srv_host) { @@ -1539,6 +1544,9 @@ void Flow::hosts_periodic_stats_update(NetworkInterface *iface, Host *cli_host, int16_t stats_protocol = getStatsProtocol(); /* The protocol (among ndpi master_ and app_) that is chosen to increase stats */ NetworkStats *cli_network_stats = NULL, *srv_network_stats = NULL; + updateServerPortsStats(srv_host, &ndpiDetectedProtocol); + updateClientContactedPorts(cli_host, &ndpiDetectedProtocol); + if(cli_network_id >= 0 && (cli_network_id == srv_network_id)) cli_and_srv_in_same_subnet = true; @@ -3904,6 +3912,68 @@ bool Flow::enqueueAlertToRecipients(FlowAlert *alert) { /* *************************************** */ +/* Update server -> client in case no traffic was already observed on the server side */ + +void Flow::updateServerPortsStats(Host *server, ndpi_protocol *proto) { + /* + NOTE use the method parameter 'server' instead of + srv_host as with view interfaces it can be invalid + */ + + if(get_bytes_srv2cli() == 0) return; + + if(server + && server->isLocalHost() + // && (get_bytes_srv2cli() == 0) + ) { + switch(protocol) { + case IPPROTO_TCP: + if((src2dst_tcp_flags & TH_SYN) == TH_SYN) + server->setServerPort(true, ntohs(srv_port), proto); + break; + + case IPPROTO_UDP: + { + u_int16_t c_port = ntohs(cli_port); + u_int16_t s_port = ntohs(srv_port); + + if(c_port > s_port) /* minimal check, to improve */ + server->setServerPort(false, s_port, proto); + break; + } + } + } +} + +/* *************************************** */ + +void Flow::updateClientContactedPorts(Host *client, ndpi_protocol *proto) { + /* + NOTE use the method parameter 'client' instead of + cli_host as with view interfaces it can be invalid + */ + + if(client->isLocalHost()) { + switch(protocol) { + case IPPROTO_TCP: + if((src2dst_tcp_flags & TH_SYN) == TH_SYN) + client->setContactedPort((protocol == IPPROTO_TCP), ntohs(srv_port), proto); + break; + + case IPPROTO_UDP: + { + u_int16_t c_port = ntohs(cli_port); + u_int16_t s_port = ntohs(srv_port); + + if(c_port > s_port) /* minimal check, to improve */ + client->setContactedPort(false, s_port, proto); + } + break; + } + } +} +/* *************************************** */ + void Flow::incStats(bool cli2srv_direction, u_int pkt_len, u_int8_t *payload, u_int payload_len, u_int8_t l4_proto, u_int8_t is_fragment, @@ -3911,19 +3981,6 @@ void Flow::incStats(bool cli2srv_direction, u_int pkt_len, u_int16_t fragment_extra_overhead) { bool update_iat = true; - /* Updated server -> client in case no traffic was already observed on the server side */ - if(srv_host - && srv_host->isLocalHost() - && (cli2srv_direction == false) - && (get_bytes_srv2cli() == 0)) { - switch(protocol) { - case IPPROTO_TCP: - case IPPROTO_UDP: - srv_host->setServerPort((protocol == IPPROTO_TCP), ntohs(srv_port)); - break; - } - } - payload_len *= iface->getScalingFactor(); updateSeen(); @@ -4018,18 +4075,6 @@ void Flow::addFlowStats(bool new_flow, updateSeen(last_seen); callFlowUpdate(last_seen); - if(srv_host - && srv_host->isLocalHost() - && (get_bytes_srv2cli() == 0) - && (out_bytes > 0)) { - switch(protocol) { - case IPPROTO_TCP: - case IPPROTO_UDP: - srv_host->setServerPort((protocol == IPPROTO_TCP), ntohs(srv_port)); - break; - } - } - if(cli2srv_direction) { stats.incStats(true, in_pkts, in_bytes, in_goodput_bytes); stats.incStats(false, out_pkts, out_bytes, out_goodput_bytes); @@ -4121,6 +4166,7 @@ void Flow::updateTcpFlags(const struct bpf_timeval *when, cli_host->incFlagStats(src2dst_direction, flags, cumulative_flags); cli_network_stats = cli_host->getNetworkStats(cli_host->get_local_network_id()); } + if(srv_host) { srv_host->incFlagStats(!src2dst_direction, flags, cumulative_flags); srv_network_stats = srv_host->getNetworkStats(srv_host->get_local_network_id()); diff --git a/src/HostPorts.cpp b/src/HostPorts.cpp index 3ef6a06caa..2993bfad4e 100644 --- a/src/HostPorts.cpp +++ b/src/HostPorts.cpp @@ -24,56 +24,67 @@ /* *************************************** */ HostPorts::HostPorts() { - host_server_ports = ndpi_bitmap_alloc(), contacted_ports = ndpi_bitmap_alloc(); + ; } /* *************************************** */ HostPorts::~HostPorts() { - if(host_server_ports) ndpi_bitmap_free(host_server_ports); - if(contacted_ports) ndpi_bitmap_free(contacted_ports); + ; } /* *************************************** */ void HostPorts::reset() { - if(host_server_ports) ndpi_bitmap_clear(host_server_ports); - if(contacted_ports) ndpi_bitmap_clear(contacted_ports); + udp_host_server_ports.clear(), tcp_host_server_ports.clear(); + udp_client_contacted_ports.clear(), tcp_client_contacted_ports.clear(); } /* *************************************** */ -void HostPorts::setLuaArray(lua_State *vm, ndpi_bitmap *ports, const char *label) { +void HostPorts::setLuaArray(lua_State *vm, NetworkInterface *iface, + bool isTCP, std::unordered_map *ports) { if(ports) { - ndpi_bitmap_iterator *i = ndpi_bitmap_iterator_alloc(ports); - - if(i) { - u_int32_t port, index = 1; + std::unordered_map::iterator it; + + for(it = ports->begin(); it != ports->end(); ++it) { + char str[32], buf[64]; - lua_createtable(vm, ndpi_bitmap_cardinality(ports), 0); - - while(ndpi_bitmap_iterator_next(i, &port)) { - lua_pushinteger(vm, port); - lua_rawseti(vm, -2, index++); - } - - ndpi_bitmap_iterator_free(i); - - lua_pushstring(vm, label); - lua_insert(vm, -2); - lua_settable(vm, -3); + snprintf(str, sizeof(str), "%s:%u", isTCP ? "tcp" : "udp", it->first); + lua_push_str_table_entry(vm, str, ndpi_protocol2name(iface->get_ndpi_struct(), it->second, buf, sizeof(buf))); } } } /* *************************************** */ -void HostPorts::lua(lua_State *vm) { +void HostPorts::lua(lua_State *vm, NetworkInterface *iface) { lua_newtable(vm); - setLuaArray(vm, host_server_ports, "local_server_ports"); - setLuaArray(vm, contacted_ports, "remote_contacted_ports"); - + lua_newtable(vm); + + /* ***************************** */ + + setLuaArray(vm, iface, true, &tcp_host_server_ports); + setLuaArray(vm, iface, false, &udp_host_server_ports); + + lua_pushstring(vm, "local_server_ports"); + lua_insert(vm, -2); + lua_settable(vm, -3); + + /* ***************************** */ + + lua_newtable(vm); + + setLuaArray(vm, iface, true, &tcp_client_contacted_ports); + setLuaArray(vm, iface, false, &udp_client_contacted_ports); + + lua_pushstring(vm, "remote_contacted_ports"); + lua_insert(vm, -2); + lua_settable(vm, -3); + + /* ***************************** */ + lua_pushstring(vm, "used_ports"); lua_insert(vm, -2); lua_settable(vm, -3); @@ -81,12 +92,18 @@ void HostPorts::lua(lua_State *vm) { /* *************************************** */ -void HostPorts::setServerPort(bool isTCP /* ignored */, u_int16_t port) { - if(host_server_ports) ndpi_bitmap_set(host_server_ports, port); +void HostPorts::setServerPort(bool isTCP, u_int16_t port, ndpi_protocol *proto) { + if(isTCP) + tcp_host_server_ports[port] = *proto; + else + udp_host_server_ports[port] = *proto; } /* *************************************** */ -void HostPorts::setContactedPort(bool isTCP /* ignored */, u_int16_t port) { - if(contacted_ports) ndpi_bitmap_set(contacted_ports, port); +void HostPorts::setContactedPort(bool isTCP, u_int16_t port, ndpi_protocol *proto) { + if(isTCP) + tcp_client_contacted_ports[port] = *proto; + else + udp_client_contacted_ports[port] = *proto; } diff --git a/src/LocalHost.cpp b/src/LocalHost.cpp index 7df64e3c60..c5cfb7d283 100644 --- a/src/LocalHost.cpp +++ b/src/LocalHost.cpp @@ -291,7 +291,7 @@ void LocalHost::lua(lua_State* vm, AddressTree *ptree, Host::lua_blacklisted_flows(vm); lua_contacts_stats(vm); - usedPorts.lua(vm); + usedPorts.lua(vm, iface); /* *** */ diff --git a/src/NetworkInterface.cpp b/src/NetworkInterface.cpp index b12d8c0b6e..db47114bb7 100644 --- a/src/NetworkInterface.cpp +++ b/src/NetworkInterface.cpp @@ -3753,22 +3753,27 @@ void NetworkInterface::refreshHostPools() { static bool count_open_server_ports(GenericHashEntry *node, void *user_data, bool *matched) { Host *h = (Host*)node; std::unordered_map *count = (std::unordered_map*)user_data; - ndpi_bitmap* ports = h->getServerPorts(); - ndpi_bitmap_iterator *i = ndpi_bitmap_iterator_alloc(ports); + std::unordered_map::iterator it; - if(i) { - u_int32_t port; + for(u_int i=0; i<2; i++) { + std::unordered_map* ports; - while(ndpi_bitmap_iterator_next(i, &port)) { - std::unordered_map::iterator it = count->find(port); + if(i == 0) + ports = h->getServerPorts(true); + else + ports = h->getServerPorts(false); - if(it == count->end()) - (*count)[port] = 1; - else - (*count)[port] = (*count)[port] + 1; + if(ports) { + for(it = ports->begin(); it != ports->end(); ++it) { + u_int16_t port = it->first; + std::unordered_map::iterator it1 = count->find(port); + + if(it1 == count->end()) + (*count)[port] = 1; + else + (*count)[port] = (*count)[port] + 1; + } } - - ndpi_bitmap_iterator_free(i); } *matched = true;