From b58f67a5de2b2604b92f3cb85a50d5b57c46d295 Mon Sep 17 00:00:00 2001 From: Simone Mainardi Date: Tue, 2 Apr 2019 12:29:32 +0200 Subject: [PATCH] Protocol detection fixes --- include/Flow.h | 2 - src/Flow.cpp | 90 +++++++++++++++++----------------------- src/NetworkInterface.cpp | 30 ++++++-------- 3 files changed, 50 insertions(+), 72 deletions(-) diff --git a/include/Flow.h b/include/Flow.h index 09901fc519..d86fd6cd2d 100644 --- a/include/Flow.h +++ b/include/Flow.h @@ -362,7 +362,6 @@ class Flow : public GenericHashEntry { u_int64_t get_current_packets_cli2srv(); u_int64_t get_current_packets_srv2cli(); inline bool idle() { return(is_ready_to_be_purged()); } - inline bool is_l7_protocol_guessed() { return(l7_protocol_guessed); }; char* print(char *buf, u_int buf_len) const; void update_hosts_stats(struct timeval *tv, bool dump_alert); u_int32_t key(); @@ -378,7 +377,6 @@ class Flow : public GenericHashEntry { bool *src2srv_direction); bool clientLessThanServer() const; void sumStats(nDPIStats *stats); - void guessProtocol(); bool dumpFlow(bool dump_alert); bool match(AddressTree *ptree); void dissectHTTP(bool src2dst_direction, char *payload, u_int16_t payload_len); diff --git a/src/Flow.cpp b/src/Flow.cpp index afa5b34e7b..5e234f5e49 100644 --- a/src/Flow.cpp +++ b/src/Flow.cpp @@ -46,7 +46,7 @@ Flow::Flow(NetworkInterface *_iface, srv2cli_last_goodput_bytes = cli2srv_last_goodput_bytes = 0, good_ssl_hs = true, flow_alerted = flow_dropped_counts_increased = false, vrfId = 0; - l7_protocol_guessed = detection_completed = false; + detection_completed = false; ndpiDetectedProtocol = ndpiUnknownProtocol; doNotExpireBefore = iface->getTimeLastPktRcvd() + DONT_NOT_EXPIRE_BEFORE_SEC; @@ -553,65 +553,49 @@ void Flow::processDetectedProtocol() { /* *************************************** */ -void Flow::guessProtocol() { - if(detection_completed) - return; /* Nothing to do */ +void Flow::setDetectedProtocol(ndpi_protocol proto_id, bool forceDetection) { + ndpi_flow_struct* ndpif; - /* This code should no longer be necessary as the nDPI API changed */ - if((protocol == IPPROTO_TCP) || (protocol == IPPROTO_UDP)) { - if(cli_host && srv_host) { - /* We can guess the protocol */ - IpAddress *cli_ip = cli_host->get_ip(), *srv_ip = srv_host->get_ip(); - ndpi_protocol guessed_proto = ndpi_guess_undetected_protocol(iface->get_ndpi_struct(), NULL, protocol, - ntohl(cli_ip ? cli_ip->get_ipv4() : 0), - ntohs(cli_port), - ntohl(srv_ip ? srv_ip->get_ipv4() : 0), - ntohs(srv_port)); - ndpiDetectedProtocol.master_protocol = guessed_proto.master_protocol; - ndpiDetectedProtocol.app_protocol = guessed_proto.app_protocol; - - /* NOTE: only overwrite the category if it was not set. - * This prevents overwriting already determined category (e.g. by IP or Host) - */ - if(ndpiDetectedProtocol.category == NDPI_PROTOCOL_CATEGORY_UNSPECIFIED) - ndpiDetectedProtocol.category = guessed_proto.category; - } - - l7_protocol_guessed = true; + /* Let the client SSL certificate win over the server SSL certificate + this addresses detection for youtube, e.g., when the client + requests s.youtube.com but the server responds with google.com */ + if(!forceDetection + && proto_id.master_protocol == NDPI_PROTOCOL_SSL + && get_packets() < NDPI_MIN_NUM_PACKETS + && (ndpif = get_ndpi_flow()) + && ndpif->protos.stun_ssl.ssl.client_certificate[0] == '\0' + && ndpif->protos.stun_ssl.ssl.server_certificate[0] == '\0') { + ndpif->detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN; + return; } - detection_completed = true; /* We give up */ -} + if(proto_id.app_protocol != NDPI_PROTOCOL_UNKNOWN + || forceDetection + || get_packets() >= NDPI_MIN_NUM_PACKETS + || !iface->is_ndpi_enabled() + || iface->isSampledTraffic()) { + ndpiDetectedProtocol.master_protocol = proto_id.master_protocol; + ndpiDetectedProtocol.app_protocol = proto_id.app_protocol; -/* *************************************** */ - -void Flow::setDetectedProtocol(ndpi_protocol proto_id, bool forceDetection) { - ndpiDetectedProtocol.category = proto_id.category; - - if(proto_id.app_protocol != NDPI_PROTOCOL_UNKNOWN) { - ndpiDetectedProtocol = proto_id; - - /* Let the client SSL certificate win over the server SSL certificate - this addresses detection for youtube, e.g., when the client - requests s.youtube.com but the server responds with google.com */ - if((proto_id.master_protocol == NDPI_PROTOCOL_SSL) - && (get_packets() < NDPI_MIN_NUM_PACKETS) - && (ndpiFlow) - && (ndpiFlow->protos.stun_ssl.ssl.client_certificate[0] == '\0') - && (ndpiFlow->protos.stun_ssl.ssl.server_certificate[0] == '\0')) { - get_ndpi_flow()->detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN; - return; - } + /* NOTE: only overwrite the category if it was not set. + * This prevents overwriting already determined category (e.g. by IP or Host) + */ + if(ndpiDetectedProtocol.category == NDPI_PROTOCOL_CATEGORY_UNSPECIFIED) + ndpiDetectedProtocol.category = proto_id.category; processDetectedProtocol(); detection_completed = true; - } else if(forceDetection - || (get_packets() >= NDPI_MIN_NUM_PACKETS) - || (!iface->is_ndpi_enabled()) - || iface->isSampledTraffic() - ) - guessProtocol(); - + +#ifdef BLACKLISTED_FLOWS_DEBUG + if(ndpiDetectedProtocol.category == CUSTOM_CATEGORY_MALWARE) { + char buf[512]; + print(buf, sizeof(buf)); + snprintf(&buf[strlen(buf)], sizeof(buf) - strlen(buf), "Malware category detected. [cli_blacklisted: %u][srv_blacklisted: %u][category: %s]", cli_host->isBlacklisted(), srv_host->isBlacklisted(), get_protocol_category_name()); + ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s", buf); + } +#endif + } + if(detection_completed) { #ifdef HAVE_NEDGE updateFlowShapers(true); diff --git a/src/NetworkInterface.cpp b/src/NetworkInterface.cpp index 5d31cec09d..98a69d668d 100644 --- a/src/NetworkInterface.cpp +++ b/src/NetworkInterface.cpp @@ -1652,24 +1652,20 @@ bool NetworkInterface::processPacket(u_int32_t bridge_iface_idx, flow->updateInterfaceLocalStats(src2dst_direction, 1, rawsize); if(!flow->isDetectionCompleted()) { - if(isSampledTraffic()) - flow->guessProtocol(); - else { - if(!is_fragment) { - struct ndpi_flow_struct *ndpi_flow = flow->get_ndpi_flow(); - struct ndpi_id_struct *cli = (struct ndpi_id_struct*)flow->get_cli_id(); - struct ndpi_id_struct *srv = (struct ndpi_id_struct*)flow->get_srv_id(); + if(!is_fragment) { + struct ndpi_flow_struct *ndpi_flow = flow->get_ndpi_flow(); + struct ndpi_id_struct *cli = (struct ndpi_id_struct*)flow->get_cli_id(); + struct ndpi_id_struct *srv = (struct ndpi_id_struct*)flow->get_srv_id(); - if(flow->get_packets() >= NDPI_MIN_NUM_PACKETS) { - flow->setDetectedProtocol(ndpi_detection_giveup(ndpi_struct, ndpi_flow, 1), false); - } else - flow->setDetectedProtocol(ndpi_detection_process_packet(ndpi_struct, ndpi_flow, - ip, ipsize, (u_int32_t)packet_time, - cli, srv), false); - } else { - // FIX - only handle unfragmented packets - // ntop->getTrace()->traceEvent(TRACE_WARNING, "IP fragments are not handled yet!"); - } + if(flow->get_packets() >= NDPI_MIN_NUM_PACKETS) { + flow->setDetectedProtocol(ndpi_detection_giveup(ndpi_struct, ndpi_flow, 1), false); + } else + flow->setDetectedProtocol(ndpi_detection_process_packet(ndpi_struct, ndpi_flow, + ip, ipsize, (u_int32_t)packet_time, + cli, srv), false); + } else { + // FIX - only handle unfragmented packets + // ntop->getTrace()->traceEvent(TRACE_WARNING, "IP fragments are not handled yet!"); } }