From e16675b7008191157d86dd7752b0a9e529315171 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Tue, 4 Aug 2020 18:09:13 +0200 Subject: [PATCH 01/32] Added new traffic category for connectivity check detection --- src/include/ndpi_typedefs.h | 5 +++++ src/lib/ndpi_content_match.c.inc | 15 +++++++++++++++ src/lib/ndpi_main.c | 2 +- tests/result/android.pcap.out | 10 +++++----- tests/result/anyconnect-vpn.pcap.out | 6 +++--- tests/result/iphone.pcap.out | 4 ++-- tests/result/teams.pcap.out | 2 +- 7 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 55fa370c9..66fac35af 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -913,6 +913,11 @@ typedef enum { NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_CATEGORY_PRODUCTIVITY, NDPI_PROTOCOL_CATEGORY_FILE_SHARING, + /* + The category below is used by sites who are used + to test connectivity + */ + NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK, /* Some custom categories */ CUSTOM_CATEGORY_MINING = 99, diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index d10d2416e..c8fe416eb 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -9175,6 +9175,21 @@ static ndpi_category_match category_match[] = { { "iptv.sky.", NDPI_PROTOCOL_CATEGORY_STREAMING }, { "pcdn.skycdn.", NDPI_PROTOCOL_CATEGORY_STREAMING }, + /* https://success.tanaza.com/s/article/How-Automatic-Detection-of-Captive-Portal-works */ + { "captive.apple.com", NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK }, + { "thinkdifferent.us", NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK }, + { "airport.us", NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK }, + { "gsp1.apple.com", NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK }, + { "msftconnecttest.com", NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK }, + { "testconnectivity.microsoft.com", NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK }, + { "msftncsi.com", NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK }, + { "msftncsi.com.edgesuite.net", NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK }, + { "teredo.ipv6.microsoft.com", NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK }, + { "teredo.ipv6.microsoft.com.nsatc.net", NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK }, + { "detectportal.firefox.com", NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK }, + { "connectivitycheck.android.com", NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK }, + { "connectivitycheck.gstatic.com", NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK }, + /* Hulu Streaming services AS23286 */ { "8.28.124.0/24", NDPI_PROTOCOL_CATEGORY_STREAMING }, { "8.28.125.0/24", NDPI_PROTOCOL_CATEGORY_STREAMING }, diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index db84efa00..bdbdc89f3 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -1875,7 +1875,7 @@ static const char *categories[] = { "Shopping", "Productivity", "FileSharing", - "", + "ConnectivityCheck", "", "", "", diff --git a/tests/result/android.pcap.out b/tests/result/android.pcap.out index 320c84bb6..3b500859b 100644 --- a/tests/result/android.pcap.out +++ b/tests/result/android.pcap.out @@ -27,8 +27,8 @@ JA3 Host Stats: 7 TCP 192.168.2.16:50384 <-> 172.217.168.206:443 [proto: 91.126/TLS.Google][cat: Web/5][11 pkts/1365 bytes <-> 9 pkts/5365 bytes][Goodput ratio: 45/89][2.49 sec][ALPN: http/1.1][bytes ratio: -0.594 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 277/69 1716/301 516/102][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 124/596 407/1484 105/544][TLSv1.2][Client: app-measurement.com][JA3C: 6ec2896feff5746955f700c0023f5804][ServerNames: *.google-analytics.com,*.fps.goog,app-measurement.com,fps.goog,google-analytics.com,googleoptimize.com,googletagmanager.com,service.urchin.com,ssl.google-analytics.com,urchin.com,www.google-analytics.com,www.googleoptimize.com,www.googletagmanager.com][JA3S: 9d9ce860f1b1cbef07b019450cb368d8][Issuer: C=US, O=Google Trust Services, CN=GTS CA 1O1][Subject: C=US, ST=California, L=Mountain View, O=Google LLC, CN=*.google-analytics.com][Certificate SHA-1: B0:D9:D3:57:C2:34:87:2C:FB:F5:E6:BD:7F:9F:54:65:08:61:AF:01][Validity: 2020-02-12 11:37:03 - 2020-05-06 11:37:03][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 0,0,11,0,0,11,0,0,0,11,11,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,22,0,0,0] 8 TCP 192.168.2.16:52486 <-> 172.217.20.74:443 [proto: 91.239/TLS.GoogleServices][cat: Web/5][12 pkts/1298 bytes <-> 10 pkts/5186 bytes][Goodput ratio: 38/87][1.75 sec][ALPN: http/1.1][bytes ratio: -0.600 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 194/37 1374/212 422/70][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 108/519 286/1484 76/570][TLSv1.2][Client: play.googleapis.com][JA3C: d8c87b9bfde38897979e41242626c2f3][ServerNames: *.storage.googleapis.com,*.appspot.com.storage.googleapis.com,*.commondatastorage.googleapis.com,*.content-storage-download.googleapis.com,*.content-storage-upload.googleapis.com,*.content-storage.googleapis.com,*.googleapis.com,*.storage-download.googleapis.com,*.storage-upload.googleapis.com,*.storage.select.googleapis.com,commondatastorage.googleapis.com,storage.googleapis.com,storage.select.googleapis.com,unfiltered.news][JA3S: eca9b8f0f3eae50309eaf901cb822d9b][Issuer: C=US, O=Google Trust Services, CN=GTS CA 1O1][Subject: C=US, ST=California, L=Mountain View, O=Google LLC, CN=*.storage.googleapis.com][Certificate SHA-1: BA:BA:BA:55:69:9F:E0:BD:48:80:23:A4:B3:AD:C1:FF:EA:4E:17:C9][Validity: 2020-02-12 11:45:22 - 2020-05-06 11:45:22][Cipher: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256][Plen Bins: 0,10,10,0,20,10,10,0,0,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10,0,0,0,0,0,0,0,0,0,0,20,0,0,0] 9 TCP 192.168.2.16:32988 <-> 216.239.38.120:443 [proto: 91.228/TLS.PlayStore][cat: SoftwareUpdate/19][8 pkts/2089 bytes <-> 7 pkts/4242 bytes][Goodput ratio: 74/89][0.97 sec][bytes ratio: -0.340 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 158/80 530/246 186/98][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 261/606 1038/1484 338/639][Risk: ** TLS (probably) not carrying HTTPS **][TLSv1.3][Client: android.clients.google.com][JA3C: 9c815150ea821166faecf80757d8826a][JA3S: eb1d94daa7e0344597e756a1fb6e7054][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,16,16,0,0,0,0,0,0,0,0,0,0,0,0,0,34,0,0,0] - 10 TCP 192.168.2.16:36888 <-> 172.217.18.3:443 [proto: 91.126/TLS.Google][cat: Web/5][9 pkts/1175 bytes <-> 7 pkts/4762 bytes][Goodput ratio: 47/90][1.62 sec][ALPN: http/1.1][bytes ratio: -0.604 (Download)][IAT c2s/s2c min/avg/max/stddev: 27/28 203/104 522/277 176/93][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 131/680 327/1484 93/575][TLSv1.2][Client: connectivitycheck.gstatic.com][JA3C: d8c87b9bfde38897979e41242626c2f3][Plen Bins: 0,0,12,0,0,0,12,0,12,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,0,0,0] - 11 TCP 192.168.2.16:36890 <-> 172.217.18.3:443 [proto: 91.126/TLS.Google][cat: Web/5][9 pkts/1151 bytes <-> 7 pkts/4762 bytes][Goodput ratio: 48/90][0.84 sec][ALPN: http/1.1][bytes ratio: -0.611 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 120/15 647/36 217/16][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 128/680 327/1484 95/575][TLSv1.2][Client: connectivitycheck.gstatic.com][JA3C: d8c87b9bfde38897979e41242626c2f3][ServerNames: *.google.com,*.android.com,*.appengine.google.com,*.cloud.google.com,*.crowdsource.google.com,*.g.co,*.gcp.gvt2.com,*.gcpcdn.gvt1.com,*.ggpht.cn,*.gkecnapps.cn,*.google-analytics.com,*.google.ca,*.google.cl,*.google.co.in,*.google.co.jp,*.google.co.uk,*.google.com.ar,*.google.com.au,*.google.com.br,*.google.com.co,*.google.com.mx,*.google.com.tr,*.google.com.vn,*.google.de,*.google.es,*.google.fr,*.google.hu,*.google.it,*.google.nl,*.google.pl,*.google.pt,*.googleadapis.com,*.googleapis.cn,*.googlecnapps.cn,*.googlecommerce.com,*.googlevideo.com,*.gstatic.cn,*.gstatic.com,*.gstaticcnapps.cn,*.gvt1.com,*.gvt2.com,*.metric.gstatic.com,*.urchin.com,*.url.google.com,*.wear.gkecnapps.cn,*.youtube-nocookie.com,*.youtube.com,*.youtubeeducation.com,*.youtubekids.com,*.yt.be,*.ytimg.com,android.clients.google.com,android.com,developer.android.google.cn,developers.android.google.cn,g.co,ggpht.cn,gkecnapps.cn,goo.gl,google-analytics.com,google.com,googlecnapps.cn,googlecommerce.com,source.android.google.cn,urchin.com,www.goo.gl,youtu.be,youtube.com,youtubeeducation.com,youtubekids.com,yt.be][JA3S: eca9b8f0f3eae50309eaf901cb822d9b][Issuer: C=US, O=Google Trust Services, CN=GTS CA 1O1][Subject: C=US, ST=California, L=Mountain View, O=Google LLC, CN=*.google.com][Certificate SHA-1: 80:50:28:F4:84:F5:C4:C6:41:DE:75:67:38:C4:A6:E2:59:FF:75:42][Validity: 2020-02-12 11:47:11 - 2020-05-06 11:47:11][Cipher: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256][Plen Bins: 0,0,12,0,0,0,12,0,12,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,0,0,0] + 10 TCP 192.168.2.16:36888 <-> 172.217.18.3:443 [proto: 91.126/TLS.Google][cat: ConnectivityCheck/30][9 pkts/1175 bytes <-> 7 pkts/4762 bytes][Goodput ratio: 47/90][1.62 sec][ALPN: http/1.1][bytes ratio: -0.604 (Download)][IAT c2s/s2c min/avg/max/stddev: 27/28 203/104 522/277 176/93][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 131/680 327/1484 93/575][TLSv1.2][Client: connectivitycheck.gstatic.com][JA3C: d8c87b9bfde38897979e41242626c2f3][Plen Bins: 0,0,12,0,0,0,12,0,12,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,0,0,0] + 11 TCP 192.168.2.16:36890 <-> 172.217.18.3:443 [proto: 91.126/TLS.Google][cat: ConnectivityCheck/30][9 pkts/1151 bytes <-> 7 pkts/4762 bytes][Goodput ratio: 48/90][0.84 sec][ALPN: http/1.1][bytes ratio: -0.611 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 120/15 647/36 217/16][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 128/680 327/1484 95/575][TLSv1.2][Client: connectivitycheck.gstatic.com][JA3C: d8c87b9bfde38897979e41242626c2f3][ServerNames: *.google.com,*.android.com,*.appengine.google.com,*.cloud.google.com,*.crowdsource.google.com,*.g.co,*.gcp.gvt2.com,*.gcpcdn.gvt1.com,*.ggpht.cn,*.gkecnapps.cn,*.google-analytics.com,*.google.ca,*.google.cl,*.google.co.in,*.google.co.jp,*.google.co.uk,*.google.com.ar,*.google.com.au,*.google.com.br,*.google.com.co,*.google.com.mx,*.google.com.tr,*.google.com.vn,*.google.de,*.google.es,*.google.fr,*.google.hu,*.google.it,*.google.nl,*.google.pl,*.google.pt,*.googleadapis.com,*.googleapis.cn,*.googlecnapps.cn,*.googlecommerce.com,*.googlevideo.com,*.gstatic.cn,*.gstatic.com,*.gstaticcnapps.cn,*.gvt1.com,*.gvt2.com,*.metric.gstatic.com,*.urchin.com,*.url.google.com,*.wear.gkecnapps.cn,*.youtube-nocookie.com,*.youtube.com,*.youtubeeducation.com,*.youtubekids.com,*.yt.be,*.ytimg.com,android.clients.google.com,android.com,developer.android.google.cn,developers.android.google.cn,g.co,ggpht.cn,gkecnapps.cn,goo.gl,google-analytics.com,google.com,googlecnapps.cn,googlecommerce.com,source.android.google.cn,urchin.com,www.goo.gl,youtu.be,youtube.com,youtubeeducation.com,youtubekids.com,yt.be][JA3S: eca9b8f0f3eae50309eaf901cb822d9b][Issuer: C=US, O=Google Trust Services, CN=GTS CA 1O1][Subject: C=US, ST=California, L=Mountain View, O=Google LLC, CN=*.google.com][Certificate SHA-1: 80:50:28:F4:84:F5:C4:C6:41:DE:75:67:38:C4:A6:E2:59:FF:75:42][Validity: 2020-02-12 11:47:11 - 2020-05-06 11:47:11][Cipher: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256][Plen Bins: 0,0,12,0,0,0,12,0,12,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,0,0,0] 12 TCP 192.168.2.16:33014 <-> 216.239.38.120:443 [proto: 91.126/TLS.Google][cat: Web/5][11 pkts/1877 bytes <-> 7 pkts/3708 bytes][Goodput ratio: 61/87][0.20 sec][ALPN: h2;http/1.1][TLS Supported Versions: GREASE;TLSv1.3;TLSv1.2;TLSv1.1;TLSv1][bytes ratio: -0.328 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 21/11 96/40 29/16][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 171/530 583/1484 180/574][TLSv1.3][Client: www.google.com][JA3C: 66918128f1b9b03303d77c6f2eefd128][JA3S: eb1d94daa7e0344597e756a1fb6e7054][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 22,0,22,0,0,0,0,0,0,0,0,0,0,11,0,0,11,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,0,0,0,0,0,0,11,0,0,0] 13 TCP 192.168.2.16:51944 <-> 172.217.21.202:443 [proto: 91.46/TLS.DataSaver][cat: Web/5][12 pkts/2171 bytes <-> 12 pkts/2705 bytes][Goodput ratio: 63/70][0.20 sec][ALPN: h2;http/1.1][TLS Supported Versions: GREASE;TLSv1.3;TLSv1.2;TLSv1.1;TLSv1][bytes ratio: -0.110 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 14/11 39/64 15/19][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 181/225 660/646 208/202][TLSv1.3][Client: datasaver.googleapis.com][JA3C: 554719594ba90b02ae410c297c6e50ad][JA3S: 2b0648ab686ee45e0e7c35fcfb0eea7e][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 15,15,15,0,0,0,15,0,0,0,7,0,0,0,0,7,0,7,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 14 TCP 192.168.2.16:43646 <-> 172.217.20.76:443 [proto: 91.46/TLS.DataSaver][cat: Web/5][8 pkts/1053 bytes <-> 6 pkts/3460 bytes][Goodput ratio: 49/88][0.20 sec][ALPN: h2;http/1.1][TLS Supported Versions: GREASE;TLSv1.3;TLSv1.2;TLSv1.1;TLSv1][bytes ratio: -0.533 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 32/16 51/61 18/26][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 132/577 583/1484 171/646][TLSv1.3][Client: proxy.googlezip.net][JA3C: 66918128f1b9b03303d77c6f2eefd128][JA3S: eb1d94daa7e0344597e756a1fb6e7054][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 0,0,0,0,0,0,25,0,0,0,0,0,0,0,0,0,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0] @@ -37,7 +37,7 @@ JA3 Host Stats: 17 UDP 0.0.0.0:68 -> 255.255.255.255:67 [proto: 18/DHCP][cat: Network/14][12 pkts/4088 bytes -> 0 pkts/0 bytes][Goodput ratio: 88/0][82.22 sec][Host: lucas-imac][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 130/0 6001/0 8764/0 3124/0][Pkt Len c2s/s2c min/avg/max/stddev: 328/0 341/0 342/0 4/0][DHCP Fingerprint: 1,121,3,6,15,119,252,95,44,46][PLAIN TEXT (android)][Plen Bins: 0,0,0,0,0,0,0,0,8,91,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 18 TCP 192.168.2.16:36834 <-> 173.194.79.114:80 [proto: 7.46/HTTP.DataSaver][cat: Web/5][8 pkts/1130 bytes <-> 5 pkts/1254 bytes][Goodput ratio: 53/73][0.30 sec][Host: check.googlezip.net][bytes ratio: -0.052 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 1/1 41/59 105/141 31/59][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 141/251 363/524 128/223][URL: check.googlezip.net/connect][StatusCode: 200][Content-Type: text/html][User-Agent: Mozilla/5.0 (Linux; Android 9; Nokia 2.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.93 Mobile Safari/537.36][Risk: ** HTTP Suspicious User-Agent **][PLAIN TEXT (GET /connect HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,50,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 19 TCP 192.168.2.16:44374 <-> 172.217.22.10:443 [proto: 91.239/TLS.GoogleServices][cat: Web/5][3 pkts/723 bytes <-> 3 pkts/1624 bytes][Goodput ratio: 71/87][0.10 sec][bytes ratio: -0.384 (Download)][IAT c2s/s2c min/avg/max/stddev: 26/9 33/38 40/66 7/28][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 241/541 583/1484 242/667][Risk: ** TLS (probably) not carrying HTTPS **][TLSv1.3][Client: android.googleapis.com][JA3C: 629b587f706aee60430ec3879c6edb66][JA3S: eb1d94daa7e0344597e756a1fb6e7054][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0] - 20 TCP 192.168.2.16:58338 <-> 17.253.53.201:80 [proto: 7.140/HTTP.Apple][cat: Web/5][6 pkts/607 bytes <-> 5 pkts/1053 bytes][Goodput ratio: 33/68][0.16 sec][Host: captive.apple.com][bytes ratio: -0.269 (Download)][IAT c2s/s2c min/avg/max/stddev: 3/0 25/23 42/46 15/23][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 101/211 269/781 75/285][URL: captive.apple.com/][StatusCode: 200][Content-Type: text/html][User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.32 Safari/537.36][Risk: ** HTTP Suspicious User-Agent **][PLAIN TEXT (GET / HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 20 TCP 192.168.2.16:58338 <-> 17.253.53.201:80 [proto: 7.140/HTTP.Apple][cat: ConnectivityCheck/30][6 pkts/607 bytes <-> 5 pkts/1053 bytes][Goodput ratio: 33/68][0.16 sec][Host: captive.apple.com][bytes ratio: -0.269 (Download)][IAT c2s/s2c min/avg/max/stddev: 3/0 25/23 42/46 15/23][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 101/211 269/781 75/285][URL: captive.apple.com/][StatusCode: 200][Content-Type: text/html][User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.32 Safari/537.36][Risk: ** HTTP Suspicious User-Agent **][PLAIN TEXT (GET / HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 21 UDP 192.168.2.1:17500 -> 192.168.2.255:17500 [proto: 121/Dropbox][cat: Cloud/13][3 pkts/1656 bytes -> 0 pkts/0 bytes][Goodput ratio: 92/0][60.10 sec][PLAIN TEXT (version)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 22 TCP 192.168.2.16:36848 <-> 173.194.79.114:80 [proto: 7.46/HTTP.DataSaver][cat: Web/5][4 pkts/569 bytes <-> 3 pkts/664 bytes][Goodput ratio: 52/69][0.11 sec][Host: check.googlezip.net][bytes ratio: -0.077 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 31/1 37/36 41/72 4/36][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 142/221 363/524 127/214][URL: check.googlezip.net/connect][StatusCode: 200][Content-Type: text/html][User-Agent: Mozilla/5.0 (Linux; Android 9; Nokia 2.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.93 Mobile Safari/537.36][Risk: ** HTTP Suspicious User-Agent **][PLAIN TEXT (GET /connect HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,50,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 23 TCP 17.248.176.75:443 -> 192.168.2.17:50580 [proto: 91.140/TLS.Apple][cat: Web/5][8 pkts/1067 bytes -> 0 pkts/0 bytes][Goodput ratio: 50/0][18.90 sec][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 294/0 2700/0 9727/0 3229/0][Pkt Len c2s/s2c min/avg/max/stddev: 97/0 133/0 143/0 17/0][Plen Bins: 12,12,75,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] @@ -45,13 +45,13 @@ JA3 Host Stats: 25 TCP 192.168.2.16:52514 <-> 172.217.20.74:443 [proto: 91.239/TLS.GoogleServices][cat: Web/5][3 pkts/723 bytes <-> 1 pkts/74 bytes][Goodput ratio: 71/0][0.27 sec][ALPN: h2][TLS Supported Versions: TLSv1.3;TLSv1.2;TLSv1.1;TLSv1][TLSv1.2][Client: semanticlocation-pa.googleapis.com][JA3C: 33490b1d5377580b19f7f9b5849d7991][PLAIN TEXT (semanticlocation)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 26 UDP 192.168.2.1:67 -> 192.168.2.16:68 [proto: 18/DHCP][cat: Network/14][2 pkts/684 bytes -> 0 pkts/0 bytes][Goodput ratio: 88/0][0.13 sec][PLAIN TEXT (iMac.local)][Plen Bins: 0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 27 TCP 17.248.185.10:443 -> 192.168.2.17:50702 [proto: 91.140/TLS.Apple][cat: Web/5][7 pkts/648 bytes -> 0 pkts/0 bytes][Goodput ratio: 29/0][13.42 sec][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 427/0 2236/0 6975/0 2385/0][Pkt Len c2s/s2c min/avg/max/stddev: 66/0 93/0 97/0 11/0][Plen Bins: 100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] - 28 UDP 192.168.2.16:52953 <-> 192.168.2.1:53 [proto: 5.140/DNS.Apple][cat: Web/5][1 pkts/77 bytes <-> 1 pkts/221 bytes][Goodput ratio: 45/81][0.04 sec][Host: captive.apple.com][17.253.53.201][PLAIN TEXT (captive)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 28 UDP 192.168.2.16:52953 <-> 192.168.2.1:53 [proto: 5.140/DNS.Apple][cat: ConnectivityCheck/30][1 pkts/77 bytes <-> 1 pkts/221 bytes][Goodput ratio: 45/81][0.04 sec][Host: captive.apple.com][17.253.53.201][PLAIN TEXT (captive)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 29 UDP 192.168.2.1:57621 -> 192.168.2.255:57621 [proto: 156/Spotify][cat: Music/25][3 pkts/258 bytes -> 0 pkts/0 bytes][Goodput ratio: 51/0][60.02 sec][PLAIN TEXT (SpotUdp)][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 30 UDP [fe80::4e6a:f6ff:fe9f:f627]:546 -> [ff02::1:2]:547 [proto: 103/DHCPV6][cat: Network/14][2 pkts/228 bytes -> 0 pkts/0 bytes][Goodput ratio: 45/0][2.16 sec][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 31 UDP 192.168.2.16:35825 <-> 192.168.2.1:53 [proto: 5/DNS][cat: Network/14][1 pkts/76 bytes <-> 1 pkts/140 bytes][Goodput ratio: 44/70][0.04 sec][Host: time.android.com][216.239.35.8][PLAIN TEXT (android)][Plen Bins: 0,50,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 32 TCP 192.168.2.16:36850 <-> 173.194.79.114:80 [proto: 7.126/HTTP.Google][cat: Web/5][2 pkts/140 bytes <-> 1 pkts/74 bytes][Goodput ratio: 0/0][0.04 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 33 UDP 192.168.2.16:35689 <-> 192.168.2.1:53 [proto: 5.239/DNS.GoogleServices][cat: Web/5][1 pkts/94 bytes <-> 1 pkts/110 bytes][Goodput ratio: 55/61][0.04 sec][Host: semanticlocation-pa.googleapis.com][172.217.20.74][PLAIN TEXT (semanticlocation)][Plen Bins: 0,50,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] - 34 UDP 192.168.2.16:47081 <-> 192.168.2.1:53 [proto: 5.126/DNS.Google][cat: Web/5][1 pkts/89 bytes <-> 1 pkts/105 bytes][Goodput ratio: 52/59][0.04 sec][Host: connectivitycheck.gstatic.com][172.217.18.3][PLAIN TEXT (connectivitycheck)][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 34 UDP 192.168.2.16:47081 <-> 192.168.2.1:53 [proto: 5.126/DNS.Google][cat: ConnectivityCheck/30][1 pkts/89 bytes <-> 1 pkts/105 bytes][Goodput ratio: 52/59][0.04 sec][Host: connectivitycheck.gstatic.com][172.217.18.3][PLAIN TEXT (connectivitycheck)][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 35 UDP 192.168.2.16:36613 <-> 192.168.2.1:53 [proto: 5.228/DNS.PlayStore][cat: SoftwareUpdate/19][1 pkts/86 bytes <-> 1 pkts/102 bytes][Goodput ratio: 51/58][0.00 sec][Host: android.clients.google.com][216.239.38.120][PLAIN TEXT (android)][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 36 UDP 192.168.2.16:7660 <-> 192.168.2.1:53 [proto: 5.46/DNS.DataSaver][cat: Web/5][1 pkts/84 bytes <-> 1 pkts/100 bytes][Goodput ratio: 49/57][0.04 sec][Host: datasaver.googleapis.com][172.217.21.202][PLAIN TEXT (datasaver)][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 37 UDP 192.168.2.16:18379 <-> 192.168.2.1:53 [proto: 5.46/DNS.DataSaver][cat: Web/5][1 pkts/84 bytes <-> 1 pkts/100 bytes][Goodput ratio: 49/57][0.00 sec][Host: datasaver.googleapis.com][172.217.21.202][PLAIN TEXT (datasaver)][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] diff --git a/tests/result/anyconnect-vpn.pcap.out b/tests/result/anyconnect-vpn.pcap.out index da6d128a6..2c80f55da 100644 --- a/tests/result/anyconnect-vpn.pcap.out +++ b/tests/result/anyconnect-vpn.pcap.out @@ -27,11 +27,11 @@ JA3 Host Stats: 4 TCP 10.0.0.227:56921 <-> 8.37.96.194:4287 [proto: 91/TLS][cat: Web/5][29 pkts/5373 bytes <-> 28 pkts/7580 bytes][Goodput ratio: 64/75][2.30 sec][bytes ratio: -0.170 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/1 91/63 593/619 145/135][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 185/271 1261/1434 259/387][Risk: ** Self-signed Certificate **** TLS (probably) not carrying HTTPS **][TLSv1.2][JA3C: e3adec914f3893f18136762f1c0d7d81][JA3S: e54965894d6b45ecb4323c7ea3d6c115][Issuer: CN=813845657003339838, O=Code42, OU=TEST, ST=MN, C=US][Subject: CN=813845657003339838, O=Code42, OU=TEST, ST=MN, C=US][Certificate SHA-1: 86:2A:47:EF:00:68:79:60:7F:94:E2:91:6F:E0:38:82:37:8A:8E:2E][Validity: 2019-08-29 00:12:40 - 2019-10-08 00:12:40][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 0,44,3,3,3,3,3,0,3,3,3,0,3,7,0,0,0,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,3,0,0,3,0,0,3,0,3,0,0,0,0,0] 5 TCP 10.0.0.227:56918 <-> 8.37.102.91:443 [proto: 91/TLS][cat: Web/5][16 pkts/2739 bytes <-> 14 pkts/7315 bytes][Goodput ratio: 61/87][0.35 sec][ALPN: http/1.1][bytes ratio: -0.455 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 23/26 48/88 21/29][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 171/522 1175/1514 274/624][Risk: ** Weak TLS cipher **** TLS Certificate Mismatch **][TLSv1.2][JA3C: 9f1a41f932f274fe47a992310a26a23a][ServerNames: *.pandion.viasat.com,pandion.viasat.com][JA3S: 82f0d8a75fa483d1cfe4b7085b784d7e (WEAK)][Issuer: C=US, O=Entrust, Inc., OU=See www.entrust.net/legal-terms, OU=(c) 2012 Entrust, Inc. - for authorized use only, CN=Entrust Certification Authority - L1K][Subject: C=US, ST=California, L=Carlsbad, O=Viasat Inc., CN=*.pandion.viasat.com][Certificate SHA-1: 92:70:CF:E3:69:4B:1D:F4:E2:DE:63:54:EC:DF:40:DB:F3:AC:D1:CA][Validity: 2019-02-05 21:43:58 - 2021-02-05 22:13:57][Cipher: TLS_RSA_WITH_AES_256_CBC_SHA][Plen Bins: 0,16,8,0,0,8,0,8,0,8,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0,0,0,0,0,0,0,8,0,25,0,0] 6 TCP 10.0.0.227:56920 <-> 99.86.34.156:443 [proto: 91.118/TLS.Slack][cat: Collaborative/15][16 pkts/2949 bytes <-> 11 pkts/1876 bytes][Goodput ratio: 64/61][11.47 sec][ALPN: h2;http/1.1][bytes ratio: 0.222 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 866/28 11074/80 2947/34][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 184/171 853/487 228/155][TLSv1.2][Client: slack.com][JA3C: d8dc5f8940df366b3a58b935569143e8][JA3S: 7bee5c1d424b7e5f943b06983bb11422][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 0,34,16,0,8,0,0,0,0,0,0,0,8,16,0,0,8,0,0,0,0,0,0,0,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] - 7 TCP 10.0.0.227:56884 <-> 184.25.56.77:80 [proto: 7/HTTP][cat: Web/5][12 pkts/2303 bytes <-> 7 pkts/2382 bytes][Goodput ratio: 67/81][18.51 sec][Host: detectportal.firefox.com][bytes ratio: -0.017 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 7/31 1824/3642 10081/10083 3593/4385][Pkt Len c2s/s2c min/avg/max/stddev: 54/66 192/340 373/450 153/173][URL: detectportal.firefox.com/success.txt?ipv4][StatusCode: 200][User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:69.0) Gecko/20100101 Firefox/69.0][Risk: ** HTTP Suspicious User-Agent **][PLAIN TEXT (GET /success.txt)][Plen Bins: 0,0,0,0,0,0,0,0,0,50,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 7 TCP 10.0.0.227:56884 <-> 184.25.56.77:80 [proto: 7/HTTP][cat: ConnectivityCheck/30][12 pkts/2303 bytes <-> 7 pkts/2382 bytes][Goodput ratio: 67/81][18.51 sec][Host: detectportal.firefox.com][bytes ratio: -0.017 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 7/31 1824/3642 10081/10083 3593/4385][Pkt Len c2s/s2c min/avg/max/stddev: 54/66 192/340 373/450 153/173][URL: detectportal.firefox.com/success.txt?ipv4][StatusCode: 200][User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:69.0) Gecko/20100101 Firefox/69.0][Risk: ** HTTP Suspicious User-Agent **][PLAIN TEXT (GET /success.txt)][Plen Bins: 0,0,0,0,0,0,0,0,0,50,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 8 TCP 10.0.0.227:56320 <-> 10.0.0.149:8009 [proto: 161/CiscoVPN][cat: VPN/2][20 pkts/2420 bytes <-> 10 pkts/1760 bytes][Goodput ratio: 45/62][45.04 sec][bytes ratio: 0.158 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 2/5003 2648/5004 5001/5006 2495/1][Pkt Len c2s/s2c min/avg/max/stddev: 66/176 121/176 176/176 55/0][Plen Bins: 0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 9 ICMPV6 [fe80::2e7e:81ff:feb0:4aa1]:0 -> [ff02::1]:0 [proto: 102/ICMPV6][cat: Network/14][16 pkts/2784 bytes -> 0 pkts/0 bytes][Goodput ratio: 64/0][45.47 sec][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 2867/0 3028/0 3072/0 84/0][Pkt Len c2s/s2c min/avg/max/stddev: 174/0 174/0 174/0 0/0][Plen Bins: 0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 10 TCP 10.0.0.227:56955 <-> 10.0.0.151:8060 [proto: 7/HTTP][cat: Web/5][6 pkts/650 bytes <-> 5 pkts/1668 bytes][Goodput ratio: 37/80][4.02 sec][Host: 10.0.0.151][bytes ratio: -0.439 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/2 4/4 9/6 3/2][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 108/334 308/1206 89/442][URL: 10.0.0.151:8060/dial/dd.xml][StatusCode: 200][Content-Type: text/xml][User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36][Risk: ** Known protocol on non standard port **** HTTP Suspicious User-Agent **][PLAIN TEXT (GET /dial/dd.xml HTTP/1.1)][Plen Bins: 0,0,0,0,0,33,0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0,0,0,0] - 11 TCP 10.0.0.227:56917 <-> 184.25.56.77:80 [proto: 7/HTTP][cat: Web/5][6 pkts/976 bytes <-> 4 pkts/1032 bytes][Goodput ratio: 62/74][18.47 sec][Host: detectportal.firefox.com][bytes ratio: -0.028 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 28/573 3694/6151 10081/10078 4344/4052][Pkt Len c2s/s2c min/avg/max/stddev: 54/66 163/258 368/450 145/192][URL: detectportal.firefox.com/success.txt][StatusCode: 200][User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:69.0) Gecko/20100101 Firefox/69.0][Risk: ** HTTP Suspicious User-Agent **][PLAIN TEXT (GET /success.txt HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,50,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 11 TCP 10.0.0.227:56917 <-> 184.25.56.77:80 [proto: 7/HTTP][cat: ConnectivityCheck/30][6 pkts/976 bytes <-> 4 pkts/1032 bytes][Goodput ratio: 62/74][18.47 sec][Host: detectportal.firefox.com][bytes ratio: -0.028 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 28/573 3694/6151 10081/10078 4344/4052][Pkt Len c2s/s2c min/avg/max/stddev: 54/66 163/258 368/450 145/192][URL: detectportal.firefox.com/success.txt][StatusCode: 200][User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:69.0) Gecko/20100101 Firefox/69.0][Risk: ** HTTP Suspicious User-Agent **][PLAIN TEXT (GET /success.txt HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,50,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 12 TCP 10.0.0.227:56954 <-> 10.0.0.149:8008 [proto: 7/HTTP][cat: Web/5][4 pkts/527 bytes <-> 3 pkts/1401 bytes][Goodput ratio: 48/85][0.01 sec][Host: 10.0.0.149][bytes ratio: -0.453 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/3 2/3 6/3 3/0][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 132/467 317/1261 107/561][URL: 10.0.0.149:8008/ssdp/device-desc.xml][StatusCode: 200][Content-Type: application/xml][User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36][Risk: ** Known protocol on non standard port **** HTTP Suspicious User-Agent **][PLAIN TEXT (HGET /ssdp/device)][Plen Bins: 0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0] 13 UDP [fe80::408:3e45:3abc:1552]:5353 -> [ff02::fb]:5353 [proto: 8/MDNS][cat: Network/14][9 pkts/1628 bytes -> 0 pkts/0 bytes][Goodput ratio: 66/0][25.40 sec][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 819/0 3174/0 11263/0 3646/0][Pkt Len c2s/s2c min/avg/max/stddev: 152/0 181/0 206/0 24/0][PLAIN TEXT (companion)][Plen Bins: 0,0,33,22,44,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 14 UDP 10.0.0.227:137 -> 10.0.0.255:137 [proto: 10/NetBIOS][cat: System/18][15 pkts/1542 bytes -> 0 pkts/0 bytes][Goodput ratio: 59/0][6.05 sec][Host: lp-rkerur-osx][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 465/0 1499/0 677/0][Pkt Len c2s/s2c min/avg/max/stddev: 92/0 103/0 110/0 9/0][PLAIN TEXT ( EMFACNFCELEFFC)][Plen Bins: 0,40,60,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] @@ -55,7 +55,7 @@ JA3 Host Stats: 32 TCP 10.0.0.227:56886 <-> 17.57.144.116:5223 [proto: 238.140/ApplePush.Apple][cat: Cloud/13][3 pkts/174 bytes <-> 2 pkts/185 bytes][Goodput ratio: 0/28][0.02 sec][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 33 UDP 10.0.0.151:1900 -> 10.0.0.227:61328 [proto: 12/SSDP][cat: System/18][1 pkts/353 bytes -> 0 pkts/0 bytes][Goodput ratio: 88/0][< 1 sec][PLAIN TEXT (HTTP/1.1 200 OK)][Plen Bins: 0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 34 TCP 10.0.0.227:56910 <-> 35.201.124.9:443 [proto: 91/TLS][cat: Web/5][2 pkts/170 bytes <-> 2 pkts/164 bytes][Goodput ratio: 22/19][0.05 sec][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] - 35 UDP 10.0.0.227:62427 <-> 75.75.75.75:53 [proto: 5/DNS][cat: Network/14][1 pkts/84 bytes <-> 1 pkts/242 bytes][Goodput ratio: 49/82][0.02 sec][Host: detectportal.firefox.com][184.25.56.82][PLAIN TEXT (detectportal)][Plen Bins: 0,50,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 35 UDP 10.0.0.227:62427 <-> 75.75.75.75:53 [proto: 5/DNS][cat: ConnectivityCheck/30][1 pkts/84 bytes <-> 1 pkts/242 bytes][Goodput ratio: 49/82][0.02 sec][Host: detectportal.firefox.com][184.25.56.82][PLAIN TEXT (detectportal)][Plen Bins: 0,50,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 36 UDP 10.0.0.227:58074 <-> 75.75.75.75:53 [proto: 5/DNS][cat: Network/14][1 pkts/75 bytes <-> 1 pkts/230 bytes][Goodput ratio: 43/81][0.01 sec][Host: www.outlook.com][40.97.222.34][PLAIN TEXT (outlook)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 37 UDP 10.0.0.227:60341 <-> 75.75.75.75:53 [proto: 5.140/DNS.Apple][cat: Web/5][1 pkts/73 bytes <-> 1 pkts/224 bytes][Goodput ratio: 42/81][0.01 sec][Host: www.apple.com][184.27.115.161][PLAIN TEXT (edgekey)][Plen Bins: 50,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 38 UDP 10.0.0.227:64193 <-> 75.75.75.75:53 [proto: 5.238/DNS.ApplePush][cat: Cloud/13][1 pkts/85 bytes <-> 1 pkts/192 bytes][Goodput ratio: 50/78][0.02 sec][Host: 24-courier.push.apple.com][17.57.144.20][PLAIN TEXT (courier)][Plen Bins: 0,50,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] diff --git a/tests/result/iphone.pcap.out b/tests/result/iphone.pcap.out index 5e1779186..9306f6805 100644 --- a/tests/result/iphone.pcap.out +++ b/tests/result/iphone.pcap.out @@ -34,7 +34,7 @@ JA3 Host Stats: 16 UDP 169.254.225.216:5353 -> 224.0.0.251:5353 [proto: 8/MDNS][cat: Network/14][4 pkts/2123 bytes -> 0 pkts/0 bytes][Goodput ratio: 92/0][33.08 sec][Luca’s iMac._odisk._tcp.local][PLAIN TEXT (s iMac)][Plen Bins: 0,25,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,0,0,0,0,0,0,0,0,0,0] 17 UDP 192.168.2.1:5353 -> 224.0.0.251:5353 [proto: 8/MDNS][cat: Network/14][4 pkts/2094 bytes -> 0 pkts/0 bytes][Goodput ratio: 92/0][33.08 sec][Luca’s iMac._odisk._tcp.local][PLAIN TEXT (s iMac)][Plen Bins: 0,25,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25,0,0,0,0,0,0,0,0,0,0,0] 18 UDP [fe80::c42c:3ff:fe60:6a64]:5353 -> [ff02::fb]:5353 [proto: 8/MDNS][cat: Network/14][3 pkts/2067 bytes -> 0 pkts/0 bytes][Goodput ratio: 91/0][33.08 sec][Luca’s iMac._odisk._tcp.local][PLAIN TEXT (s iMac)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,66,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33,0,0,0,0,0,0,0,0,0,0,0] - 19 TCP 192.168.2.17:49152 <-> 17.253.105.202:80 [proto: 7.140/HTTP.Apple][cat: Web/5][5 pkts/473 bytes <-> 4 pkts/968 bytes][Goodput ratio: 28/72][0.33 sec][Host: captive.apple.com][bytes ratio: -0.344 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/2 82/80 171/158 82/78][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 95/242 197/762 51/300][URL: captive.apple.com/hotspot-detect.html][StatusCode: 200][Content-Type: text/html][User-Agent: CaptiveNetworkSupport-390.60.1 wispr][PLAIN TEXT (GET /hotspot)][Plen Bins: 0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 19 TCP 192.168.2.17:49152 <-> 17.253.105.202:80 [proto: 7.140/HTTP.Apple][cat: ConnectivityCheck/30][5 pkts/473 bytes <-> 4 pkts/968 bytes][Goodput ratio: 28/72][0.33 sec][Host: captive.apple.com][bytes ratio: -0.344 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/2 82/80 171/158 82/78][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 95/242 197/762 51/300][URL: captive.apple.com/hotspot-detect.html][StatusCode: 200][Content-Type: text/html][User-Agent: CaptiveNetworkSupport-390.60.1 wispr][PLAIN TEXT (GET /hotspot)][Plen Bins: 0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 20 UDP 192.168.2.1:17500 -> 192.168.2.255:17500 [proto: 121/Dropbox][cat: Cloud/13][2 pkts/1104 bytes -> 0 pkts/0 bytes][Goodput ratio: 92/0][30.05 sec][PLAIN TEXT (version)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 21 UDP 192.168.2.1:67 -> 192.168.2.17:68 [proto: 18/DHCP][cat: Network/14][2 pkts/684 bytes -> 0 pkts/0 bytes][Goodput ratio: 88/0][1.02 sec][PLAIN TEXT (iMac.local)][Plen Bins: 0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 22 UDP [fe80::823:3f17:8298:a29c]:5353 -> [ff02::fb]:5353 [proto: 8/MDNS][cat: Network/14][4 pkts/512 bytes -> 0 pkts/0 bytes][Goodput ratio: 51/0][3.56 sec][PLAIN TEXT (homekit)][Plen Bins: 0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] @@ -50,7 +50,7 @@ JA3 Host Stats: 32 UDP 192.168.2.17:63677 <-> 192.168.2.1:53 [proto: 5.145/DNS.AppleiTunes][cat: Streaming/17][1 pkts/81 bytes <-> 1 pkts/222 bytes][Goodput ratio: 48/81][0.04 sec][Host: sync.itunes.apple.com][95.101.24.53][PLAIN TEXT (itunes)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 33 UDP 192.168.2.17:53983 <-> 192.168.2.1:53 [proto: 5.145/DNS.AppleiTunes][cat: Streaming/17][1 pkts/80 bytes <-> 1 pkts/221 bytes][Goodput ratio: 47/81][0.05 sec][Host: bag.itunes.apple.com][95.101.24.53][PLAIN TEXT (itunes)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 34 UDP 192.168.2.17:63377 <-> 192.168.2.1:53 [proto: 5.145/DNS.AppleiTunes][cat: Streaming/17][1 pkts/80 bytes <-> 1 pkts/221 bytes][Goodput ratio: 47/81][0.05 sec][Host: bag.itunes.apple.com][95.101.24.53][PLAIN TEXT (itunes)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] - 35 UDP 192.168.2.17:51007 <-> 192.168.2.1:53 [proto: 5.140/DNS.Apple][cat: Web/5][1 pkts/77 bytes <-> 1 pkts/221 bytes][Goodput ratio: 45/81][0.04 sec][Host: captive.apple.com][17.253.105.202][PLAIN TEXT (captive)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 35 UDP 192.168.2.17:51007 <-> 192.168.2.1:53 [proto: 5.140/DNS.Apple][cat: ConnectivityCheck/30][1 pkts/77 bytes <-> 1 pkts/221 bytes][Goodput ratio: 45/81][0.04 sec][Host: captive.apple.com][17.253.105.202][PLAIN TEXT (captive)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 36 UDP 192.168.2.17:55457 <-> 192.168.2.1:53 [proto: 5.140/DNS.Apple][cat: Web/5][1 pkts/74 bytes <-> 1 pkts/214 bytes][Goodput ratio: 43/80][0.04 sec][Host: mesu.apple.com][17.253.105.202][PLAIN TEXT (akadns)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 37 UDP 192.168.2.17:62526 <-> 192.168.2.1:53 [proto: 5.140/DNS.Apple][cat: Web/5][1 pkts/73 bytes <-> 1 pkts/212 bytes][Goodput ratio: 42/80][0.05 sec][Host: cl4.apple.com][104.73.61.30][PLAIN TEXT (origin)][Plen Bins: 50,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 38 UDP 192.168.2.17:52682 <-> 192.168.2.1:53 [proto: 5.143/DNS.AppleiCloud][cat: Web/5][1 pkts/74 bytes <-> 1 pkts/203 bytes][Goodput ratio: 43/79][0.04 sec][Host: www.icloud.com][23.45.74.46][PLAIN TEXT (icloud)][Plen Bins: 0,50,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] diff --git a/tests/result/teams.pcap.out b/tests/result/teams.pcap.out index f8bbaee62..c88ad0ee5 100644 --- a/tests/result/teams.pcap.out +++ b/tests/result/teams.pcap.out @@ -95,7 +95,7 @@ JA3 Host Stats: 74 UDP 192.168.1.6:51033 <-> 192.168.1.1:53 [proto: 5.125/DNS.Skype][cat: VoIP/10][1 pkts/80 bytes <-> 1 pkts/182 bytes][Goodput ratio: 47/77][0.04 sec][Host: eu-api.asm.skype.com][52.114.75.69][PLAIN TEXT (trafficmanager)][Plen Bins: 0,50,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 75 UDP 192.168.1.6:51309 <-> 192.168.1.1:53 [proto: 5/DNS][cat: Network/14][1 pkts/93 bytes <-> 1 pkts/169 bytes][Goodput ratio: 54/75][0.01 sec][Host: skypedataprdcolneu04.cloudapp.net][::][PLAIN TEXT (skypedataprdcolneu04)][Plen Bins: 0,50,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 76 UDP 192.168.1.6:62863 <-> 192.168.1.1:53 [proto: 5.250/DNS.Teams][cat: Collaborative/15][1 pkts/103 bytes <-> 1 pkts/158 bytes][Goodput ratio: 59/73][0.07 sec][Host: emea.ng.msg.teams-msgapi.trafficmanager.net][52.114.108.8][PLAIN TEXT (msgapi)][Plen Bins: 0,50,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] - 77 UDP 192.168.1.6:56634 <-> 192.168.1.1:53 [proto: 5.140/DNS.Apple][cat: Web/5][1 pkts/89 bytes <-> 1 pkts/142 bytes][Goodput ratio: 52/70][0.03 sec][Host: captive.apple.com.edgekey.net][23.50.158.88][PLAIN TEXT (captive)][Plen Bins: 0,50,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 77 UDP 192.168.1.6:56634 <-> 192.168.1.1:53 [proto: 5.140/DNS.Apple][cat: ConnectivityCheck/30][1 pkts/89 bytes <-> 1 pkts/142 bytes][Goodput ratio: 52/70][0.03 sec][Host: captive.apple.com.edgekey.net][23.50.158.88][PLAIN TEXT (captive)][Plen Bins: 0,50,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 78 UDP 192.168.1.6:60813 <-> 192.168.1.1:53 [proto: 5/DNS][cat: Network/14][1 pkts/93 bytes <-> 1 pkts/109 bytes][Goodput ratio: 54/61][0.01 sec][Host: skypedataprdcolneu04.cloudapp.net][52.114.77.33][PLAIN TEXT (skypedataprdcolneu04)][Plen Bins: 0,50,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 79 TCP 192.168.1.6:58533 -> 149.154.167.91:443 [proto: 91.185/TLS.Telegram][cat: Chat/9][3 pkts/186 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][4.29 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 80 ICMP 93.71.110.205:0 -> 192.168.1.6:0 [proto: 81/ICMP][cat: Network/14][2 pkts/140 bytes -> 0 pkts/0 bytes][Goodput ratio: 40/0][0.01 sec][Plen Bins: 100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] From 00b27633999ce2a439101ff1e00261cfc8e072ae Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Tue, 4 Aug 2020 21:59:45 +0200 Subject: [PATCH 02/32] Added check on payload lenght during extra packet processing --- src/lib/protocols/http.c | 10 ++++++---- src/lib/protocols/telnet.c | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index 2b96e55b4..eb64265ee 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -71,6 +71,7 @@ static void ndpi_analyze_content_signature(struct ndpi_flow_struct *flow) { static int ndpi_search_http_tcp_again(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { + ndpi_search_http_tcp(ndpi_struct, flow); #ifdef HTTP_DEBUG @@ -133,7 +134,7 @@ static ndpi_protocol_category_t ndpi_http_check_content(struct ndpi_detection_mo } /* check for attachment */ - if (packet->content_disposition_line.len > 0) { + if(packet->content_disposition_line.len > 0) { u_int8_t attachment_len = sizeof("attachment; filename"); if(packet->content_disposition_line.len > attachment_len) { @@ -224,7 +225,7 @@ static void rtsp_parse_packet_acceptline(struct ndpi_detection_module_struct static void setHttpUserAgent(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, char *ua) { - if ( !strcmp(ua, "Windows NT 5.0")) ua = "Windows 2000"; + if( !strcmp(ua, "Windows NT 5.0")) ua = "Windows 2000"; else if(!strcmp(ua, "Windows NT 5.1")) ua = "Windows XP"; else if(!strcmp(ua, "Windows NT 5.2")) ua = "Windows Server 2003"; else if(!strcmp(ua, "Windows NT 6.0")) ua = "Windows Vista"; @@ -741,7 +742,8 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct packet->packet_lines_parsed_complete = 0; /* Check if we so far detected the protocol in the request or not. */ - if(flow->l4.tcp.http_stage == 0) { + if((packet->payload_packet_len > 0) /* Needed in case of extra packet processing */ + && (flow->l4.tcp.http_stage == 0)) { /* Expected a request */ flow->http_detected = 0; @@ -823,7 +825,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct /* try to get some additional request header info even if the packet may not be HTTP */ ndpi_parse_packet_line_info(ndpi_struct, flow); - if (packet->http_num_headers > 0) { + if(packet->http_num_headers > 0) { check_content_type_and_change_protocol(ndpi_struct, flow); return; } diff --git a/src/lib/protocols/telnet.c b/src/lib/protocols/telnet.c index 8e688eca0..bc3211f3d 100644 --- a/src/lib/protocols/telnet.c +++ b/src/lib/protocols/telnet.c @@ -42,9 +42,9 @@ static int search_telnet_again(struct ndpi_detection_module_struct *ndpi_struct, printf("==> %s() [%s][direction: %u]\n", __FUNCTION__, packet->payload, packet->packet_direction); #endif - if (packet->payload == NULL || packet->payload_packet_len == 0) - return(1); - if(packet->payload[0] == 0xFF) + if((packet->payload == NULL) + || (packet->payload_packet_len == 0) + || (packet->payload[0] == 0xFF)) return(1); if(flow->protos.telnet.username_detected) { From 07d9fa7f96d50aea4a1d8ed40330afa7d4944151 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Wed, 5 Aug 2020 11:13:27 +0200 Subject: [PATCH 03/32] Win #define fix --- src/include/ndpi_win32.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/include/ndpi_win32.h b/src/include/ndpi_win32.h index 7b0b37de1..a39a2401a 100644 --- a/src/include/ndpi_win32.h +++ b/src/include/ndpi_win32.h @@ -76,9 +76,6 @@ typedef unsigned __int64 u_int64_t; extern unsigned long waitForNextEvent(unsigned long ulDelay /* ms */); #define sleep(a /* sec */) waitForNextEvent(1000*a /* ms */) -#ifndef localtime_r -#define localtime_r(a, b) localtime_s(b, a) -#endif #define strtok_r strtok_s #define timegm _mkgmtime From 79b89d286605635f15edfe3c21297aaa3b5f3acf Mon Sep 17 00:00:00 2001 From: Nardi Ivan Date: Wed, 5 Aug 2020 17:13:23 +0200 Subject: [PATCH 04/32] Add risk flag about suspicious ESNI usage In a Client Hello, the presence of both SNI and ESNI may obfuscate the real domain of an HTTPS connection, fooling DPI engines and firewalls, similarly to Domain Fronting. Such technique is reported in a presentation at DEF CON 28: "Domain Fronting is Dead, Long Live Domain Fronting: Using TLS 1.3 to evade censors, bypass network defenses, and blend in with the noise" Full credit for the idea must go the original author At the moment, the only way to get the pdf presention and related video is via https://forum.defcon.org/node/234492 Hopefully a direct link (and an example pcap) will be available soon --- python/ndpi.py | 3 ++- src/include/ndpi_typedefs.h | 1 + src/lib/ndpi_utils.c | 5 ++++- src/lib/protocols/tls.c | 5 +++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/python/ndpi.py b/python/ndpi.py index 227db5bb5..85378f526 100644 --- a/python/ndpi.py +++ b/python/ndpi.py @@ -312,6 +312,7 @@ typedef enum { NDPI_SSH_OBSOLETE_CLIENT_VERSION_OR_CIPHER, NDPI_SSH_OBSOLETE_SERVER_VERSION_OR_CIPHER, NDPI_SMB_INSECURE_VERSION, + NDPI_TLS_SUSPICIOUS_ESNI_USAGE, /* Leave this as last member */ NDPI_MAX_RISK } ndpi_risk_enum; @@ -1446,4 +1447,4 @@ class NDPI(): def ndpi_exit_detection_module(self): """ Exit function for nDPI module """ self._ndpi.ndpi_exit_detection_module(self._mod) - self._ffi.dlclose(self._ndpi) \ No newline at end of file + self._ffi.dlclose(self._ndpi) diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 66fac35af..53d143327 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -79,6 +79,7 @@ typedef enum { NDPI_SSH_OBSOLETE_CLIENT_VERSION_OR_CIPHER, NDPI_SSH_OBSOLETE_SERVER_VERSION_OR_CIPHER, NDPI_SMB_INSECURE_VERSION, + NDPI_TLS_SUSPICIOUS_ESNI_USAGE, /* Leave this as last member */ NDPI_MAX_RISK diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c index 347e65d52..9fc5d2d7f 100644 --- a/src/lib/ndpi_utils.c +++ b/src/lib/ndpi_utils.c @@ -1532,7 +1532,10 @@ const char* ndpi_risk2str(ndpi_risk_enum risk) { case NDPI_SMB_INSECURE_VERSION: return("SMB Insecure Version"); - + + case NDPI_TLS_SUSPICIOUS_ESNI_USAGE: + return("TLS Suspicious ESNI Usage"); + default: snprintf(buf, sizeof(buf), "%d", (int)risk); return(buf); diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c index ec267ba5e..5cf2cac19 100644 --- a/src/lib/protocols/tls.c +++ b/src/lib/protocols/tls.c @@ -1432,6 +1432,11 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct, NDPI_SET_BIT(flow->risk, NDPI_TLS_NOT_CARRYING_HTTPS); } + if(flow->protos.stun_ssl.ssl.encrypted_sni.esni && + flow->protos.stun_ssl.ssl.client_requested_server_name[0] != '\0') { + NDPI_SET_BIT(flow->risk, NDPI_TLS_SUSPICIOUS_ESNI_USAGE); + } + return(2 /* Client Certificate */); } else { #ifdef DEBUG_TLS From d3fb1fb25a2b3ab37d4e1964f4fdde70d24349e1 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Wed, 5 Aug 2020 17:57:00 +0200 Subject: [PATCH 05/32] Fixed possible memory leak in TLS certificate handling --- example/ndpiReader.c | 3 +-- src/lib/protocols/tls.c | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/example/ndpiReader.c b/example/ndpiReader.c index c525346fa..03ab1df4a 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -3321,7 +3321,7 @@ static void dgaUnitTest() { }; int i; NDPI_PROTOCOL_BITMASK all; - struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(ndpi_no_prefs); + struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(ndpi_no_prefs); assert(ndpi_str != NULL); @@ -3338,7 +3338,6 @@ static void dgaUnitTest() { for(i=0; non_dga[i] != NULL; i++) assert(ndpi_check_dga_name(ndpi_str, NULL, (char*)non_dga[i]) == 0); - ndpi_exit_detection_module(ndpi_str); } diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c index ec267ba5e..5642ebdf0 100644 --- a/src/lib/protocols/tls.c +++ b/src/lib/protocols/tls.c @@ -316,7 +316,9 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi printf("[TLS] %s() IssuerDN [%s]\n", __FUNCTION__, rdnSeqBuf); #endif - if(rdn_len) flow->protos.stun_ssl.ssl.issuerDN = ndpi_strdup(rdnSeqBuf); + if(rdn_len && (flow->protos.stun_ssl.ssl.issuerDN == NULL)) + flow->protos.stun_ssl.ssl.issuerDN = ndpi_strdup(rdnSeqBuf); + rdn_len = 0; /* Reset buffer */ } From 5b6ffad2788ad3590426d2573d981705702b1f53 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Wed, 5 Aug 2020 21:45:38 +0200 Subject: [PATCH 06/32] Added new ndpi_string_sha1_hash API call --- src/include/ndpi_api.h.in | 2 ++ src/lib/ndpi_community_id.c | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/include/ndpi_api.h.in b/src/include/ndpi_api.h.in index 0fa02e3c7..6c01bb05c 100644 --- a/src/include/ndpi_api.h.in +++ b/src/include/ndpi_api.h.in @@ -904,6 +904,8 @@ extern "C" { const u_int8_t *src, u_int src_len); u_char* ndpi_base64_decode(const u_char *src, size_t len, size_t *out_len); char* ndpi_base64_encode(unsigned char const* bytes_to_encode, size_t in_len); + void ndpi_string_sha1_hash(const uint8_t *message, size_t len, u_char *hash /* 20-bytes */); + int ndpi_load_ipv4_ptree(struct ndpi_detection_module_struct *ndpi_str, const char *path, u_int16_t protocol_id); int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct, diff --git a/src/lib/ndpi_community_id.c b/src/lib/ndpi_community_id.c index 30519b59e..3d7b1173d 100644 --- a/src/lib/ndpi_community_id.c +++ b/src/lib/ndpi_community_id.c @@ -167,13 +167,15 @@ static int ndpi_community_id_peer_v4_is_less_than(u_int32_t ip1, u_int32_t ip2, static int ndpi_community_id_peer_v6_is_less_than(struct ndpi_in6_addr *ip1, struct ndpi_in6_addr *ip2, u_int16_t p1, u_int16_t p2) { int comp = memcmp(ip1, ip2, sizeof(struct ndpi_in6_addr)); + return comp < 0 || (comp == 0 && p1 < p2); } /* **************************************************** */ -static void ndpi_community_id_sha1_hash(const uint8_t *message, size_t len, u_char *hash /* 20-bytes */) { +void ndpi_string_sha1_hash(const uint8_t *message, size_t len, u_char *hash /* 20-bytes */) { SHA1_CTX ctx; + SHA1Init(&ctx); SHA1Update(&ctx, message, len); SHA1Final(hash, &ctx); @@ -185,7 +187,8 @@ static void ndpi_community_id_sha1_hash(const uint8_t *message, size_t len, u_ch https://github.com/corelight/community-id-spec/blob/bda913f617389df07cdaa23606e11bbd318e265c/community-id.py#L285 */ static int ndpi_community_id_finalize_and_compute_hash(u_int8_t *comm_buf, u_int16_t off, u_int8_t l4_proto, - u_int16_t src_port, u_int16_t dst_port, char *hash_buf, u_int8_t hash_buf_len) { + u_int16_t src_port, u_int16_t dst_port, + char *hash_buf, u_int8_t hash_buf_len) { u_int8_t pad = 0; uint32_t hash[5]; char *community_id; @@ -209,7 +212,7 @@ static int ndpi_community_id_finalize_and_compute_hash(u_int8_t *comm_buf, u_int } /* Compute SHA1 */ - ndpi_community_id_sha1_hash(comm_buf, off, (u_char*)hash); + ndpi_string_sha1_hash(comm_buf, off, (u_char*)hash); /* Base64 encoding */ community_id = ndpi_base64_encode((u_int8_t*)hash, sizeof(hash)); From c2156a516193ada6fee37ab6b7a26f712ee02e14 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Thu, 6 Aug 2020 09:19:04 +0200 Subject: [PATCH 07/32] Added note on memory management --- src/include/ndpi_api.h.in | 2 +- src/lib/ndpi_community_id.c | 4 ++-- src/lib/ndpi_utils.c | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/include/ndpi_api.h.in b/src/include/ndpi_api.h.in index 6c01bb05c..e5d2ffad3 100644 --- a/src/include/ndpi_api.h.in +++ b/src/include/ndpi_api.h.in @@ -903,7 +903,7 @@ extern "C" { void ndpi_user_pwd_payload_copy(u_int8_t *dest, u_int dest_len, u_int offset, const u_int8_t *src, u_int src_len); u_char* ndpi_base64_decode(const u_char *src, size_t len, size_t *out_len); - char* ndpi_base64_encode(unsigned char const* bytes_to_encode, size_t in_len); + char* ndpi_base64_encode(unsigned char const* bytes_to_encode, size_t in_len); /* NOTE: caller MUST free the returned pointer */ void ndpi_string_sha1_hash(const uint8_t *message, size_t len, u_char *hash /* 20-bytes */); int ndpi_load_ipv4_ptree(struct ndpi_detection_module_struct *ndpi_str, diff --git a/src/lib/ndpi_community_id.c b/src/lib/ndpi_community_id.c index 3d7b1173d..72f60c746 100644 --- a/src/lib/ndpi_community_id.c +++ b/src/lib/ndpi_community_id.c @@ -217,7 +217,7 @@ static int ndpi_community_id_finalize_and_compute_hash(u_int8_t *comm_buf, u_int /* Base64 encoding */ community_id = ndpi_base64_encode((u_int8_t*)hash, sizeof(hash)); - if (community_id == NULL) + if(community_id == NULL) return -1; #if 0 /* Debug Info */ @@ -234,7 +234,7 @@ static int ndpi_community_id_finalize_and_compute_hash(u_int8_t *comm_buf, u_int printf("Base64: %s\n", community_id); #endif - if (hash_buf_len < 2 || hash_buf_len-2 < strlen(community_id)+1) { + if(hash_buf_len < 2 || hash_buf_len-2 < strlen(community_id)+1) { ndpi_free(community_id); return -1; } diff --git a/src/lib/ndpi_utils.c b/src/lib/ndpi_utils.c index 347e65d52..00583dd26 100644 --- a/src/lib/ndpi_utils.c +++ b/src/lib/ndpi_utils.c @@ -874,6 +874,7 @@ u_char* ndpi_base64_decode(const u_char *src, size_t len, size_t *out_len) { /* ********************************** */ +/* NOTE: caller MUST free returned pointer */ char* ndpi_base64_encode(unsigned char const* bytes_to_encode, size_t in_len) { size_t len = 0, ret_size; char *ret; From 2722861d6e79d416d3377af4cf6fdaaba2a18de4 Mon Sep 17 00:00:00 2001 From: Nardi Ivan Date: Thu, 6 Aug 2020 10:29:35 +0200 Subject: [PATCH 08/32] Suspicious ESNI usage: add a comment and a pcap example See: 79b89d286605635f15edfe3c21297aaa3b5f3acf --- src/lib/protocols/tls.c | 2 ++ tests/pcap/tls_esni_sni_both.pcap | Bin 0 -> 16531 bytes tests/result/tls_esni_sni_both.pcap.out | 9 +++++++++ 3 files changed, 11 insertions(+) create mode 100644 tests/pcap/tls_esni_sni_both.pcap create mode 100644 tests/result/tls_esni_sni_both.pcap.out diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c index f96745dc6..883de7666 100644 --- a/src/lib/protocols/tls.c +++ b/src/lib/protocols/tls.c @@ -1434,6 +1434,8 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct, NDPI_SET_BIT(flow->risk, NDPI_TLS_NOT_CARRYING_HTTPS); } + /* Suspicious Domain Fronting: + https://github.com/SixGenInc/Noctilucent/blob/master/docs/ */ if(flow->protos.stun_ssl.ssl.encrypted_sni.esni && flow->protos.stun_ssl.ssl.client_requested_server_name[0] != '\0') { NDPI_SET_BIT(flow->risk, NDPI_TLS_SUSPICIOUS_ESNI_USAGE); diff --git a/tests/pcap/tls_esni_sni_both.pcap b/tests/pcap/tls_esni_sni_both.pcap new file mode 100644 index 0000000000000000000000000000000000000000..7d7834798fe739c59b68181a9b9bc7c76c6347a9 GIT binary patch literal 16531 zcmch;1ymi)wk^ET;KAM9J-7yUhoHgTW#jG=+}$O(1$PhbP9S)2claSWSH5%ZJ>&o0 z8{+}R*xe1YXLa?Ot7`S^nVy;)PyjgKzrMf$Ag>#RX1JeWPC)^(ub@?i`j8PA;N$D|Tl<6i=N!M$S>Mn(Ax*R2kl)S#(y}mOz@~updWLHs zJ?(%|3?%|M1blj0Duk#0_p?;0JVGVd4Zvk1toB4(n?E!C6!sHmC14@ z@GETiH!I*-r!UtHVgszfDw3=+(vAoN%=;u&000sN39ZiiK zjOg_2jp(dx9O+)S=nU;`%nTjqTy30QcUs!mSeRLx&>7p<(;L`Wy^MiaqksVL0EElD zw~T4J#|qIOGDt6F*cR=zPw#AIdMm?RW+u*nrk?skdAW& z4x7!i8#=)hxcaS)jtHUz)j%PYP`EEW@-(mfN z`$Dtw=NscV6z7oI&Vs7r9&3PH7}9Qiwam1)eAo3kys2LhVf%xlZoym(s7e?49nrIe z?fL!yRigj>&fSD*HD~kWiETzPks%G>-g~koU-r6ZsUK0Omz7wKPwl<_Qho;j0s!$A zGynk1>o?TvrN;nZ0~`60gd0ns%9yi(Q=N# zj39qRfIofFA9v3fCI$Ck@4Fzs(TqOsw79HENV=jI=*nczf#*>*%Xc*xRyL48RhYI= z61d0P%=;WCQfN?0>2W<1BCK3eb5HMh57V4vePI%V$(YulJ3f0=R-jRk23lbcdJ36{ z$mP2OZPf3cvNs+I7oFz-`vkfR8#IX=aluFlxprj|gR`nh~$R#b3Kx*BF{U6eZpJ}W+9hY~ND9Uyt!*Wy3 zp2X`FQ*oNCmAh~%uH(_ehVd3W+99Ug6S7~z%H(8i$~;9qxa;$;)T1nj)3yqJg0=933o_fvQNW8N$4BuObLAjFEZK=IQvJTI`y%FBg4Em zVjMvjvAML5dYW`Ogd{PPcimJEJlueQBj1Vd72+zM12`p^nWDzQrhq9H;Zf+vnxBM; z=XmB}^KihC%^xZS#dY!FNMKNs_3F?Dak=QEh?7`4m3NDt=IS+HS66pHDY1~wxfePd zRv4=NTxOcX{lI-=!_t~hNdGBqAdb&D(1^l^ifBI(KWPfPEU=usH2y22J3;8j8m!Dw zp<-_G?*JXO@X#Eg7)4Fb>xx}&wHIPyec`XSKPx#bHQ-s>NyU`&(oxNG;(unNgUZcg zvAWe7K_S#l72$={2v8q5Ysw;$otvu$@88239`K*ten;CCBR5myk8Eght;J`Iru3YK ziV*uX9}}E)R+stBmSp_co1nepma3QZEAdUni6nvBJHA$qyJ9*7A5LC}CqjdUu_1(m zyksY%OSS#w4h@q13T9;HSg=T2P@|^VlJn{{Zi<3Z(e*p;-0EfP%3L z{5=VgXPpXDXz=hQ*whz7K*$fo8aDs)$hwKH<5lkzx|41m_+v_VD?qGP%f+XIdr(QZ zU8TMFab)VTC?kgN_7Ey#0yU&n6yit`XflYX`?G~~aA4DgqbVZZFNh>l+{Z_Q*B?eI zFn)qKcE$YqB!*6dPTm+L8Fkz&yu@0_18weG1rvhCHB_=Zdufm|f6IyziGETK1~Q_y zu3nJQ74oanT=O}7FU?)O3|V|a*uY_g&3r53x#f~e^h%B~m@B`Cmuoh5RB02Bib+-) zQMv)%JjCi#__xWfSXp1MSvqm?o~mT3@py;MJGRw`xSZbd$H#xK{K&tRfBUb>uR8W0`Q!X`YKTFd8;3;+y#Nc`io3ww#K zRVw!S)i2a-KZ+Y)9?sF*<0K#cAOm;v5A*hB7?k?}6T&{Ce)`zV8NG+ZXj34G;rL~a zn^|ipD3M{2ILu{B^xO|~;(Nya!t~~y^sO89ZA$Z_cqJa(sgqHS@HYWG5u{<4?>PDO zEt9xP)aN(=st;_7;ZGSTg;>%6-mDD4I2A4zykR&KS&ht$5f6MtTk-1?^D&W8Xv8Vk zEfz1|H4EME&YVR?sz@(kBFz!&A>-2BE12cr@fXV>1Wh z25OOLI)5C2w-r!X21k`Z%amuIbeDK0!95gC1MKF0F(+ZD*umFkJs;#bExqVrgn|kRR1zW%zR4ahb4O z@R&wx^n2_(>6!JSfDXSw<=&6a7Pq*SZbiF&Sa!)$JyH}bnQ76xjdBU~JbMOM5z0&EV3;m6 z+`z&;FLoJv3Zb*}Cg$XDM=CmRzi&ApjcaO&{|0T!wLK6#p5xtBR!aPRox(2gh%XzU zQpOxKJr6FEK`|bcAKO@_BTY(0(xK~x7|AF~xF{+T8_YR9+Z+A<5d@h1*;UwVxIbHr z#Yyu)V;8%zy%Dj%eVJmx=VdaSf1Xd14McNjPu`;LA;xuR5B9nR0UDR=9G-9~_fn8@ zzC`8g>O=Aa{BHI+=&2zTHDr4>W{YOU`hnN@ylq#-)+F@V%)b7<{XR{B=XKxSM8wb( zP)!^h;;WM%5I0fC&BnaIn19$o=Bzfl?j_jd`CR+^K#_ zv|vq3GQ@+SFXNR2%LWi($IpAgE~8OB8(cDWt&n@*)VU~Xs*S4_-U!xnTg)O3eY|kt zsM`%qX2>EGVu|e8hEvGKi)=BG6q8uA2>Kj7;G=Rx01J7>$CJLZGWR0w|8D(BiZ!n`U;I_k@deFRU zLDo$Ea)Jya(N>_@+yp1=@Szkm-4-Y5fxb&4<;Q3ZjND>XOu!s4I`Je-wHVaO1>LN$ zlakpOg=DSvk}oi@$R?Q71BV{tOMCbX!8-@haTJB_vPD!?I`=o_7kO2Fg4a^OTl?oZ zKRD71%A4{>SN>J`wfuf7KPKSg_g?^jTnG`ri$JSmOq>6p=ID*w_r6Kh2TD&a^x!IK zkj2b`j3E*mOnAXwsL9Oy?3n(taZ2E4t@+ZSt8$sARPkfJVT}BhI=vt8--<2vLP^3F z)qt=@|;N?vD>EmXm;Dy)3R%;DQy zCcEnh@+E$XFXZ1#upKZ$GegG?fBESL!EJl@^JN{uXU{5S&*&g5tu$(-Poe(HVc@m* zuUWnJ)QKa$3ypf3Z6hHgs| z(^R9Ie?=5w_>Wc}KY^Ws-}cPGfXdhhy!?WmGJ}1IT@`(<(DGAFo}|+5sall03tQ4> z)!h=Pf!ytUKc2@U#WqDhkBk*QP&_3)e*MW)9m5e7fb?Mpi`<^Tf^!0%kjvTqC~*N=P4s#C?tm{qQ>qUvPyUmB=(8ECvyW zQCY024dL4tfNg}42)I{1+G*Nh;SMMUt}OJXs9rQ))8T} zs)I}O{b>ON*`u<|1v77!{kAqryM@^iI~zM!a?}@Cp4w-@lEYALvv=sAIp-kF%;7+d zMbD4sQ_RO#zKg9%KOgxNC~+FND{hC^vsLz+d5vv`N{Z-qnvkM}SlyxN%)y7gXlmp1 z2!H3jR|zhRk`CxS{{+geS@Yb~{;0?rK;9HGykUJchT>U_rtkmGw3>zwg$kRh^LU+K zr^t$4$dyN-bZgILSVzOTT$B1me=3NolzKd@jJTu(q1A58d|ttVBulHn&Z;Pj2KjnR zF|ZQgEvq-I+a@(|j2U%(s?rB1UD1Xq!)KVZrHz;$6}i1vwx7Jz@d~Z4%9V_kYjO!g)_f?U{!{0+ zcM=LgzoP^+6gCk&-vXNIT7Z1z%KkKjmvVe?Z^x#dN|FfaqoP8r+?!l8jk)`Xz2MrMA5Z)Md==B_=EC9_J79Z4FwS};Q9xpoDv)Wd@Z}YwSOrf zn*O7JX!+OD-&>DfQC>@ba$f;}|Exj)J`~xk)S)wNK?g2f4&USwGwu#i(c1F{R^=B2 zgmtq`?=0p>_H;q70GNNtaG(mj3uGCcEfXY_}Z|z@Lv!Pn zLYmE-lwL_>TKo@p7^jfCmy>~9PLBuGs8fb0U*}iB=a)$Xy2v9ZzS|yCs*?Tua7Lj& zO#|%LbyDoq&6ih-=%HRWj|mmhd!uiY{a-`z&shXOIfMGU{*?6BETU`innkMK>Q8ew ze}*C<_urzR{YFXrR}^Wg-zZ*xP(p9P0R^vBthe^hTp*HW(BDv^$p1%_*Nm2&@cU9i zm?sB$1um3F1ln8+3tyUCqf~30M#0+bf&*$)lY} zcA@{QTu~0poTIrtaw)On$7*Iyp`4kZfUH*l2$nwP3fa=h6R&a~;3 z5hO(Q6#R*$N`J^~R{hsGh=d;b0^xgwsggD!<^NTQt0J3zFg zGDJo6nRWcV@)HBtCHw6Cfb=lC7kaP2%QcY(6EaG^T*_uAf`tMXfs0a_SiFMnd$=HB zF`~e{ux<5PKE*MYhE@l*s)(olQYlrKuOb77+n<7=zA=UVlKlm2K9YkebS1m%WfVdQ z>4M%FS$pTG9EbDnN#n?(B8s3b{e_TPFz?J;DZj4LIyfaBXzEvoEKgfPMs+vh%^a}9 zO(9%MJNs>vdmKofwgFEFNzZuX6^XkQeW<+oI1VC zmH(6&8H607Bja064t0qknKsNhTzW~D$>F3cf;4%=9QKTGN}Jk!^I&$zJ?h_%t~2z9%8X{Jm9J*+3REcVEs%@Sq zj+3LE&YlvXT`&9#Q|$&A#FJiCKwnUf?HFQk^>fxK%e(LPR+^ab%q(og;}-4@vN#nM zMk;MEP)}QoqlrA$3FLYDaUZJ}MW*e<0x-Ipy4j@Cvavfed$=oB^5E{8snKoVSu4HH zSaxi4Ngmb<*}~?;Gq+r}mVgZ#2MM@~;xIP}siEPK*0M)}a~BU>3X+-%;G5PJcV5rZ z?FJ?gc#z|vjXAR>j+>u;QOALMczgALhy$ha_g1$giUz=9GjeEjV4_HGYc3*?ajlqX zP*{BMlcZI)t|wu^RqWS-npoK$WEdCpj-BXGy}6qahLFE%3f(Ta8^OU#a958mdAjww z56jfd)2K{F7^p2Y%~M>`26IY_0{gs?yAHgK^4eyI8VgVwj(&W)ko%QHlv{wm92$EK zYu3+{xkp@UkCgv;ZkBU@chzIx1WGBCc0Bu`R;o20y2NCB;U_ZITLNi+745$}uaKrP zU^7~Joe})%GWjF@koVw!`{Ct3P+oI6+4!FDjH6|K_TcZkqhnMyGo#c>T z(f>9_!QUK_5CL5re>fsN{M&~t7;ot*a{aIA$!G~J<4=-v%l}7uTB%2+76IYWFJ1eN zxy`1)hihrhTrM8Hc>&=?#pL@1vUQgQ2VFP2_U3)XQ`?1Ef_~DocB5eRtWA;%ATVYm zb4|`67pYHL3B#R2C6Dgd0GTeKNlOuP`HqtIZoGol%xFO!yQJjk=(Sqp6F0_g8_qA+ zec^rKIuKl>PfNLctD1n)>bk=hQLI@f#*Cv|kh<5Xr@u+x&Sg|}M7?ME(fpFtiL~9J z_L3uGU|scAwpJ??Rfrw2=INv`!rarv{<1G2z(1Lmx|1DQ1gOcvici*4LitU-zBzC2 zsjV06BPd%FnQL3Owp!XkvNzEdmSLDd=7v}B_q6pG{-XTCTBe`9Y6&l`{i2$Wm@J&7 z_DkBX7Z4&Z9$@_ZeeX9?2JK&jV$;7((S?e!-kA_-1nw?5+-xXtID6wzt+B7ng_V3S zzjbqO#50C3O%o8Y8ZfjW;S*uN3Mh_d#+^De;9`Q0Ptla$}9145~23c9YxUdgU zo*mR);rU?=SxDyOBwXjxg)@_VMa|w%@KNaHSy_}lA`n3idoz*RVSU6SIZFiL{Ys44 zgPs|>$i*VK;5fK;nbiVEO8(f!{rEDK4i363`VxITQNb*Yew9!4MyFGHwMIgT&-xUF zO(S!i_145s=ZKsv6nY_QiKjkQ1WAfJI4Ub2sQEH?TE4)6k13d^iQC-~UUnRID8qHy zk@+h_YxMg`)GcwUb3b4tp)g%V)asO%(qy){;>CeaCJ6Kh0-{<~ymAG-z$+w2SWJbx zy34zg;Z>^H(CS%k%r>Fk0QZ^_sX^Jc_FC0Xt*`-GWBztd-tgyDqO!dlOH=Xc^Tqm5 z9dUO0r73meX^)I|39F3g`1UdP#HX}(r@~l+0qeot#t&X?#m(*oW)Dv1K{nIUyj*R!YAs+_|$#nv6mJW7$RS_JJ5bMlcomsXJP!{;a`kW?&p z#g@3BRJlK9U&Vs#oHTIQQ4S3664h$=twL$7S@WUY zIQCMvb16#8a{fsVyM9IB0HzVPiyw(F@!W3KUXIRL>_nxUk-HDjyp}DER}5D2d%1oguQHMtcw_rO2f)19L@-mXGed2|jE zLq@(zl|DB*BpHGc0>52jc=D;Tj(RDbUFUUS1!0at{dB4LtD)TB;CxRhW7|Z8=n5`J zOpDuWpzn;Yw6UT1_|tZzQx(xR2-~{=wHh9Y4*}^Xln|&@Q~vR3$tV_WuQ`s13ij`0 zrzMb?V7gJ#Wp0#GlFqORe^$dn@)*9K*f3?=U zA+n~z{+)dHhp*u)YR%jF+UxcYuZ1*NK>aJnxAxB~ThO9^SZnD1b7lMAQC@R02g@I8 zjoYg?1_%bGO*&~WvaMf^nvlz#m2QCJQspNeyi_%+?`jQ!^POM~k&z{|)kX&KI?xMm zYAh8i<#i`U z;83Jm2)SBzNESu-e07)!w#R~r*gB2c!%R{CSq{E@rV(X`LMlYEt#e99nChTj-Je=p zxqem?#oxFsH|loHG&j~yD@TNQ_;U>x_=u5gx>dSnqC24?ym#-(VFoywq`=27GY_GT z)DK@*K@jB$Pc=z;<%mo{p9=0qA10}qb4K#>Np!OCGtf(Hc4<#Q%JNzIlU60BI5M#- zeZ7_U?Y7RN*J2JExjq5&YTc7|tXo^6E1tP8;#Hcd4;$*x@z8*ft?l?Sa9}aBK*diQ z_?8k>?_~s@tbQIh8{Ak~dBzH|vS^kZ?4jgid43%{F$OFCHh^7eZu51K21VqDQf@ql zNKvUd^mva8PgJXYxp>Wy7I>^|#>16Y>)QVVf6d@RV^ir9>2TtUFi}_8=2eYjAXc%* z%aQ0U8AX~ih3!B5HE*FX%K-PEq43W&1psAS%-f`G7W=eq)12pJiNST*7tx))q3XFS8)HF{5&<<);AJ3JNO8+exWj@wD{g09+98~*z}c& zDys6m?v&U21kA*jNNa4Uu~a03j%JlF@6*&zx=A?~-iFavlkoBDV=c45sL+px$COGt z^1s4p=i$eP%qKgw?dQrGlq9qkcbVgwAxP&5!#oHfGdel(nIsimlfZ+9P07fYOeQV+MM-a^7r98L$?cAHV1tU7k$bJd zLg{!Eej|!3y?dw?k{@1S5OL|$xVx2sA$o(GZ(pMjT+};M%$9|$x45}CB;zC~8GDIP zH}ZnzLlgQ{+G0142iSqqccvbE%`f+0Li$|5;K&{<_Im{5RZE6tO8`wg+fQ> zU!-6V@Z_M4Qfr~!rMMEFg@DWi z4r??^PtB@QDR$5Etpsb;&(+CpCcPh&`gN@bpb$J`)vYu~m)44JY)T3Bk-x0(MO||Q z!_Q`}0};deA~sj=0r0A{`*qzq@)3o#fX$W)lJ=)&{=A&iPg7|=`CQ;xWH1nXVZhHd z6ek#oqQy13_XCa$&gg21mL}y>xQYwyPT`8k!ivi_Xd0YWP9Ui**}fu2>@J`3bHjDY z#-#RYjy%Yz2ouk>fQXT>ot8P+SA-~IRnt4JwLD6b_wj9^kr8HanSFDyKPLo}6Av41 z{f()!WxtCVJUNNw<#}~*rZ3L`DqM`F>J^OrqYsy>-fW~)J_cdI;J%hFAZ zRhmyBv*Q!v2SUr?*y8R~U8Fw*&4`ajmtadh*Jioav<1?j;qa}%g7}8W7Y}7Ex<_=_ z;bU{3hDyE4QrSm^wYN)3zLHy!IJl4H61U4}$BV{LHK-el%FjdeY32-Ns~K>OCdvk` zhL>NKMXgFBPWhkpF8+!XKatks475^cl4X!vkTQ{-wDt+PIbnE*=r^qIHx{O9(!jJa zd2FMlr>uuiRf>$}MM5Lw`3(nUOp(je^q{3 ziCCes}dhZF5IOO<*y zSaLQ7t|U1plD@zm7tfU3z(CUZVEb5>a1E8#nc&)$i`H*Y#*%_W3o3Kx=ObucXpJ#Q z4QRs~o(UwyrKK+g@tCoZa5C zKiXYxrIk)Q0uqNY`c{@>7vkVX%>Xt$?A)8A;%mW5g!QSP7vUp=RHZk*N{KsiH0TQF zKd6iw_tEP;v&MWS$?#mM!^LJXH~Vt1eh(qKypxw=4J&mxIU7>sSmD@wYL@K??nTnH z@?gCGiCLCmNp|Xv`~kK1MV#8OwDH9D6F6@LRa;guiV{X<`s->hVpf%FsCK(5PpRq4 zvTfxL#cGfpbD)DD*s*uQVeglMqjST3K-txI%%tqnhMQS7qdj?h5g3|}37yb(+C+;u zQ(BzIC2=}+rbJikxX*#B^VsfbBZC%wMDV#N@(&3^yA&Hwyn?!^>5sOlhPZ_}5ZAZi zUI=tL*Sz$lJ~&R!KKI`QTb7-W9)`pCFC)>o6*dpL0!z>Kxnz>$j&^?FN;g9&neqPI zuG}}$WyZgS1q;i5`C-)-0VKZ7-TfG(Z_MNsx0Iad0cW6U-Pn$RDG1A_pgW!wp7^83 zf3rUMtBHd~>bs-~N=cwY=La$;keq_XqAy?v@QO;)O>o^$>zY}&v!$si&22ru<`GJD zs_|uSI(F%gxo_Rq8*;g>Qmayy6*jx{Gf9}wVREcW+(<9>TTRf3husATS#h90ga`r6%|x z!wa`fbuPm!NaszPk(Us@F`&Ms&MQE2VE_-R=YFt2~y_m8fj2^&1FihD@DGZ>KBBn)!-YR zaM7)9Un@=@zMr~G`gRA|=RQxwSYP9>LNF2G^K*2_#s~i*Fel>6#<8~?*I=U|d$MX* zAm4c>+~7z^ECol(DZK2jM>LnML`zYnocm48hD@W10aw(c9S`wC_tCL>efI`50v!c;(mRjf8g|_=rW9WbDvk zp0t=&2FEHRJbLhER{9C2WKiT${#IsH`;}gzrAmIJD{IxUwFOvHQ$3uzYyPEk#YVgr zRFm=RpvJY)2Y~(<1-*$`0-v%V$~xw*dg2w`auIvgU683d5GO$vX0$p2ugX6+V34q@XZsu5cJ z*{+W;``8l0TR!&i2kq6TPXBr(7kJ6N2J!yR5z9gf}1Foo#6GrTbu2i z$jkQFRnmRE@JcP`nGOY_ChVUTBEE~Ek*%sTy^rH#DiGF*jphQy9K>7sBo|IJBHlTf zt?IifJ?HLp&;Di&;&9(FeSrW!Lq;ZYA;|liya5P=lMYX#$kvcB_@{3Me8R zW=wa8HcMT0S&)LmdG?SVB>gB`^O8c+#fZ0U$kW1U~d7wH?4xpzPryI*z%qi9v&Z6j5LqjLf!WIG8paJ5*MKbQMue zH`3#~pXNV>yAQF2+XsUs$%z%5CdX4~11d|{!G&0XiH9p=(8 zfyX!a)%=0ZPvXMxi&h#gM%c#^><8gdi2<92gm1!8Q_xdICX~$8AsP{7_MLF#udiol zw#hzMG@ls~)mi&}H#QH6u0KN0rZXeO)1Or-JsI}F3srMgS$V<}De!w_YThq9T&yJJ z*{mHFMkCg_8V?^k1jhAKl`2}}p4!Q6x(QfLIT-8J}mVbNXvJT5!qDX84e!x4!& zoY$4k#NMy^Nic*;weaI?wy$}#(_($Z1q+Trh5L-u*0A2umA1aZNV|<;)s8UsQ?;f} zB)oUWXSY}$XO>EH2vzb3EYjm)+N~Bz6*|&O%J!3DC9=h)#VZbbQ-4Of8Ho=rO*4{m z1W9l_pq6BltvhM_q15q4D| z7l_%5^+?ASA|3ZWyvaqut6cnzXH37VWoMg)sm!mhv%TN9dwt>U{|1E(>=k9vO70H| zx;Wh5amM^7msgb6%H3f3Uz{-^aFV1AX&SVC4q$Q8x;AV2*<6%#l@i!Brm{Ti+PzSm zd*fftay3ih5!Zo@v<1!Z%fswbZ+!`WW$`zhF~3=ylNhG{VUhakIQs33Aul2MCdxZ* z+|+~w;$51+v3ufP*e$-^d&Ld`>-Ljl5E}Gu_GGTNthQ!8;sl6brhKSmV`^X3bRNzazoMr<`+4d4pN2!KUWSA2lycf zh2d~V?2Ud#{eKPU|4=zf_+2?d|EvD9ft}l)le|@qJPQ8|#jl2ci<0;o<@XzzZ-vy? z_aE1F$y&36bhHcBOS}_ z*mn@Tl1L}kI0u!trV2?>Y)z@U4P!+;XSLjZRPJeHL?GyFaqngE@1I)W##6Yv@L4Sp zYiQKdap_`bZ>By6=qu&vQbPBC^lAPi9YR1PbT=e>)-8d?=3N=3so0!8KDAi6TUb#t z!LL9N2BtJ7Dm8)&e{L~~ipuu|C59Ce;jFLF&Gza2z&9dZz3;|eW$R^DV;i#E6b4rt zRa-?l4wpp*RJLO#Ou&qv&=#3c@;{1f#H;;%2X5aZIN~5F%okBOj?s1`s6s>IC$LN- zW%V*pZ6vNhrHDlYT7N6{`B-Ad6c?;~w}Vu!f1CE+~B7ys_$uf zZQ`?ivWM5AVUAIi1m}ZtN(RruQ{u2>b)NYM@rEwXk8Or%63np{7gEl(p-o0L`0T4p zinPt@Q>=ByOZt7cjNS_zXqCiXY#C`m$`2D;NOUcnDR$W9N2z(7I^^@HJk!(fNAR15 z{4X|V@n%qgYBRE{7f~GAW8XcNFcbu@@ONr$dB z+=rkH&GJI<9HQ`V@*MrXJ6F9lmhR^!-T@OGW3IAcqEt#&{bhG)^9N+TTB%y?!js2J z6^$U0We<>33c(U2r@jR1j=lEEa@0u}1fQnc`sIbDY4Sd&$}q|ET5sViFZNDpYSg>+wE`0E?qJN(nk4S4>1@ z8X-9C;WpA1^%g!(`_N?_wK#B2NhrbwyOZ=;BFJFNBsf;t$COAz9~0_`+X1+IWS|sC zA`IHJwaV!EF)KCYxey4ZZ*3#?dBV>je24Mn*>2DRH(y`d%iHiydX=k5+z9Bc%Ak^L z=Bp2^!gu|RVUbaaIa-Cx0XDp9PqiNxgl6e5c5c>1&_X5(@LxakwF=Ta? zPM(}S{#hl@x4ytE5x#0r=X4#$vR#vF*A~86>A{oy3aytSD>{UTF@LM~`0+dEH7zoG z%^=lPku9)m)!76{EsLee;xe?xDLmi#nUMPJ1EbWZ>DL)=Gi!Ggn_B|nU~2T_)ZFcZ z2^QtU!t;w#oLC+_?*_bcnjyKTJ-!m3p$%I&)}IpUr8A^?eFK|aEq6U2Yg!^ff0+*x zN)hM8gg}jRp%aGMOiggvBo<=>NpjLIQ+sJ^gXI$Kgr``;*=i|?y+Ze*f-(wvS%t=I z+;L*c86YGB#|EGA%V#i9@{s5FoPv0WpDaXcL@TlWPzH)u(o;Jw;=cPsdy+~)R;=duu&tR4(TW$@pM$eUsKy!7C7- z5mj`ra5Gs_C}+J=^>lZo3u?Kn(XpD&Ag`|RY>+>HS~qm3Q}b>n;v$XsRG>7G9H;;Y zNM3Jl5C_YwGm?S?4Bs!>#M3m#HdT$J_s(ztB0>%}IOZ>7%bAE(5l(dM9GF2xxEvmk zZ6+5>{q_nQZ}BhN#7Xe1{F#L)8r*~MK#ZQL7(7aueC;)|) 104.17.175.85:443 [proto: 91.220/TLS.Cloudflare][cat: Web/5][11 pkts/1461 bytes <-> 9 pkts/7270 bytes][Goodput ratio: 58/93][0.13 sec][bytes ratio: -0.665 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 15/10 53/43 21/15][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 133/808 688/1514 179/685][Risk: ** TLS (probably) not carrying HTTPS **** TLS Suspicious ESNI Usage **][TLSv1.3][Client: these-are-not-the-droids-youre-looking-for.com][JA3C: 077d20c3f8c5a1f091dc937c515b69c1][JA3S: d75f9129bb5d05492a65ff78e081bcb2][ESNI: B8845D1A37225D055CB7187B6D7CBD852ADFDA5269097680CB388AF4FC9F5C7BCE03772D5259820AFC2DA5C949A663F997D270BBD2525D0D7C4D83E6FA9CFA038C1E78C2C847BB2033853998E7D391737C1CBB3796A9F7F24D5E88F6C5AF94E95D93C2F8A168E73F18D090EAB69D7C689AFA7AD9BCAAFEFCF496509DD4DFEB3E96CE334F2B00A6C03C1F9C1BF5040BA031E789D03185DDB6BD2D2105A91463519A23CAAE0295E3F068B701D99B1AB486583C7254DDA07BE99D50C23E4681CB62A5FAA669ADFEF76693137788B3C0A5B0EEC36E004F8A11E7B5B14DD37F50C1F6F20D68828620BEFB7460A5D6910255C126F921FE6B70F2E9C7299683FAE6F9D068B139BAD1EF709DA821642B00FA7FD1BBA44EF6C3DAC61043C434224F3E570F62DAA4BF][ESNI Cipher: TLS_AES_128_GCM_SHA256][Cipher: TLS_CHACHA20_POLY1305_SHA256][PLAIN TEXT (mw/KUc)][Plen Bins: 11,0,11,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,11,0,0,0,0,0,0,0,0,0,0,0,11,0,0,0,0,0,0,0,0,0,0,0,11,0,33,0,0] + 2 TCP 192.168.1.21:55514 <-> 104.17.175.85:443 [proto: 91.220/TLS.Cloudflare][cat: Web/5][10 pkts/1412 bytes <-> 8 pkts/5756 bytes][Goodput ratio: 60/92][0.12 sec][bytes ratio: -0.606 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 15/11 50/38 20/14][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 141/720 693/1514 188/676][Risk: ** TLS (probably) not carrying HTTPS **** TLS Suspicious ESNI Usage **][TLSv1.3][Client: you-think-thats-normal-tls-traffic-youre-seeing.com][JA3C: 077d20c3f8c5a1f091dc937c515b69c1][JA3S: d75f9129bb5d05492a65ff78e081bcb2][ESNI: 4B80F11C3E3E40385229D888F5DB7398460E5FF5EC9E03E8331810BCD314C33227B55F4F0DADD4B813C9274B884ABDC0D2434429EBB559832A5D54B5D55B138366BDA28FF8DE68A292825CA14522CB5FB23A04CC654E9C6D9C5B967B20520D7FC4EFEC9D04154A40428DD4FB89742AFBAADD01105020F05B23C44F216802FDA5F9CDFBD129BAA1CCA4A4235E9F1E9D16A96FE72CEF01ACA433C697DD49D2389E1AAF817F54E971E4F290DE91ECB83A5876A3B37B97E66EBCBB90AEFE3BF39455BA2AECB7B4161D157606BCBE1E4D0C0391D57652585A1E6C49290F4D40FD6DE2EEBA63F76D6D7D924134335BA9EBE813A4197DAAC8EF6603A8B6C71AAF6A6FAAD92B1345DF53A2943845A7AB6A09CFC3783C8FBC72AD48B2ED5C04924E9DFBF57EBCDEBF][ESNI Cipher: TLS_AES_128_GCM_SHA256][Cipher: TLS_CHACHA20_POLY1305_SHA256][Plen Bins: 12,0,12,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,12,0,0,0,0,0,0,0,0,0,0,0,12,0,0,0,0,0,0,0,0,0,0,0,12,0,25,0,0] From 8da5f42fa0e0cda39e079c97f315ffcafdb587fb Mon Sep 17 00:00:00 2001 From: Toni Uhlig Date: Sat, 8 Aug 2020 11:50:59 +0200 Subject: [PATCH 09/32] Changed ndpi_ssl_version2str function call in ndpiSimpleIntegration. Fixes build error introduced with 23c072153. Signed-off-by: Toni Uhlig --- example/ndpiSimpleIntegration.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/example/ndpiSimpleIntegration.c b/example/ndpiSimpleIntegration.c index d5bd117d8..f30e7cf99 100644 --- a/example/ndpiSimpleIntegration.c +++ b/example/ndpiSimpleIntegration.c @@ -873,7 +873,8 @@ static void ndpi_process_packet(uint8_t * const args, workflow->packets_captured, reader_thread->array_index, flow_to_process->flow_id, - ndpi_ssl_version2str(flow_to_process->ndpi_flow->protos.stun_ssl.ssl.ssl_version, + ndpi_ssl_version2str(flow_to_process->ndpi_flow, + flow_to_process->ndpi_flow->protos.stun_ssl.ssl.ssl_version, &unknown_tls_version), flow_to_process->ndpi_flow->protos.stun_ssl.ssl.client_requested_server_name, (flow_to_process->ndpi_flow->protos.stun_ssl.ssl.alpn != NULL ? @@ -889,7 +890,8 @@ static void ndpi_process_packet(uint8_t * const args, workflow->packets_captured, reader_thread->array_index, flow_to_process->flow_id, - ndpi_ssl_version2str(flow_to_process->ndpi_flow->protos.stun_ssl.ssl.ssl_version, + ndpi_ssl_version2str(flow_to_process->ndpi_flow, + flow_to_process->ndpi_flow->protos.stun_ssl.ssl.ssl_version, &unknown_tls_version), flow_to_process->ndpi_flow->protos.stun_ssl.ssl.server_names_len, flow_to_process->ndpi_flow->protos.stun_ssl.ssl.server_names, From 4ead6d6594e64716b58603537c779f6c9be44f37 Mon Sep 17 00:00:00 2001 From: Toni Uhlig Date: Sat, 8 Aug 2020 12:17:54 +0200 Subject: [PATCH 10/32] travis-ci: build ndpiSimpleIntegration as well Signed-off-by: Toni Uhlig --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2c359b356..c583c3a0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -170,6 +170,7 @@ before_script: script: - if [ -n "$QA_FUZZ" ]; then ./configure --enable-fuzztargets ; else ./configure ; fi - make + - make -C example ndpiSimpleIntegration #after_script: - cd tests From 8f74d5733d6a994acf1ddc985c5a3673619fd805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Zgorza=C5=82ek?= Date: Sat, 1 Aug 2020 23:25:13 +0100 Subject: [PATCH 11/32] OpenBSD: Introduce pkt_timeval to deal with (bpf_)_timeval Some BSD APIs called in example/ return `struct bpf_timeval`, where nDPI APIs expect `struct timeval`. These two structs, besides having a different name, share the exact same set of fields. --- example/reader_util.c | 8 ++--- example/reader_util.h | 11 +++--- src/include/Makefile.am | 1 + src/include/ndpi_classify.h | 26 +++++++-------- src/include/ndpi_includes.h | 6 +++- src/include/ndpi_includes_OpenBSD.h | 43 ++++++++++++++++++++++++ src/lib/ndpi_classify.c | 52 ++++++++++++++--------------- src/lib/ndpi_community_id.c | 1 + 8 files changed, 99 insertions(+), 49 deletions(-) create mode 100644 src/include/ndpi_includes_OpenBSD.h diff --git a/example/reader_util.c b/example/reader_util.c index 7e68a378d..d0f16ab62 100644 --- a/example/reader_util.c +++ b/example/reader_util.c @@ -692,7 +692,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow u_int8_t **payload, u_int16_t *payload_len, u_int8_t *src_to_dst_direction, - struct timeval when) { + pkt_timeval when) { u_int32_t idx, l4_offset, hashval; struct ndpi_flow_info flow; void *ret; @@ -979,7 +979,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo u_int8_t **payload, u_int16_t *payload_len, u_int8_t *src_to_dst_direction, - struct timeval when) { + pkt_timeval when) { struct ndpi_iphdr iph; memset(&iph, 0, sizeof(iph)); @@ -1300,7 +1300,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow, u_int16_t ipsize, u_int16_t rawsize, const struct pcap_pkthdr *header, const u_char *packet, - struct timeval when, + pkt_timeval when, FILE * csv_fp) { struct ndpi_id_struct *src, *dst; struct ndpi_flow_info *flow = NULL; @@ -1330,7 +1330,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow, &payload, &payload_len, &src_to_dst_direction, when); if(flow != NULL) { - struct timeval tdiff; + pkt_timeval tdiff; workflow->stats.ip_packet_count++; workflow->stats.total_wire_bytes += rawsize + 24 /* CRC etc */, diff --git a/example/reader_util.h b/example/reader_util.h index d4e3dc74f..4dba29ddc 100644 --- a/example/reader_util.h +++ b/example/reader_util.h @@ -31,6 +31,7 @@ #include "uthash.h" #include +#include "ndpi_includes.h" #include "ndpi_classify.h" #include "ndpi_typedefs.h" @@ -128,13 +129,13 @@ struct flow_metrics { struct ndpi_entropy { // Entropy fields - struct timeval src2dst_last_pkt_time, dst2src_last_pkt_time, flow_last_pkt_time; + pkt_timeval src2dst_last_pkt_time, dst2src_last_pkt_time, flow_last_pkt_time; u_int16_t src2dst_pkt_len[MAX_NUM_PKTS]; /*!< array of packet appdata lengths */ - struct timeval src2dst_pkt_time[MAX_NUM_PKTS]; /*!< array of arrival times */ + pkt_timeval src2dst_pkt_time[MAX_NUM_PKTS]; /*!< array of arrival times */ u_int16_t dst2src_pkt_len[MAX_NUM_PKTS]; /*!< array of packet appdata lengths */ - struct timeval dst2src_pkt_time[MAX_NUM_PKTS]; /*!< array of arrival times */ - struct timeval src2dst_start; /*!< first packet arrival time */ - struct timeval dst2src_start; /*!< first packet arrival time */ + pkt_timeval dst2src_pkt_time[MAX_NUM_PKTS]; /*!< array of arrival times */ + pkt_timeval src2dst_start; /*!< first packet arrival time */ + pkt_timeval dst2src_start; /*!< first packet arrival time */ u_int32_t src2dst_opackets; /*!< non-zero packet counts */ u_int32_t dst2src_opackets; /*!< non-zero packet counts */ u_int16_t src2dst_pkt_count; /*!< packet counts */ diff --git a/src/include/Makefile.am b/src/include/Makefile.am index db4e40f35..19d6c60cf 100644 --- a/src/include/Makefile.am +++ b/src/include/Makefile.am @@ -8,4 +8,5 @@ library_include_HEADERS = ndpi_api.h \ ndpi_protocol_ids.h \ ndpi_protocols.h \ ndpi_win32.h \ + ndpi_includes_OpenBSD.h \ ndpi_includes.h diff --git a/src/include/ndpi_classify.h b/src/include/ndpi_classify.h index 4d2cfff97..ab9212832 100644 --- a/src/include/ndpi_classify.h +++ b/src/include/ndpi_classify.h @@ -43,7 +43,7 @@ #ifndef NDPI_CLASSIFY_H #define NDPI_CLASSIFY_H - +#include "ndpi_includes.h" /* constants */ #define NUM_PARAMETERS_SPLT_LOGREG 208 @@ -66,27 +66,27 @@ extern float parameters_bd[NUM_PARAMETERS_BD_LOGREG]; extern float parameters_splt[NUM_PARAMETERS_SPLT_LOGREG]; /* Classifier functions */ -float ndpi_classify(const unsigned short *pkt_len, const struct timeval *pkt_time, - const unsigned short *pkt_len_twin, const struct timeval *pkt_time_twin, - struct timeval start_time, struct timeval start_time_twin, uint32_t max_num_pkt_len, +float ndpi_classify(const unsigned short *pkt_len, const pkt_timeval *pkt_time, + const unsigned short *pkt_len_twin, const pkt_timeval *pkt_time_twin, + pkt_timeval start_time, pkt_timeval start_time_twin, uint32_t max_num_pkt_len, uint16_t sp, uint16_t dp, uint32_t op, uint32_t ip, uint32_t np_o, uint32_t np_i, uint32_t ob, uint32_t ib, uint16_t use_bd, const uint32_t *bd, const uint32_t *bd_t); -void ndpi_merge_splt_arrays(const uint16_t *pkt_len, const struct timeval *pkt_time, - const uint16_t *pkt_len_twin, const struct timeval *pkt_time_twin, - struct timeval start_time, struct timeval start_time_twin, +void ndpi_merge_splt_arrays(const uint16_t *pkt_len, const pkt_timeval *pkt_time, + const uint16_t *pkt_len_twin, const pkt_timeval *pkt_time_twin, + pkt_timeval start_time, pkt_timeval start_time_twin, uint16_t s_idx, uint16_t r_idx, uint16_t *merged_lens, uint16_t *merged_times); void ndpi_update_params(classifier_type_codes_t param_type, const char *param_file); void ndpi_flow_info_freer(void *node); -unsigned int ndpi_timer_eq(const struct timeval *a, const struct timeval *b); -unsigned int ndpi_timer_lt(const struct timeval *a, const struct timeval *b); -void ndpi_timer_sub(const struct timeval *a, const struct timeval *b, struct timeval *result); -void ndpi_timer_clear(struct timeval *a); -unsigned int ndpi_timeval_to_milliseconds(struct timeval ts); -unsigned int ndpi_timeval_to_microseconds(struct timeval ts); +unsigned int ndpi_timer_eq(const pkt_timeval *a, const pkt_timeval *b); +unsigned int ndpi_timer_lt(const pkt_timeval *a, const pkt_timeval *b); +void ndpi_timer_sub(const pkt_timeval *a, const pkt_timeval *b, pkt_timeval *result); +void ndpi_timer_clear(pkt_timeval *a); +unsigned int ndpi_timeval_to_milliseconds(pkt_timeval ts); +unsigned int ndpi_timeval_to_microseconds(pkt_timeval ts); void ndpi_log_timestamp(char *log_ts, uint32_t log_ts_len); #endif /* NDPI_CLASSIFY_H */ diff --git a/src/include/ndpi_includes.h b/src/include/ndpi_includes.h index f8bde5194..99c50fe02 100644 --- a/src/include/ndpi_includes.h +++ b/src/include/ndpi_includes.h @@ -57,7 +57,7 @@ #if defined __NetBSD__ || defined __OpenBSD__ #include -#ifdef __OpenBSD__ +#if defined __OpenBSD__ #include #endif @@ -67,4 +67,8 @@ #endif /* Win32 */ +#if defined __OpenBSD__ +#include "ndpi_includes_OpenBSD.h" +#endif /* __OpenBSD__ */ + #endif /* __NDPI_INCLUDES_H__ */ diff --git a/src/include/ndpi_includes_OpenBSD.h b/src/include/ndpi_includes_OpenBSD.h new file mode 100644 index 000000000..4efdbd844 --- /dev/null +++ b/src/include/ndpi_includes_OpenBSD.h @@ -0,0 +1,43 @@ +/* + * ndpi_includes_OpenBSD.h + * + * Copyright (C) 2011-16 - ntop.org + * + * This file is part of nDPI, an open source deep packet inspection + * library based on the OpenDPI and PACE technology by ipoque GmbH + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ + +#ifndef __NDPI_INCLUDES_OPENBSD_H__ +#define __NDPI_INCLUDES_OPENBSD_H__ + +#ifdef __OpenBSD__ + +#ifndef IPPROTO_SCTP +#define IPPROTO_SCTP 132 +#endif /* IPPROTO_SCTP */ + +#endif /* __OpenBSD__ */ + + +#ifdef __OpenBSD__ +#include +typedef struct bpf_timeval pkt_timeval; +#else +typedef struct timeval pkt_timeval; +#endif /* __OpenBSD__ */ + +#endif /* __NDPI_INCLUDES_OPENBSD_H__ */ diff --git a/src/lib/ndpi_classify.c b/src/lib/ndpi_classify.c index 9791db324..7b410e05f 100644 --- a/src/lib/ndpi_classify.c +++ b/src/lib/ndpi_classify.c @@ -240,9 +240,9 @@ float ndpi_parameters_bd[NUM_PARAMETERS_BD_LOGREG] = { }; /** - * \fn void ndpi_merge_splt_arrays (const uint16_t *pkt_len, const struct timeval *pkt_time, - const uint16_t *pkt_len_twin, const struct timeval *pkt_time_twin, - struct timeval start_time, struct timeval start_time_twin, + * \fn void ndpi_merge_splt_arrays (const uint16_t *pkt_len, const pkt_timeval *pkt_time, + const uint16_t *pkt_len_twin, const pkt_timeval *pkt_time_twin, + pkt_timeval start_time, pkt_timeval start_time_twin, uint16_t s_idx, uint16_t r_idx, uint16_t *merged_lens, uint16_t *merged_times, uint32_t max_num_pkt_len, uint32_t max_merged_num_pkts) @@ -260,16 +260,16 @@ float ndpi_parameters_bd[NUM_PARAMETERS_BD_LOGREG] = { * \return none */ void -ndpi_merge_splt_arrays (const uint16_t *pkt_len, const struct timeval *pkt_time, - const uint16_t *pkt_len_twin, const struct timeval *pkt_time_twin, - struct timeval start_time, struct timeval start_time_twin, +ndpi_merge_splt_arrays (const uint16_t *pkt_len, const pkt_timeval *pkt_time, + const uint16_t *pkt_len_twin, const pkt_timeval *pkt_time_twin, + pkt_timeval start_time, pkt_timeval start_time_twin, uint16_t s_idx, uint16_t r_idx, uint16_t *merged_lens, uint16_t *merged_times) { int s,r; - struct timeval ts_start = { 0, 0 }; /* initialize to avoid spurious warnings */ - struct timeval tmp, tmp_r; - struct timeval start_m; + pkt_timeval ts_start = { 0, 0 }; /* initialize to avoid spurious warnings */ + pkt_timeval tmp, tmp_r; + pkt_timeval start_m; if(r_idx + s_idx == 0) { return ; @@ -419,9 +419,9 @@ ndpi_get_mc_rep_times (uint16_t *times, float *time_mc, uint16_t num_packets) } /** - * \fn float classify (const unsigned short *pkt_len, const struct timeval *pkt_time, - const unsigned short *pkt_len_twin, const struct timeval *pkt_time_twin, - struct timeval start_time, struct timeval start_time_twin, uint32_t max_num_pkt_len, + * \fn float classify (const unsigned short *pkt_len, const pkt_timeval *pkt_time, + const unsigned short *pkt_len_twin, const pkt_timeval *pkt_time_twin, + pkt_timeval start_time, pkt_timeval start_time_twin, uint32_t max_num_pkt_len, uint16_t sp, uint16_t dp, uint32_t op, uint32_t ip, uint32_t np_o, uint32_t np_i, uint32_t ob, uint32_t ib, uint16_t use_bd, const uint32_t *bd, const uint32_t *bd_t) * \param pkt_len length of the packet @@ -445,9 +445,9 @@ ndpi_get_mc_rep_times (uint16_t *times, float *time_mc, uint16_t num_packets) * \return float score */ float -ndpi_classify (const unsigned short *pkt_len, const struct timeval *pkt_time, - const unsigned short *pkt_len_twin, const struct timeval *pkt_time_twin, - struct timeval start_time, struct timeval start_time_twin, uint32_t max_num_pkt_len, +ndpi_classify (const unsigned short *pkt_len, const pkt_timeval *pkt_time, + const unsigned short *pkt_len_twin, const pkt_timeval *pkt_time_twin, + pkt_timeval start_time, pkt_timeval start_time_twin, uint32_t max_num_pkt_len, uint16_t sp, uint16_t dp, uint32_t op, uint32_t ip, uint32_t np_o, uint32_t np_i, uint32_t ob, uint32_t ib, uint16_t use_bd, const uint32_t *bd, const uint32_t *bd_t) { @@ -604,8 +604,8 @@ ndpi_update_params (classifier_type_codes_t param_type, const char *param_file) * \return 1 if equal, 0 otherwise */ unsigned int -ndpi_timer_eq(const struct timeval *a, - const struct timeval *b) +ndpi_timer_eq(const pkt_timeval *a, + const pkt_timeval *b) { if(a->tv_sec == b->tv_sec && a->tv_usec == b->tv_usec) { return 1; @@ -615,8 +615,8 @@ ndpi_timer_eq(const struct timeval *a, } unsigned int -ndpi_timer_lt(const struct timeval *a, - const struct timeval *b) +ndpi_timer_lt(const pkt_timeval *a, + const pkt_timeval *b) { return (a->tv_sec == b->tv_sec) ? (a->tv_usec < b->tv_usec):(a->tv_sec < b->tv_sec); @@ -630,9 +630,9 @@ ndpi_timer_lt(const struct timeval *a, * \return none */ void -ndpi_timer_sub(const struct timeval *a, - const struct timeval *b, - struct timeval *result) +ndpi_timer_sub(const pkt_timeval *a, + const pkt_timeval *b, + pkt_timeval *result) { result->tv_sec = a->tv_sec - b->tv_sec; result->tv_usec = a->tv_usec - b->tv_usec; @@ -648,7 +648,7 @@ ndpi_timer_sub(const struct timeval *a, * \return none */ void -ndpi_timer_clear(struct timeval *a) +ndpi_timer_clear(pkt_timeval *a) { a->tv_sec = a->tv_usec = 0; } @@ -659,7 +659,7 @@ ndpi_timer_clear(struct timeval *a) * \return unsigned int - Milliseconds */ unsigned int -ndpi_timeval_to_milliseconds(struct timeval ts) +ndpi_timeval_to_milliseconds(pkt_timeval ts) { unsigned int result = ts.tv_usec / 1000 + ts.tv_sec * 1000; return result; @@ -671,7 +671,7 @@ ndpi_timeval_to_milliseconds(struct timeval ts) * \return unsigned int - Milliseconds */ unsigned int -ndpi_timeval_to_microseconds(struct timeval ts) +ndpi_timeval_to_microseconds(pkt_timeval ts) { unsigned int result = ts.tv_usec + ts.tv_sec * 1000 * 1000; return result; @@ -680,7 +680,7 @@ ndpi_timeval_to_microseconds(struct timeval ts) void ndpi_log_timestamp(char *log_ts, uint32_t log_ts_len) { - struct timeval tv; + pkt_timeval tv; time_t nowtime; struct tm nowtm_r; char tmbuf[NDPI_TIMESTAMP_LEN]; diff --git a/src/lib/ndpi_community_id.c b/src/lib/ndpi_community_id.c index 72f60c746..cc8436928 100644 --- a/src/lib/ndpi_community_id.c +++ b/src/lib/ndpi_community_id.c @@ -31,6 +31,7 @@ #include "ndpi_api.h" #include "ndpi_config.h" +#include "ndpi_includes.h" #include #ifndef WIN32 From 56a3a33bc0e21640ab1c345b7953e7b5c076673b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Zgorza=C5=82ek?= Date: Sun, 2 Aug 2020 01:29:54 +0100 Subject: [PATCH 12/32] OpenBSD: Do not redefine __LITTLE_ENDIAN__ Will silence omnipresent compiler warnings when building ntopng. --- src/include/ndpi_define.h.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/include/ndpi_define.h.in b/src/include/ndpi_define.h.in index 990f84bf4..1fb0d282c 100644 --- a/src/include/ndpi_define.h.in +++ b/src/include/ndpi_define.h.in @@ -35,7 +35,9 @@ #include #define __BYTE_ORDER BYTE_ORDER #if BYTE_ORDER == LITTLE_ENDIAN +#ifndef __LITTLE_ENDIAN__ #define __LITTLE_ENDIAN__ +#endif /* __LITTLE_ENDIAN__ */ #else #define __BIG_ENDIAN__ #endif/* BYTE_ORDER */ From dfa9dd66c0d22bcf4af0ae18999d3c330cdf50b6 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 10 Aug 2020 19:36:43 +0200 Subject: [PATCH 13/32] Added case-insensitive substring matching --- src/include/ndpi_typedefs.h | 1 + src/lib/ndpi_main.c | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 53d143327..5dd36bff2 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -80,6 +80,7 @@ typedef enum { NDPI_SSH_OBSOLETE_SERVER_VERSION_OR_CIPHER, NDPI_SMB_INSECURE_VERSION, NDPI_TLS_SUSPICIOUS_ESNI_USAGE, + BDPI_BLACKLISTED_HOST, /* Leave this as last member */ NDPI_MAX_RISK diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index bdbdc89f3..83c10a1d7 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -2591,11 +2591,17 @@ int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_str, char *rule, is_ip = 1, value = &attr[3]; else if(strncmp(attr, "host:", 5) == 0) { /* host:"",host:"",.....@ */ + u_int i, max_len; + value = &attr[5]; if(value[0] == '"') value++; /* remove leading " */ - if(value[strlen(value) - 1] == '"') - value[strlen(value) - 1] = '\0'; /* remove trailing " */ + + max_len = strlen(value) - 1; + if(value[max_len] == '"') + value[max_len] = '\0'; /* remove trailing " */ + + for(i=0; i 2) && (name[0] == '*') && (name[1] == '.')) @@ -6113,6 +6119,9 @@ int ndpi_match_hostname_protocol(struct ndpi_detection_module_struct *ndpi_struc else what = name, what_len = name_len; + /* Convert it first to lowercase: we assume meory is writable as in nDPI dissctors */ + for(i=0; i Date: Mon, 10 Aug 2020 21:17:36 +0200 Subject: [PATCH 14/32] Fix typo. --- src/include/ndpi_typedefs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 5dd36bff2..40c27329e 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -80,7 +80,7 @@ typedef enum { NDPI_SSH_OBSOLETE_SERVER_VERSION_OR_CIPHER, NDPI_SMB_INSECURE_VERSION, NDPI_TLS_SUSPICIOUS_ESNI_USAGE, - BDPI_BLACKLISTED_HOST, + NDPI_BLACKLISTED_HOST, /* Leave this as last member */ NDPI_MAX_RISK From 8c2c388d54032f25824c1a0cce4dd379a87bed17 Mon Sep 17 00:00:00 2001 From: aouinizied Date: Mon, 10 Aug 2020 21:19:17 +0200 Subject: [PATCH 15/32] Add Connectivity check category and blacklisted host risk. --- python/ndpi.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/ndpi.py b/python/ndpi.py index 85378f526..ee33f6a37 100644 --- a/python/ndpi.py +++ b/python/ndpi.py @@ -313,6 +313,7 @@ typedef enum { NDPI_SSH_OBSOLETE_SERVER_VERSION_OR_CIPHER, NDPI_SMB_INSECURE_VERSION, NDPI_TLS_SUSPICIOUS_ESNI_USAGE, + NDPI_BLACKLISTED_HOST, /* Leave this as last member */ NDPI_MAX_RISK } ndpi_risk_enum; @@ -822,6 +823,12 @@ typedef enum { NDPI_PROTOCOL_CATEGORY_PRODUCTIVITY, NDPI_PROTOCOL_CATEGORY_FILE_SHARING, + /* + The category below is used by sites who are used + to test connectivity + */ + NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK, + /* Some custom categories */ CUSTOM_CATEGORY_MINING = 99, CUSTOM_CATEGORY_MALWARE = 100, From 0e363d0ca6ab4f1df16159e3d3b4bebba9372772 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Tue, 11 Aug 2020 16:23:35 +0200 Subject: [PATCH 16/32] Added HLL notes --- src/lib/third_party/src/hll/hll.c | 52 ++++++++++++++++--------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/src/lib/third_party/src/hll/hll.c b/src/lib/third_party/src/hll/hll.c index a7006c7ed..c526c6af0 100644 --- a/src/lib/third_party/src/hll/hll.c +++ b/src/lib/third_party/src/hll/hll.c @@ -34,6 +34,7 @@ u_int32_t _hll_hash(const struct ndpi_hll *hll) { return MurmurHash3_x86_32(hll->registers, (u_int32_t)hll->size, 0); } +/* Count the number of leading zero's */ static __inline u_int8_t _hll_rank(u_int32_t hash, u_int8_t bits) { u_int8_t i; @@ -48,24 +49,26 @@ static __inline u_int8_t _hll_rank(u_int32_t hash, u_int8_t bits) { } /* - IMPORTANT: memory usage notes + IMPORTANT: HyperLogLog Memory and StandardError Notes - [i: 4] 16 bytes - [i: 5] 32 bytes - [i: 6] 64 bytes - [i: 7] 128 bytes - [i: 8] 256 bytes - [i: 9] 512 bytes - [i: 10] 1024 bytes - [i: 11] 2048 bytes - [i: 12] 4096 bytes - [i: 13] 8192 bytes - [i: 14] 16384 bytes - [i: 15] 32768 bytes - [i: 16] 65536 bytes - [i: 17] 131072 bytes - [i: 18] 262144 bytes - [i: 19] 524288 bytes + StdError = 1.04/sqrt(2^i) + + [i: 4] 16 bytes [StdError: 26% ] + [i: 5] 32 bytes [StdError: 18.4%] + [i: 6] 64 bytes [StdError: 13% ] + [i: 7] 128 bytes [StdError: 9.2% ] + [i: 8] 256 bytes [StdError: 6.5% ] + [i: 9] 512 bytes [StdError: 4.6% ] + [i: 10] 1024 bytes [StdError: 3.25%] + [i: 11] 2048 bytes [StdError: 2.3% ] + [i: 12] 4096 bytes [StdError: 1.6% ] + [i: 13] 8192 bytes [StdError: 1.15%] + [i: 14] 16384 bytes [StdError: 0.81%] + [i: 15] 32768 bytes [StdError: 0.57%] + [i: 16] 65536 bytes [StdError: 0.41%] + [i: 17] 131072 bytes [StdError: 0.29%] + [i: 18] 262144 bytes [StdError: 0.2% ] + [i: 19] 524288 bytes [StdError: 0.14%] */ int hll_init(struct ndpi_hll *hll, u_int8_t bits) { if(bits < 4 || bits > 20) { @@ -73,9 +76,9 @@ int hll_init(struct ndpi_hll *hll, u_int8_t bits) { return -1; } - hll->bits = bits; - hll->size = (size_t)1 << bits; - hll->registers = ndpi_calloc(hll->size, 1); + hll->bits = bits; /* Number of bits of buckets number */ + hll->size = (size_t)1 << bits; /* Number of buckets 2^bits */ + hll->registers = ndpi_calloc(hll->size, 1); /* Create the bucket register counters */ /* printf("%lu bytes\n", hll->size); */ return 0; @@ -96,12 +99,11 @@ void hll_reset(struct ndpi_hll *hll) { static __inline void _hll_add_hash(struct ndpi_hll *hll, u_int32_t hash) { if(hll->registers) { - u_int32_t index = hash >> (32 - hll->bits); - u_int8_t rank = _hll_rank(hash, hll->bits); + u_int32_t index = hash >> (32 - hll->bits); /* Use the first 'hll->bits' bits as bucket index */ + u_int8_t rank = _hll_rank(hash, hll->bits); /* Count the number of leading 0 */ - if(rank > hll->registers[index]) { - hll->registers[index] = rank; - } + if(rank > hll->registers[index]) + hll->registers[index] = rank; /* Store the largest number of lesding zeros for the bucket */ } } From d5cac570d6aa541df1a30dcf891e9563a476608b Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Tue, 11 Aug 2020 17:13:40 +0200 Subject: [PATCH 17/32] Improved DGA detection algoritm --- src/lib/ndpi_content_match.c.inc | 2 +- src/lib/ndpi_main.c | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index c8fe416eb..29e2a4277 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -9341,7 +9341,7 @@ static const char *ndpi_en_popular_bigrams[] = { static const char *ndpi_en_impossible_bigrams[] = { "bk", "bq", "bx", "cb", "cf", "cg", "cj", "cp", "cv", "cw", "cx", "dx", "fk", "fq", "fv", "fx", /* "ee", removed it can be found in 'meeting' */ "fz", "gq", "gv", "gx", "hh", "hk", "hv", "hx", "hz", "iy", "jb", /* "jc", jcrew.com */ "jd", "jf", "jg", "jh", "jk", - "jl", "jm", "jn", "jp", "jq", /* "jr",*/ /* "js", */ "jt", "jv", "jw", "jx", "jy", "jz", "kg", "kq", "kv", "kx", + "jl", "jm", "jn", "jp", "jq", /* "jr",*/ /* "js", */ "jt", "jv", "jw", "jx", "jy", "jz", /* "kg", */ "kq", "kv", "kx", "kz", "lq", "lx", /* "mg" tamgrt.com , */ "mj", /* "mq", mqtt */ "mx", "mz", "pq", "pv", "px", "qb", "qc", "qd", "qe", "qf", "ii", "qg", "qh", "qj", "qk", "ql", "qm", "qn", "qo", "qp", "qr", "qs", "qt", "qv", "qw", "qx", "qy", "uu", "qz", "sx", "sz", "tq", "tx", "vb", "vc", "vd", "vf", "vg", "vh", "vj", "vm", "vn", /* "vp", Removed for vpbank.com */ "bw", /* "vk", "zr" Removed for kavkazr */ diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 83c10a1d7..79fe7c836 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -6579,7 +6579,7 @@ static int enough(int a, int b) { /* ******************************************************************** */ -// #define DGA_DEBUG 1 +/* #define DGA_DEBUG 1 */ int ndpi_check_dga_name(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow, @@ -6656,18 +6656,16 @@ int ndpi_check_dga_name(struct ndpi_detection_module_struct *ndpi_str, printf("-> Checking %c%c\n", word[i], word[i+1]); #endif - if(ndpi_match_bigram(ndpi_str, &ndpi_str->bigrams_automa, &word[i])) { - num_found++; - } else { - if(ndpi_match_bigram(ndpi_str, - &ndpi_str->impossible_bigrams_automa, - &word[i])) { + if(ndpi_match_bigram(ndpi_str, + &ndpi_str->impossible_bigrams_automa, + &word[i])) { #ifdef DGA_DEBUG - printf("IMPOSSIBLE %s\n", &word[i]); + printf("IMPOSSIBLE %s\n", &word[i]); #endif - num_impossible++; - } - } + num_impossible++; + } else if(ndpi_match_bigram(ndpi_str, &ndpi_str->bigrams_automa, &word[i])) { + num_found++; + } } /* for */ } /* for */ From 9edddee0b7e63ff4fd6e5c19156e422d5712375c Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Wed, 12 Aug 2020 11:08:28 +0200 Subject: [PATCH 18/32] Fixes invalid detection on traffic on non standard ports --- src/lib/ndpi_main.c | 19 ++++++++++++++++--- tests/result/googledns_android10.pcap.out | 10 +++++----- tests/result/smb_deletefile.pcap.out | 2 +- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 79fe7c836..35123c1c9 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -4736,21 +4736,34 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct if(found && (found->proto->protoId != NDPI_PROTOCOL_UNKNOWN) - && (found->proto->protoId != ret.master_protocol)) { + && (found->proto->protoId != ret.master_protocol) + && (found->proto->protoId != ret.app_protocol) + ) { // printf("******** %u / %u\n", found->proto->protoId, ret.master_protocol); if(!ndpi_check_protocol_port_mismatch_exceptions(ndpi_str, flow, found, &ret)) NDPI_SET_BIT(flow->risk, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT); } else if(default_ports && (default_ports[0] != 0)) { - u_int8_t found = 0, i; + u_int8_t found = 0, i, num_loops = 0; + check_default_ports: for(i=0; (ipacket.udp) + default_ports = ndpi_str->proto_defaults[ret.app_protocol].udp_default_ports; + else + default_ports = ndpi_str->proto_defaults[ret.app_protocol].tcp_default_ports; + + num_loops = 1; + goto check_default_ports; + } + if(!found) { // printf("******** Invalid default port\n"); NDPI_SET_BIT(flow->risk, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT); diff --git a/tests/result/googledns_android10.pcap.out b/tests/result/googledns_android10.pcap.out index 0521f3212..4b0a7f748 100644 --- a/tests/result/googledns_android10.pcap.out +++ b/tests/result/googledns_android10.pcap.out @@ -6,11 +6,11 @@ JA3 Host Stats: 1 192.168.1.159 2 - 1 TCP 192.168.1.159:48210 <-> 8.8.4.4:853 [proto: 91.196/TLS.DoH_DoT][cat: Network/14][121 pkts/19065 bytes <-> 120 pkts/45726 bytes][Goodput ratio: 58/83][72.27 sec][bytes ratio: -0.411 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 711/474 15173/5940 1940/1160][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 158/381 384/1484 93/280][Risk: ** Known protocol on non standard port **** TLS (probably) not carrying HTTPS **][TLSv1.2][Client: dns.google][JA3C: 2c776785ee603cc85d37df996bb90cc8][ServerNames: dns.google,*.dns.google.com,8888.google,dns.google.com,dns64.dns.google][JA3S: b44baa8a20901c5663b3a9664ba8a767][Issuer: C=US, O=Google Trust Services, CN=GTS CA 1O1][Subject: C=US, ST=California, L=Mountain View, O=Google LLC, CN=dns.google][Certificate SHA-1: 5B:59:09:FC:7D:50:E6:F7:D1:08:8E:57:42:A2:D8:AE:1F:03:FF:EC][Validity: 2020-05-26 15:20:02 - 2020-08-18 15:20:02][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 0,0,0,0,42,0,0,0,0,5,0,0,0,0,0,51,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0] - 2 TCP 192.168.1.159:48098 <-> 8.8.4.4:853 [proto: 91.196/TLS.DoH_DoT][cat: Network/14][68 pkts/9706 bytes <-> 65 pkts/18916 bytes][Goodput ratio: 54/77][117.95 sec][bytes ratio: -0.322 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 2097/1988 15177/15193 3804/3968][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 143/291 583/565 94/247][Risk: ** Known protocol on non standard port **** TLS (probably) not carrying HTTPS **][TLSv1.2][Client: dns.google][JA3C: b734f75d22aaff9866fbd5d27eef9106][JA3S: 1249fb68f48c0444718e4d3b48b27188][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 1,1,0,0,49,0,0,0,0,0,0,0,0,0,0,47,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] - 3 TCP 192.168.1.159:48048 <-> 8.8.4.4:853 [proto: 91.196/TLS.DoH_DoT][cat: Network/14][52 pkts/7375 bytes <-> 52 pkts/20720 bytes][Goodput ratio: 53/83][41.01 sec][bytes ratio: -0.475 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 882/623 15271/15287 2537/2442][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 142/398 384/1484 84/406][Risk: ** Known protocol on non standard port **** TLS (probably) not carrying HTTPS **][TLSv1.2][Client: dns.google][JA3C: 2c776785ee603cc85d37df996bb90cc8][ServerNames: dns.google,*.dns.google.com,8888.google,dns.google.com,dns64.dns.google][JA3S: b44baa8a20901c5663b3a9664ba8a767][Issuer: C=US, O=Google Trust Services, CN=GTS CA 1O1][Subject: C=US, ST=California, L=Mountain View, O=Google LLC, CN=dns.google][Certificate SHA-1: 5B:59:09:FC:7D:50:E6:F7:D1:08:8E:57:42:A2:D8:AE:1F:03:FF:EC][Validity: 2020-05-26 15:20:02 - 2020-08-18 15:20:02][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 1,0,1,0,44,0,0,1,0,3,0,0,0,0,0,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,3,0,0,0] - 4 TCP 192.168.1.159:48044 <-> 8.8.4.4:853 [proto: 91.196/TLS.DoH_DoT][cat: Network/14][11 pkts/1097 bytes <-> 10 pkts/4148 bytes][Goodput ratio: 33/84][0.12 sec][bytes ratio: -0.582 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 12/9 34/19 13/8][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 100/415 220/1484 51/544][Risk: ** Known protocol on non standard port **** TLS (probably) not carrying HTTPS **][TLSv1.2][Client: dns.google][JA3C: 2c776785ee603cc85d37df996bb90cc8][ServerNames: dns.google,*.dns.google.com,8888.google,dns.google.com,dns64.dns.google][JA3S: b44baa8a20901c5663b3a9664ba8a767][Issuer: C=US, O=Google Trust Services, CN=GTS CA 1O1][Subject: C=US, ST=California, L=Mountain View, O=Google LLC, CN=dns.google][Certificate SHA-1: 5B:59:09:FC:7D:50:E6:F7:D1:08:8E:57:42:A2:D8:AE:1F:03:FF:EC][Validity: 2020-05-26 15:20:02 - 2020-08-18 15:20:02][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 11,0,22,11,11,0,0,11,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,22,0,0,0] - 5 TCP 192.168.1.159:56024 <-> 8.8.8.8:853 [proto: 91.196/TLS.DoH_DoT][cat: Network/14][11 pkts/1097 bytes <-> 10 pkts/4148 bytes][Goodput ratio: 33/84][0.14 sec][bytes ratio: -0.582 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 15/12 46/31 17/11][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 100/415 220/1484 51/544][Risk: ** Known protocol on non standard port **** TLS (probably) not carrying HTTPS **][TLSv1.2][Client: dns.google][JA3C: 2c776785ee603cc85d37df996bb90cc8][ServerNames: dns.google,*.dns.google.com,8888.google,dns.google.com,dns64.dns.google][JA3S: b44baa8a20901c5663b3a9664ba8a767][Issuer: C=US, O=Google Trust Services, CN=GTS CA 1O1][Subject: C=US, ST=California, L=Mountain View, O=Google LLC, CN=dns.google][Certificate SHA-1: 5B:59:09:FC:7D:50:E6:F7:D1:08:8E:57:42:A2:D8:AE:1F:03:FF:EC][Validity: 2020-05-26 15:20:02 - 2020-08-18 15:20:02][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 11,0,22,11,11,0,0,11,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,22,0,0,0] + 1 TCP 192.168.1.159:48210 <-> 8.8.4.4:853 [proto: 91.196/TLS.DoH_DoT][cat: Network/14][121 pkts/19065 bytes <-> 120 pkts/45726 bytes][Goodput ratio: 58/83][72.27 sec][bytes ratio: -0.411 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 711/474 15173/5940 1940/1160][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 158/381 384/1484 93/280][Risk: ** TLS (probably) not carrying HTTPS **][TLSv1.2][Client: dns.google][JA3C: 2c776785ee603cc85d37df996bb90cc8][ServerNames: dns.google,*.dns.google.com,8888.google,dns.google.com,dns64.dns.google][JA3S: b44baa8a20901c5663b3a9664ba8a767][Issuer: C=US, O=Google Trust Services, CN=GTS CA 1O1][Subject: C=US, ST=California, L=Mountain View, O=Google LLC, CN=dns.google][Certificate SHA-1: 5B:59:09:FC:7D:50:E6:F7:D1:08:8E:57:42:A2:D8:AE:1F:03:FF:EC][Validity: 2020-05-26 15:20:02 - 2020-08-18 15:20:02][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 0,0,0,0,42,0,0,0,0,5,0,0,0,0,0,51,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0] + 2 TCP 192.168.1.159:48098 <-> 8.8.4.4:853 [proto: 91.196/TLS.DoH_DoT][cat: Network/14][68 pkts/9706 bytes <-> 65 pkts/18916 bytes][Goodput ratio: 54/77][117.95 sec][bytes ratio: -0.322 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 2097/1988 15177/15193 3804/3968][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 143/291 583/565 94/247][Risk: ** TLS (probably) not carrying HTTPS **][TLSv1.2][Client: dns.google][JA3C: b734f75d22aaff9866fbd5d27eef9106][JA3S: 1249fb68f48c0444718e4d3b48b27188][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 1,1,0,0,49,0,0,0,0,0,0,0,0,0,0,47,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 3 TCP 192.168.1.159:48048 <-> 8.8.4.4:853 [proto: 91.196/TLS.DoH_DoT][cat: Network/14][52 pkts/7375 bytes <-> 52 pkts/20720 bytes][Goodput ratio: 53/83][41.01 sec][bytes ratio: -0.475 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 882/623 15271/15287 2537/2442][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 142/398 384/1484 84/406][Risk: ** TLS (probably) not carrying HTTPS **][TLSv1.2][Client: dns.google][JA3C: 2c776785ee603cc85d37df996bb90cc8][ServerNames: dns.google,*.dns.google.com,8888.google,dns.google.com,dns64.dns.google][JA3S: b44baa8a20901c5663b3a9664ba8a767][Issuer: C=US, O=Google Trust Services, CN=GTS CA 1O1][Subject: C=US, ST=California, L=Mountain View, O=Google LLC, CN=dns.google][Certificate SHA-1: 5B:59:09:FC:7D:50:E6:F7:D1:08:8E:57:42:A2:D8:AE:1F:03:FF:EC][Validity: 2020-05-26 15:20:02 - 2020-08-18 15:20:02][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 1,0,1,0,44,0,0,1,0,3,0,0,0,0,0,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,3,0,0,0] + 4 TCP 192.168.1.159:48044 <-> 8.8.4.4:853 [proto: 91.196/TLS.DoH_DoT][cat: Network/14][11 pkts/1097 bytes <-> 10 pkts/4148 bytes][Goodput ratio: 33/84][0.12 sec][bytes ratio: -0.582 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 12/9 34/19 13/8][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 100/415 220/1484 51/544][Risk: ** TLS (probably) not carrying HTTPS **][TLSv1.2][Client: dns.google][JA3C: 2c776785ee603cc85d37df996bb90cc8][ServerNames: dns.google,*.dns.google.com,8888.google,dns.google.com,dns64.dns.google][JA3S: b44baa8a20901c5663b3a9664ba8a767][Issuer: C=US, O=Google Trust Services, CN=GTS CA 1O1][Subject: C=US, ST=California, L=Mountain View, O=Google LLC, CN=dns.google][Certificate SHA-1: 5B:59:09:FC:7D:50:E6:F7:D1:08:8E:57:42:A2:D8:AE:1F:03:FF:EC][Validity: 2020-05-26 15:20:02 - 2020-08-18 15:20:02][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 11,0,22,11,11,0,0,11,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,22,0,0,0] + 5 TCP 192.168.1.159:56024 <-> 8.8.8.8:853 [proto: 91.196/TLS.DoH_DoT][cat: Network/14][11 pkts/1097 bytes <-> 10 pkts/4148 bytes][Goodput ratio: 33/84][0.14 sec][bytes ratio: -0.582 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 15/12 46/31 17/11][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 100/415 220/1484 51/544][Risk: ** TLS (probably) not carrying HTTPS **][TLSv1.2][Client: dns.google][JA3C: 2c776785ee603cc85d37df996bb90cc8][ServerNames: dns.google,*.dns.google.com,8888.google,dns.google.com,dns64.dns.google][JA3S: b44baa8a20901c5663b3a9664ba8a767][Issuer: C=US, O=Google Trust Services, CN=GTS CA 1O1][Subject: C=US, ST=California, L=Mountain View, O=Google LLC, CN=dns.google][Certificate SHA-1: 5B:59:09:FC:7D:50:E6:F7:D1:08:8E:57:42:A2:D8:AE:1F:03:FF:EC][Validity: 2020-05-26 15:20:02 - 2020-08-18 15:20:02][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 11,0,22,11,11,0,0,11,0,11,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,22,0,0,0] 6 ICMP 192.168.1.159:0 <-> 8.8.8.8:0 [proto: 81.126/ICMP.Google][cat: Network/14][2 pkts/196 bytes <-> 2 pkts/196 bytes][Goodput ratio: 57/57][0.99 sec][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 7 TCP 8.8.8.8:853 <-> 192.168.1.159:55856 [proto: 196.126/DoH_DoT.Google][cat: Web/5][5 pkts/330 bytes <-> 1 pkts/54 bytes][Goodput ratio: 0/0][1.80 sec][bytes ratio: 0.719 (Upload)][IAT c2s/s2c min/avg/max/stddev: 223/0 449/0 911/0 281/0][Pkt Len c2s/s2c min/avg/max/stddev: 66/54 66/54 66/54 0/0][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 8 TCP 8.8.4.4:853 <-> 192.168.1.159:47968 [proto: 196.126/DoH_DoT.Google][cat: Web/5][1 pkts/66 bytes <-> 1 pkts/54 bytes][Goodput ratio: 0/0][0.09 sec][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] diff --git a/tests/result/smb_deletefile.pcap.out b/tests/result/smb_deletefile.pcap.out index 03bcd48d9..95a35655e 100644 --- a/tests/result/smb_deletefile.pcap.out +++ b/tests/result/smb_deletefile.pcap.out @@ -1,3 +1,3 @@ SMBv23 101 30748 1 - 1 TCP 192.168.1.118:56848 <-> 192.168.1.187:445 [proto: 10.41/NetBIOS.SMBv23][cat: System/18][62 pkts/14382 bytes <-> 39 pkts/16366 bytes][Goodput ratio: 77/87][2.38 sec][bytes ratio: -0.065 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 46/80 2157/2158 299/394][Pkt Len c2s/s2c min/avg/max/stddev: 54/60 232/420 530/1514 194/299][Risk: ** Known protocol on non standard port **][Plen Bins: 0,0,4,7,1,0,1,1,0,1,7,9,20,21,6,13,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0] + 1 TCP 192.168.1.118:56848 <-> 192.168.1.187:445 [proto: 10.41/NetBIOS.SMBv23][cat: System/18][62 pkts/14382 bytes <-> 39 pkts/16366 bytes][Goodput ratio: 77/87][2.38 sec][bytes ratio: -0.065 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 46/80 2157/2158 299/394][Pkt Len c2s/s2c min/avg/max/stddev: 54/60 232/420 530/1514 194/299][Plen Bins: 0,0,4,7,1,0,1,1,0,1,7,9,20,21,6,13,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0] From aa856735c0a5a78b76e98f054d5324e3fe74a7a7 Mon Sep 17 00:00:00 2001 From: Toni Uhlig Date: Thu, 13 Aug 2020 10:32:31 +0200 Subject: [PATCH 19/32] num_extra_packets_checked check can be 0 for some protocols and therefor requires lesser-or-equal condition for max_extra_packets_to_check Signed-off-by: Toni Uhlig --- example/ndpiSimpleIntegration.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/ndpiSimpleIntegration.c b/example/ndpiSimpleIntegration.c index f30e7cf99..bf16dbd95 100644 --- a/example/ndpiSimpleIntegration.c +++ b/example/ndpiSimpleIntegration.c @@ -847,7 +847,7 @@ static void ndpi_process_packet(uint8_t * const args, } } - if (flow_to_process->ndpi_flow->num_extra_packets_checked < + if (flow_to_process->ndpi_flow->num_extra_packets_checked <= flow_to_process->ndpi_flow->max_extra_packets_to_check) { /* From b31fde4bbbfe58f7ca2ca14fa6be2895ab115824 Mon Sep 17 00:00:00 2001 From: Toni Uhlig Date: Sat, 15 Aug 2020 12:35:49 +0200 Subject: [PATCH 20/32] Replaced obsolete libpcap pcap_lookupdev with pcap_findalldevs. Signed-off-by: Toni Uhlig --- example/ndpiSimpleIntegration.c | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/example/ndpiSimpleIntegration.c b/example/ndpiSimpleIntegration.c index d5bd117d8..0b022efaf 100644 --- a/example/ndpiSimpleIntegration.c +++ b/example/ndpiSimpleIntegration.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #define MAX_FLOW_ROOTS_PER_THREAD 2048 @@ -133,7 +134,8 @@ static struct nDPI_workflow * init_workflow(char const * const file_or_device) } if (workflow->pcap_handle == NULL) { - fprintf(stderr, "pcap_open_live / pcap_open_offline_with_tstamp_precision: %s\n", pcap_error_buffer); + fprintf(stderr, "pcap_open_live / pcap_open_offline_with_tstamp_precision: %.*s\n", + (int) PCAP_ERRBUF_SIZE, pcap_error_buffer); free_workflow(&workflow); return NULL; } @@ -204,9 +206,25 @@ static void free_workflow(struct nDPI_workflow ** const workflow) *workflow = NULL; } +static char * get_default_pcapdev(char *errbuf) +{ + char * ifname; + pcap_if_t * all_devices = NULL; + + if (pcap_findalldevs(&all_devices, errbuf) != 0) + { + return NULL; + } + + ifname = strdup(all_devices[0].name); + pcap_freealldevs(all_devices); + + return ifname; +} + static int setup_reader_threads(char const * const file_or_device) { - char const * file_or_default_device; + char * file_or_default_device; char pcap_error_buffer[PCAP_ERRBUF_SIZE]; if (reader_thread_count > MAX_READER_THREADS) { @@ -214,23 +232,28 @@ static int setup_reader_threads(char const * const file_or_device) } if (file_or_device == NULL) { - file_or_default_device = pcap_lookupdev(pcap_error_buffer); + file_or_default_device = get_default_pcapdev(pcap_error_buffer); if (file_or_default_device == NULL) { - fprintf(stderr, "pcap_lookupdev: %s\n", pcap_error_buffer); + fprintf(stderr, "pcap_findalldevs: %.*s\n", (int) PCAP_ERRBUF_SIZE, pcap_error_buffer); return 1; } } else { - file_or_default_device = file_or_device; + file_or_default_device = strdup(file_or_device); + if (file_or_default_device == NULL) { + return 1; + } } for (int i = 0; i < reader_thread_count; ++i) { reader_threads[i].workflow = init_workflow(file_or_default_device); if (reader_threads[i].workflow == NULL) { + free(file_or_default_device); return 1; } } + free(file_or_default_device); return 0; } From 13cbd1e4aebf7b665a10360957de9c307decedfc Mon Sep 17 00:00:00 2001 From: Toni Uhlig Date: Sat, 15 Aug 2020 12:38:47 +0200 Subject: [PATCH 21/32] Fixed invalid dpdk fn call. Signed-off-by: Toni Uhlig --- example/ndpiReader.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 03ab1df4a..5202c8b78 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -3121,7 +3121,7 @@ void * processing_thread(void *_thread_id) { gettimeofday(&h.ts, NULL); ndpi_process_packet((u_char*)&thread_id, &h, (const u_char *)data); - rte_pktmbuf_ndpi_free(bufs[i]); + rte_pktmbuf_free(bufs[i]); } } #else From 98a9afc40cb585107507ffce2f0b6910921d8aa1 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Sun, 16 Aug 2020 10:01:40 +0200 Subject: [PATCH 22/32] Added support for discord --- src/include/ndpi_protocol_ids.h | 2 +- src/lib/ndpi_content_match.c.inc | 7 +++++++ src/lib/ndpi_main.c | 4 ++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h index 01f54c0f9..d184ff4a5 100644 --- a/src/include/ndpi_protocol_ids.h +++ b/src/include/ndpi_protocol_ids.h @@ -87,7 +87,7 @@ typedef enum { NDPI_PROTOCOL_ZATTOO = 55, NDPI_PROTOCOL_SHOUTCAST = 56, NDPI_PROTOCOL_SOPCAST = 57, - NDPI_PROTOCOL_FREE_58 = 58, /* Free */ + NDPI_PROTOCOL_DISCORD = 58, NDPI_PROTOCOL_TVUPLAYER = 59, NDPI_PROTOCOL_HTTP_DOWNLOAD = 60, NDPI_PROTOCOL_QQLIVE = 61, diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index 29e2a4277..761ec53d5 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -9138,6 +9138,13 @@ static ndpi_protocol_match host_match[] = { ".net.anydesk.com", "AnyDesk", NDPI_PROTOCOL_ANYDESK, NDPI_PROTOCOL_CATEGORY_REMOTE_ACCESS, NDPI_PROTOCOL_ACCEPTABLE }, + { "discordapp.com", "Discord", NDPI_PROTOCOL_DISCORD, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE }, + { "discordapp.net", "Discord", NDPI_PROTOCOL_DISCORD, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE }, + { "discord.com", "Discord", NDPI_PROTOCOL_DISCORD, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE }, + { "discord.gg", "Discord", NDPI_PROTOCOL_DISCORD, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE }, + { "discord.media", "Discord", NDPI_PROTOCOL_DISCORD, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE }, + + { NULL, NULL, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_SAFE } }; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 35123c1c9..707347c76 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -914,8 +914,8 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp no_master, no_master, "Sopcast", NDPI_PROTOCOL_CATEGORY_VIDEO, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); - ndpi_set_proto_defaults(ndpi_str, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_58, 0 /* can_have_a_subprotocol */, - no_master, no_master, "Free58", NDPI_PROTOCOL_CATEGORY_VIDEO, + ndpi_set_proto_defaults(ndpi_str, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DISCORD, 0 /* can_have_a_subprotocol */, + no_master, no_master, "Discord", NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_str, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_TVUPLAYER, 0 /* can_have_a_subprotocol */, From b501239d3f20fae84cd82b1945a6fc05de32bfe7 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 17 Aug 2020 17:17:09 +0200 Subject: [PATCH 23/32] Added README for building libndpi under OpenWRT --- packages/openwrt/Makefile | 4 ++-- packages/openwrt/README | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 packages/openwrt/README diff --git a/packages/openwrt/Makefile b/packages/openwrt/Makefile index 4b8429b59..ce0b410cf 100644 --- a/packages/openwrt/Makefile +++ b/packages/openwrt/Makefile @@ -10,11 +10,11 @@ PKG_RELEASE:=1 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://github.com/ntop/nDPI.git -PKG_SOURCE_VERSION:=ab2f3cefc89017d73e67faa4eb4011e7e3f2044d +PKG_SOURCE_VERSION:=ebf89f46e3f69d0bb8aa4c836c52ee4964431f6c PKG_SOURCE_SUBDIR:=$(PKG_NAME)-$(PKG_VERSION) PKG_SOURCE_PROTO:=git -PKG_MAINTAINER:=Emanuele Faranda +PKG_MAINTAINER:=Luca Deri PKG_LICENSE:=GPL3 PKG_BUILD_DEPENDS:=+libpcap PKG_BUILD_PARALLEL:=1 diff --git a/packages/openwrt/README b/packages/openwrt/README new file mode 100644 index 000000000..89ae5a8f1 --- /dev/null +++ b/packages/openwrt/README @@ -0,0 +1,18 @@ +Howto Compile lindpi on OpenWRT +------------------------------- + +cd myopenwrt_directory +mkdir package/network/services/libndpi +cd package/network/services/libndpi +cp ~/nDPI/packages/openwrt/Makefile . +cd myopenwrt_directory +make menuconfig + +Go under network and select + + libndpi.............................. nDPI Deep Packet Inspection Library + + +Other Documents +--------------- +https://openwrt.org/packages/pkgdata/libndpi \ No newline at end of file From 34a98abcc0c6cda11ea3468ade724e6e155a6d2e Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 17 Aug 2020 18:00:38 +0200 Subject: [PATCH 24/32] Added --with-only-libndpi configure option to build just lindpi --- Makefile.am | 2 +- configure.seed | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index e4d8c58c9..6238c8b3c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,5 +1,5 @@ ACLOCAL_AMFLAGS = -I m4 -SUBDIRS = src/lib example tests +SUBDIRS = src/lib @EXTRA_TARGETS@ if BUILD_FUZZTARGETS SUBDIRS += fuzz diff --git a/configure.seed b/configure.seed index b344064cd..db4cf1b25 100644 --- a/configure.seed +++ b/configure.seed @@ -4,6 +4,14 @@ AC_CONFIG_MACRO_DIR([m4]) AM_INIT_AUTOMAKE([foreign subdir-objects]) +EXTRA_TARGETS="example tests" +AC_ARG_WITH(only-libndpi, AS_HELP_STRING([--with-only-libndpi], [Build only libndpi (no examples, tests etc)])) +AS_IF([test "${with_only_libndpi+set}" = set],[ + EXTRA_TARGETS="" +]) + + + AC_ARG_WITH(sanitizer, AS_HELP_STRING([--with-sanitizer], [Build with support for address, undefined and leak sanitizer])) AC_ARG_ENABLE(fuzztargets, AS_HELP_STRING([--enable-fuzztargets], [Enable fuzz targets]),[enable_fuzztargets=$enableval],[enable_fuzztargets=no]) AM_CONDITIONAL([BUILD_FUZZTARGETS], [test "x$enable_fuzztargets" = "xyes"]) @@ -187,4 +195,5 @@ AC_SUBST(DPDK_TARGET) AC_SUBST(HAVE_PTHREAD_SETAFFINITY_NP) AC_SUBST(CUSTOM_NDPI) AC_SUBST(NDPI_API_VERSION) +AC_SUBST(EXTRA_TARGETS) AC_OUTPUT From 1f921562d1d7962f1d23ca5b59c25f9b65073460 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 17 Aug 2020 18:11:18 +0200 Subject: [PATCH 25/32] Added fix for API versions starting with 0 --- configure.seed | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configure.seed b/configure.seed index db4cf1b25..087150a39 100644 --- a/configure.seed +++ b/configure.seed @@ -67,6 +67,8 @@ else NDPI_API_VERSION=`date +%s | cut -c7-10` fi +NDPI_API_VERSION=`echo $NDPI_API_VERSION | sed 's/^0*//'` + AC_DEFINE_UNQUOTED(NDPI_GIT_RELEASE, "${GIT_RELEASE}", [GIT Release]) AC_DEFINE_UNQUOTED(NDPI_GIT_DATE, "${GIT_DATE}", [Last GIT change]) From 673d8b7fac1f7c1f3865348190a9d48bff58f176 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Mon, 17 Aug 2020 18:17:43 +0200 Subject: [PATCH 26/32] Updated OpenWRT instructions --- packages/openwrt/Makefile | 12 +++++------- packages/openwrt/README | 7 +++++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/openwrt/Makefile b/packages/openwrt/Makefile index ce0b410cf..5d56e18ad 100644 --- a/packages/openwrt/Makefile +++ b/packages/openwrt/Makefile @@ -1,22 +1,22 @@ # -# Copyright (C) 2018 - ntop.org +# Copyright (C) 2018-20 - ntop.org # include $(TOPDIR)/rules.mk PKG_NAME:=libndpi -PKG_VERSION:=1333.ab2f3ce +PKG_VERSION:=17022020 PKG_RELEASE:=1 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://github.com/ntop/nDPI.git -PKG_SOURCE_VERSION:=ebf89f46e3f69d0bb8aa4c836c52ee4964431f6c +PKG_SOURCE_VERSION:=1f921562d1d7962f1d23ca5b59c25f9b65073460 PKG_SOURCE_SUBDIR:=$(PKG_NAME)-$(PKG_VERSION) PKG_SOURCE_PROTO:=git PKG_MAINTAINER:=Luca Deri PKG_LICENSE:=GPL3 -PKG_BUILD_DEPENDS:=+libpcap +PKG_BUILD_DEPENDS:= PKG_BUILD_PARALLEL:=1 # autogen fix @@ -27,7 +27,6 @@ include $(INCLUDE_DIR)/package.mk define Package/libndpi SECTION:=network CATEGORY:=Network - #DEPENDS:=+libc +libjson-c +libpthread TITLE:=nDPI Deep Packet Inspection Library URL:=https://www.ntop.org endef @@ -37,8 +36,7 @@ define Package/libndpi/description endef CONFIGURE_ARGS += \ - --with-pic \ - --disable-json-c \ + --with-only-libndpi define Build/Prepare $(call Build/Prepare/Default) diff --git a/packages/openwrt/README b/packages/openwrt/README index 89ae5a8f1..5a2cf2712 100644 --- a/packages/openwrt/README +++ b/packages/openwrt/README @@ -13,6 +13,13 @@ Go under network and select libndpi.............................. nDPI Deep Packet Inspection Library +Build Commands +-------------- + +If you want to build just libndpi do: +make -j1 V=s package/network/services/libndpi/clean +make -j1 V=s package/network/services/libndpi/compile + Other Documents --------------- https://openwrt.org/packages/pkgdata/libndpi \ No newline at end of file From 8e93f48c43f270414818c87335296b23014334a6 Mon Sep 17 00:00:00 2001 From: Toni Uhlig Date: Tue, 18 Aug 2020 16:21:26 +0200 Subject: [PATCH 27/32] Added support for SOAP. Signed-off-by: Toni Uhlig --- src/include/ndpi_protocol_ids.h | 1 + src/include/ndpi_protocols.h | 1 + src/include/ndpi_typedefs.h | 3 ++ src/lib/ndpi_main.c | 7 ++++ src/lib/protocols/soap.c | 70 +++++++++++++++++++++++++++++++++ 5 files changed, 82 insertions(+) create mode 100644 src/lib/protocols/soap.c diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h index 01f54c0f9..c653429ef 100644 --- a/src/include/ndpi_protocol_ids.h +++ b/src/include/ndpi_protocol_ids.h @@ -282,6 +282,7 @@ typedef enum { NDPI_PROTOCOL_MSTEAMS = 250, NDPI_PROTOCOL_WEBSOCKET = 251, /* Leonn Paiva */ NDPI_PROTOCOL_ANYDESK = 252, /* Toni Uhlig */ + NDPI_PROTOCOL_SOAP = 253, /* Toni Uhlig */ #ifdef CUSTOM_NDPI_PROTOCOLS #include "../../../nDPI-custom/custom_ndpi_protocol_ids.h" diff --git a/src/include/ndpi_protocols.h b/src/include/ndpi_protocols.h index 417c6fb8d..392abf9e1 100644 --- a/src/include/ndpi_protocols.h +++ b/src/include/ndpi_protocols.h @@ -213,5 +213,6 @@ void init_dnp3_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int void init_104_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_s7comm_dissector(struct ndpi_detection_module_struct *ndpi_struct,u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); void init_websocket_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); +void init_soap_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask); #endif /* __NDPI_PROTOCOLS_H__ */ diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 40c27329e..d5baf9fe7 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -711,6 +711,9 @@ struct ndpi_flow_tcp_struct { /* NDPI_PROTOCOL_MAIL_IMAP */ u_int32_t mail_imap_stage:3, mail_imap_starttls:2; + /* NDPI_PROTOCOL_SOAP */ + u_int32_t soap_stage:1; + /* NDPI_PROTOCOL_SKYPE */ u_int8_t skype_packet_id; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 35123c1c9..0a4ee3a42 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -1493,6 +1493,10 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp no_master, "AnyDesk", NDPI_PROTOCOL_CATEGORY_REMOTE_ACCESS, ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); + ndpi_set_proto_defaults(ndpi_str, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_SOAP, 1 /* no subprotocol */, + no_master, no_master, "SOAP", NDPI_PROTOCOL_CATEGORY_RPC, + ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, + ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); #ifdef CUSTOM_NDPI_PROTOCOLS #include "../../../nDPI-custom/custom_ndpi_main.c" @@ -3311,6 +3315,9 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n /* WEBSOCKET */ init_websocket_dissector(ndpi_str, &a, detection_bitmask); + /* SOAP */ + init_soap_dissector(ndpi_str, &a, detection_bitmask); + #ifdef CUSTOM_NDPI_PROTOCOLS #include "../../../nDPI-custom/custom_ndpi_main_init.c" #endif diff --git a/src/lib/protocols/soap.c b/src/lib/protocols/soap.c new file mode 100644 index 000000000..dfbaf6c1e --- /dev/null +++ b/src/lib/protocols/soap.c @@ -0,0 +1,70 @@ +/* + * soap.c + * + * Copyright (C) 2020 - ntop.org + * + * nDPI is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * nDPI is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with nDPI. If not, see . + * + */ + +#include "ndpi_protocol_ids.h" + +#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_SOAP + +#include "ndpi_api.h" + +static void ndpi_int_soap_add_connection(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) +{ + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_SOAP, NDPI_PROTOCOL_UNKNOWN); +} + +void ndpi_search_soap(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) +{ + struct ndpi_packet_struct *packet = &flow->packet; + + NDPI_LOG_DBG(ndpi_struct, "search soap\n"); + + if (flow->packet_counter > 3) + { + if (flow->l4.tcp.soap_stage == 1) + { + ndpi_int_soap_add_connection(ndpi_struct, flow); + } + else { + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); + } + } + + if (flow->l4.tcp.soap_stage == 0 && + packet->payload_packet_len >= 19) + { + if (strncmp((char*)packet->payload, "l4.tcp.soap_stage = 1; + } + } +} + +void init_soap_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, + NDPI_PROTOCOL_BITMASK *detection_bitmask) +{ + ndpi_set_bitmask_protocol_detection( + "SOAP", ndpi_struct, detection_bitmask, *id, + NDPI_PROTOCOL_SOAP, ndpi_search_soap, NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD, + SAVE_DETECTION_BITMASK_AS_UNKNOWN, ADD_TO_DETECTION_BITMASK); + *id += 1; +} + From 3a767861216cc6b5a8f40357ce3c0b5e9a7da485 Mon Sep 17 00:00:00 2001 From: Toni Uhlig Date: Wed, 19 Aug 2020 18:33:06 +0200 Subject: [PATCH 28/32] Updated MySQL protocol detection to support server version 8. Signed-off-by: Toni Uhlig --- src/lib/protocols/mysql.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/protocols/mysql.c b/src/lib/protocols/mysql.c index 485456e91..aa95d46c4 100644 --- a/src/lib/protocols/mysql.c +++ b/src/lib/protocols/mysql.c @@ -40,7 +40,7 @@ void ndpi_search_mysql_tcp(struct ndpi_detection_module_struct *ndpi_struct, str && get_u_int8_t(packet->payload, 2) == 0x00 //3rd byte of packet length && get_u_int8_t(packet->payload, 3) == 0x00 //packet sequence number is 0 for startup packet && get_u_int8_t(packet->payload, 5) > 0x30 //server version > 0 - && get_u_int8_t(packet->payload, 5) < 0x37 //server version < 7 + && get_u_int8_t(packet->payload, 5) < 0x39 //server version < 9 && get_u_int8_t(packet->payload, 6) == 0x2e //dot ) { #if 0 From 115888ef7aeb66a461e3f5df98d3e5d43ade1c89 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Wed, 19 Aug 2020 21:56:34 +0200 Subject: [PATCH 29/32] Compilation fix --- src/include/ndpi_includes.h | 2 ++ src/include/ndpi_includes_OpenBSD.h | 10 +--------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/include/ndpi_includes.h b/src/include/ndpi_includes.h index 99c50fe02..197f32fce 100644 --- a/src/include/ndpi_includes.h +++ b/src/include/ndpi_includes.h @@ -69,6 +69,8 @@ #if defined __OpenBSD__ #include "ndpi_includes_OpenBSD.h" +#else +typedef struct timeval pkt_timeval; #endif /* __OpenBSD__ */ #endif /* __NDPI_INCLUDES_H__ */ diff --git a/src/include/ndpi_includes_OpenBSD.h b/src/include/ndpi_includes_OpenBSD.h index 4efdbd844..65716c8f3 100644 --- a/src/include/ndpi_includes_OpenBSD.h +++ b/src/include/ndpi_includes_OpenBSD.h @@ -24,20 +24,12 @@ #ifndef __NDPI_INCLUDES_OPENBSD_H__ #define __NDPI_INCLUDES_OPENBSD_H__ -#ifdef __OpenBSD__ - #ifndef IPPROTO_SCTP #define IPPROTO_SCTP 132 #endif /* IPPROTO_SCTP */ -#endif /* __OpenBSD__ */ - - -#ifdef __OpenBSD__ #include + typedef struct bpf_timeval pkt_timeval; -#else -typedef struct timeval pkt_timeval; -#endif /* __OpenBSD__ */ #endif /* __NDPI_INCLUDES_OPENBSD_H__ */ From 59ac73b37f6d058b9058bc9c6e7d652f8f0eeb90 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Wed, 19 Aug 2020 22:01:03 +0200 Subject: [PATCH 30/32] Configure code cleanup --- configure.seed | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/configure.seed b/configure.seed index 087150a39..ecde0579b 100644 --- a/configure.seed +++ b/configure.seed @@ -78,16 +78,15 @@ ADDITIONAL_LIBS= PCAP_HOME=$HOME/PF_RING/userland DPDK_TARGET= +AC_MSG_CHECKING([DPDK (used by ndpiReader)]) if test -d $HOME/DPDK; then : - echo "Enabling DPDK support in ndpiReader" + AC_MSG_RESULT(yes) DPDK_TARGET=dpdk else - echo "DPDK support disabled (missing $HOME/DPDK)" + AC_MSG_RESULT([no (missing $HOME/DPDK)]) fi -if test -d $PCAP_HOME; then : - echo -n "" -else +if ! test -d $PCAP_HOME; then : PCAP_HOME=`pwd`/../../PF_RING/userland fi SHORT_MACHINE=`uname -m | cut -b1-3` From f4421314b04b6771aeb7d62f08dfb4fa81647ec0 Mon Sep 17 00:00:00 2001 From: Toni Uhlig Date: Thu, 20 Aug 2020 23:41:46 +0200 Subject: [PATCH 31/32] Added (manipulated) MySQL 8 test pcap. Signed-off-by: Toni Uhlig --- tests/pcap/mysql-8.pcap | Bin 0 -> 455 bytes tests/result/mysql-8.pcap.out | 3 +++ 2 files changed, 3 insertions(+) create mode 100644 tests/pcap/mysql-8.pcap create mode 100644 tests/result/mysql-8.pcap.out diff --git a/tests/pcap/mysql-8.pcap b/tests/pcap/mysql-8.pcap new file mode 100644 index 0000000000000000000000000000000000000000..7cd92c7e04da583d87ede6cb27d76f5c8d3e499a GIT binary patch literal 455 zcmca|c+)~A1{MYcU}0bcavS|m+oWbgvAL0Ab4T)=>X!Igo*#)990!GTSA&4Cq+ znOs^z$CQ+KUQJ?kR0Wx|fT>8}_+6g6!GXb^EoL{!RFLUBuauO8e2)YJO;=kWRH*P^Ba+FCf^i_zfhMzqOb(mM$lwID z34}4+Y+@k*cJpR5H>&|n4`~o!JXv=Y=tf4Udx0i{OpKk*$S@OV8VIAgm4l)CDi_GD z?U8714FS2eL4x6N_D!gXP 10.10.0.1:3306 [proto: 20/MySQL][cat: Database/11][2 pkts/140 bytes <-> 2 pkts/227 bytes][Goodput ratio: 0/38][0.00 sec][PLAIN TEXT (DDDDDD)][Plen Bins: 0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] From da2684dbe17dd45d3c0b9534d1e2a01cce1168b7 Mon Sep 17 00:00:00 2001 From: Luca Deri Date: Fri, 21 Aug 2020 07:17:34 +0200 Subject: [PATCH 32/32] MySQL8 update --- tests/result/mysql-8.pcap.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/result/mysql-8.pcap.out b/tests/result/mysql-8.pcap.out index bc8c2b34d..5e2b2da0f 100644 --- a/tests/result/mysql-8.pcap.out +++ b/tests/result/mysql-8.pcap.out @@ -1,3 +1,3 @@ MySQL 4 367 1 - 1 TCP 192.168.1.1:8738 <-> 10.10.0.1:3306 [proto: 20/MySQL][cat: Database/11][2 pkts/140 bytes <-> 2 pkts/227 bytes][Goodput ratio: 0/38][0.00 sec][PLAIN TEXT (DDDDDD)][Plen Bins: 0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 1 TCP 192.168.1.105:8738 <-> 10.42.18.198:3306 [proto: 20/MySQL][cat: Database/11][2 pkts/140 bytes <-> 2 pkts/227 bytes][Goodput ratio: 0/38][0.00 sec][PLAIN TEXT (DDDDDD)][Plen Bins: 0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]