From 44459895889042e8d4e434c2f2b5cdece15a5728 Mon Sep 17 00:00:00 2001 From: Ivan Nardi <12729895+IvanNardi@users.noreply.github.com> Date: Mon, 4 Jul 2022 13:21:11 +0200 Subject: [PATCH] Update host content list match (#1633) Improve classifications of Outlook, Cachefly, Cloudflare, Tiktok and Cybersecurity. --- src/lib/ndpi_content_match.c.inc | 10 ++++++---- tests/pcap/cachefly.pcapng | Bin 0 -> 6476 bytes tests/result/cachefly.pcapng.out | 13 +++++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 tests/pcap/cachefly.pcapng create mode 100644 tests/result/cachefly.pcapng.out diff --git a/src/lib/ndpi_content_match.c.inc b/src/lib/ndpi_content_match.c.inc index db535fa81..f8d5bcfe3 100644 --- a/src/lib/ndpi_content_match.c.inc +++ b/src/lib/ndpi_content_match.c.inc @@ -1091,6 +1091,7 @@ static ndpi_protocol_match host_match[] = { ".goog", "Google", NDPI_PROTOCOL_GOOGLE, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }, { "mail.outlook.com", "Outlook", NDPI_PROTOCOL_MS_OUTLOOK, NDPI_PROTOCOL_CATEGORY_MAIL, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, + { "hotmail.com", "Outlook", NDPI_PROTOCOL_MS_OUTLOOK, NDPI_PROTOCOL_CATEGORY_MAIL, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, { ".last.fm", "LastFM", NDPI_PROTOCOL_LASTFM, NDPI_PROTOCOL_CATEGORY_MUSIC, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL }, @@ -1343,10 +1344,7 @@ static ndpi_protocol_match host_match[] = { "office.live.com", "Microsoft365", NDPI_PROTOCOL_MICROSOFT_365, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, { ".onenote.", "Microsoft365", NDPI_PROTOCOL_MICROSOFT_365, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, - /* http://www.urlquery.net/report.php?id=1453233646161 */ - { "lifedom.top", "Cloudflare", NDPI_PROTOCOL_CLOUDFLARE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, - { "coby.ns.cloudflare.com", "Cloudflare", NDPI_PROTOCOL_CLOUDFLARE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, - { "amanda.ns.cloudflare.com", "Cloudflare", NDPI_PROTOCOL_CLOUDFLARE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, + { "cloudflare.com", "Cloudflare", NDPI_PROTOCOL_CLOUDFLARE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, { "d295hzzivaok4k.cloudfront.net","OpenDNS", NDPI_PROTOCOL_OPENDNS, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, { ".opendns.com", "OpenDNS", NDPI_PROTOCOL_OPENDNS, NDPI_PROTOCOL_CATEGORY_NETWORK, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, @@ -1462,6 +1460,7 @@ static ndpi_protocol_match host_match[] = { ".tik-tokcdn.com", "TikTok", NDPI_PROTOCOL_TIKTOK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL }, { "v19.byteicdn.com", "TikTok", NDPI_PROTOCOL_TIKTOK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL }, { "v16m.byteicdn.com", "TikTok", NDPI_PROTOCOL_TIKTOK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL }, + { ".tiktokcdn-us.com", "TikTok", NDPI_PROTOCOL_TIKTOK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL }, { "brasilbandalarga.com.br", "EAQ", NDPI_PROTOCOL_EAQ, NDPI_PROTOCOL_CATEGORY_NETWORK, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, { ".eaqbr.com.br", "EAQ", NDPI_PROTOCOL_EAQ, NDPI_PROTOCOL_CATEGORY_NETWORK, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, @@ -1602,6 +1601,7 @@ static ndpi_protocol_match host_match[] = { "avira.com", "Cybersec", NDPI_PROTOCOL_CYBERSECURITY, NDPI_PROTOCOL_CATEGORY_CYBERSECURITY, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }, { "pandasecurity.com", "Cybersec", NDPI_PROTOCOL_CYBERSECURITY, NDPI_PROTOCOL_CATEGORY_CYBERSECURITY, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }, { "avast.com", "Cybersec", NDPI_PROTOCOL_CYBERSECURITY, NDPI_PROTOCOL_CATEGORY_CYBERSECURITY, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }, + { ".avcdn.net", "Cybersec", NDPI_PROTOCOL_CYBERSECURITY, NDPI_PROTOCOL_CATEGORY_CYBERSECURITY, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }, { "malwarebytes.com", "Cybersec", NDPI_PROTOCOL_CYBERSECURITY, NDPI_PROTOCOL_CATEGORY_CYBERSECURITY, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }, { "trendmicro.com", "Cybersec", NDPI_PROTOCOL_CYBERSECURITY, NDPI_PROTOCOL_CATEGORY_CYBERSECURITY, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }, { ".eset.com", "Cybersec", NDPI_PROTOCOL_CYBERSECURITY, NDPI_PROTOCOL_CATEGORY_CYBERSECURITY, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }, @@ -1700,6 +1700,8 @@ static ndpi_protocol_match host_match[] = { "psiphon3.net", "Psiphon", NDPI_PROTOCOL_PSIPHON, NDPI_PROTOCOL_CATEGORY_VPN, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, { ".psiphon3.net", "Psiphon", NDPI_PROTOCOL_PSIPHON, NDPI_PROTOCOL_CATEGORY_VPN, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, + { "cachefly.com", "CacheFly", NDPI_PROTOCOL_CACHEFLY, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL }, + #ifdef CUSTOM_NDPI_PROTOCOLS #include "../../../nDPI-custom/custom_ndpi_content_match_host_match.c.inc" #endif diff --git a/tests/pcap/cachefly.pcapng b/tests/pcap/cachefly.pcapng new file mode 100644 index 0000000000000000000000000000000000000000..7c0c7beeb9d05cdf14e0ee5140faf8dae28e597a GIT binary patch literal 6476 zcmdT|c~le0*6+?HyFel!h(u%;LI)H983h**#RX76TreadiG+0QP69#3C2B?y6%kwz z6-N|AVHg<|)DZ+%P#Hxm|2Kq!}f{VBpgNtY89Q>I~R*sJL3_m-4zg6%-ua4qo)$)*ePjA_l&G^(N zUZ;rJu!ZTj)309b@vB{EX@$=*pC0ZHuVV`)dverf~# zm8?Uh!a6h>3}g`G36S6ni~&7sh?(=IjoTTS`PSFoWZQApxWlwJNi({d6!ERRv(7Iv z6TXMsfHwj>`g*>;WD1!~CXr#t3L=vr(o_-fM+W{=QweLspz|8n|F8cM_#$vt0H91X z0%w&A0(;eSQXXXSf4WZAT+M|DD#?hs_D#ehCUAwvM6+PrJ` z9hfq$(}zrJA=4`$L5ND9hNyH~bIBx_L^7ar!zo4d)`_zhl}Zsqny!m_y84|V4C_-N zWG-7D(WQ~+Q)qP3GqZp3!*)RIh#@lFdeBf4C7lk&)(x9&9$a^FTbK|rxiHu(6+9FIay^m?V7a2_d*ijUwwYq5?&_S!mbuPIK41^Ec4Q36IREKt+FoQY z-P~yS_YRBw(heQ&Zn~Utx93(w*#SR6RP%jEITCqm{dVGXQ%~_PHRcg1Z+~5P0pGBs z^x*X-AE)Pj+cWk&*r4=!7Io9a>OwRlD&EK>r?9@WqI9P20^dKflRe!EgRW1L)&*NH zf4Ez5fc9)rYxswc1>Bb5s=w*^``Nk^O%^tcUUgP#|NOq~gTl=5 zj#0J4?QPb~3MC%^l1Dr<~QZfM8RJ;VWq52}V-LR3SlG<$A66oz{#6f%jLi>%o~ z(-}j9snj7bOj)xAS%pmP7a%0HH^DT%?(R6BE64aOfd(x&KnshDCb$Fk;$t#*0gmAu zRDt7SE{DJdi;KyS@dH+1^_V^@C>NK0ZR6|7NCe2THIp(2nS}(5F%z-e8gUAvS6Tdcz@ues(lA)N=9RV~bSYrT=hX(%EbP5Rp$e`RKoS0BllseF< zRec31m9c;?W63y4TqRe`Czh%*j78CS++%Fgcum&=*ynS_D9-1xPz4|3VIqZA(S*z5 z@DzNB4D1)CAb4G6#zPY&VwA&Uak!`)mjSSAjL^9}35$c_z>-ix2nl5qlq+R%ajtf& zHF^x7=LS~Em&q_rT%3rjky>%%ScCy7DwptZ87f7&@gQn0%96*chV>0{Ik*s5mE&5E zW_{x2auGKk{Nweh4@8KwPze_V&&M=A=$hvfOMvnD5;o$2XoSOXG)|Vt0dPPiG9h0m zkJX5cxhRj%N|0kb6cuPCq67g8gv{eh`*awh1df0YoP4om=D3UlHid=pIXq>Y7)|73 zxYiS~K1{?FYWewVp0+(*;pyrxdHO^|KTlITzT|0)FXk&aVD%c;mdiQ)Q33z| zj1ej7+ZB#LB<9D8WZGEG1|H=caU90yf#G;jO4f2YN32zpq8O3n1eL|0VU3A_5%X`3@a3vO7Tr*p82{QL zE^`A@5jFutHz^;BL$L&nFFFUu`Cy?+j)bQKDjJ~)aUr&m-09=KAS}#72}Ads7NA{poy9gh7xo6JON)WK{bNm{RzNXQeK=} z5?2UnU~Q7k2XY2N<0lXx)FyMpzyME#a|yJHB)C)rAWy8)v=12yRE)~_+LJYyK?ofa z7nW2k2f+cd6lxPj2hS}K+n3m90HGEK7#|0O(T6@4mRJOMVURXgD3SnI5|x1q3*`}c z(!>Ic4j3Fzv2f9xrEiRY$yjVs|Ky=AtS@Oz&xOU2;37>D^obZ0d~HAm0~-RMfmjpl zX-_dj82yY)h6nCQhz&73nqq;Nt!BJ$Th(zrNl@?z-Dhgpjq7TGVpybhe{lq};+@a6%^r$A&Xi8DANJF>r9 zE^!-?^)O0SwZ!4&vL`~?k~=mernjjp|?fn@F| z#Z@Q~3OqEwMgDOc6FKqYM$zAIzc+CWke%xeE?a-J3>Rss1-+?h}8mA!h2x5WC0^jh6ovg~GUS6z=smS>02 zG3PQzzifyS2I`U7h>Gl`E$R*7))-ss&#@D3d-|@A{3(e?h(=ogX)3yF%J?agL`YF^WbihgkSzwi$6pDOqK=&N@OEJi`2 z@9Z4fQrxsi7$3iN?oxX4n%bSMTT(fhGaZy&lzf*@c@HAawC3i$S&_PGzD>|}j-h0E zsIR@9j(ghEcN_0!)csVNKMDEk#N$!LJ)Kj{yYF%DyVtLYj*qDiDmm71bb_67j?HD( zpXI}RQ!GM6r6-sRbeA|+b6>Cm7Zzrw2OE{Y>6~#u*F9zItl!Q;7isgAv>QMEX52n~ z%FFWT6It_rW#6*;PJQmcYeUmU*WGqSK~*UEMY8mS^xAbt*F$Mrsx166uCj2HW*o8I zzwqG2uushwsdq)fMYQtP(2Y-1jRI!v$w=4R_{uE4txbL|Ik4>apj7J-to%8Bl{@SV zDAE4Q${kiCrUcE>iAr7vp@ARpm0%FFev^0yGxOFrGglR`rr8zGn){KPz3I`jCabf^ z#C{qFF9RL`2CV)*0;_EYRq-DGqvi&f!#oi$A|@M(JK~0n$#u?kRNMDW;$q@&O>x6K z!P|lu2_oh(DgMYr#E(V+x6uEx-v9EwS3_L8)33P2OisP6AJ@2xjGNrvpbm2iPyJN% zxEtT%E1)!1hcl!M-IVGl?;A$!vi$MF#Ku?9?={ln?Y79Mh{pON<&rf~`U#BH<2upB z)-BmV)ZPV~p`ZO9r0?2cd0L#XDy_40_2V=i?zKay6v_Uqc)t(cnwurs{A%6lSIxIK zcsC|Z*!7#&3g?nrVHW9+!lppQ%N(yT_NlXuJzdR4R%Y9u*>sFD`tWY(W&16M*F#rM zNF9^mO}E~vzOD`ndNDR-VIFB>DksHO5gwfx?x`-zD8PO?@FDC68}^~kC2Jzxum64H zVV7N9{^!o59(?MJdrukES4R9$eWWYo$>)4L!t7Z6Geq9O%pNkyOX4^JESed8d$ zd!T@)-+(dV`ue*E<=GuQf~PYhzvStPM;e~CfNW^_U-C3z0Ui|obvvTdpbu~`=x~U42i$xo-&w->Vmyf6JrNr3PvSvPA zstSA>cGO@^kW}2nyV=s|dT^-fJA*OBI~z_i)W*s)6X>fG({!E&-&z*yZnNa$4&mdC zj#=LOX$=#v);cUO#2=MhdF5cVcG#u1g_~|smevFxcIpyM=&HTqw5@q>DkpOMF&X2b z?1^tH=FZCb^xK(=&hMb7G|!i+F~#ABea-N+rT`Gki(e&bFe&Q@rftikS2Cwmq?*48>38iOi0W zA4p!U-M(CLNwi*NuBEKWi2qk~Mbmu$(b5$aAp^P^#8uyhM%nH`XBz_C)gTJ1E%vnKw9to$_OR-C6--`UWZW%KY^ETvhx;C zpJ~-wo|i|83qPCgcpZMp?YW(}Jd^wBVYpBbF>!5K-MCTJSA{({9#n-FUQ17X^=5Hk zT2@nXl#`{q^`q<1_<*f{O-*PBciOq~t$;FeSVQWLU+N!)IOc^|;ke>&^W9Z8$wt4s z=@jXWaP9RwI^SsMwv}Ej)seyHp0eChmTsyJ2&IqSd*hu)d?sA5`wv%78{Y}`x77YO zCb+7Tu5#QNr&}B6qmS3MQgvtj>U0mgQ6D>(Ke8ob=UVpt(i1g7QIlf8_dj@+0R#t= zA@cx=TejixSZB#>Pj$`~pi2JbHCf)Wz-X$okB? zoMoXS{UW+A)PI)q@SXyvJ)!Kn$?MsHLJ;CJJV%-&a0MH>FRZ#KlVmnGgTNI zc{aO6Z*kt@Or0@rU72OPAl*Mc1dwAZsuUp`Q`)+Jm;M>w`*T`g?Zx=rBfwMy5B>{8p|A=7 literal 0 HcmV?d00001 diff --git a/tests/result/cachefly.pcapng.out b/tests/result/cachefly.pcapng.out new file mode 100644 index 000000000..107f767ed --- /dev/null +++ b/tests/result/cachefly.pcapng.out @@ -0,0 +1,13 @@ +Guessed flow protos: 0 + +DPI Packets (TCP): 6 (6.00 pkts/flow) +Confidence DPI : 1 (flows) + +Cachefly 6 6163 1 + +JA3 Host Stats: + IP Address # JA3C + 1 10.10.10.1 1 + + + 1 TCP 10.10.10.1:443 <-> 192.168.0.1:43766 [proto: 91.289/TLS.Cachefly][Encrypted][Confidence: DPI][cat: Cloud/13][5 pkts/5580 bytes <-> 1 pkts/583 bytes][Goodput ratio: 94/89][0.35 sec][Hostname/SNI: apptv.cachefly.net][ALPN: h2;http/1.1][TLS Supported Versions: GREASE;TLSv1.3;TLSv1.2;TLSv1.1;TLSv1][bytes ratio: 0.811 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 88/0 351/0 152/0][Pkt Len c2s/s2c min/avg/max/stddev: 74/583 1116/583 1414/583 524/0][TLSv1.2][JA3C: cd08e31494f9531f560d64c695473da9][ServerNames: *.cachefly.net,get.taxcycle.com,books24x7.com,siteclosed.overdrive.com,c.adventurerv.net,download.acoustica.com,cdn.arstechnica.net,ocp.cscglobal.com,cdn-w.gettraffic.com,cf.cdn.poundstopocket.co.uk,cf.cdn.cashnetusa.com,cf.cdn.quickquid.co.uk,downloads.oncenter.com,cache.green1020.com,software.onthehub.com,code.murdoog.com,img.tradepub.com,images.overdrive.com,static.readyflowers.com,cdn.richrelevance.com,qastatic.richrelevance.net,cache.agilebits.com,cachefly.alfredapp.com,download.fosshub.com,cdncontent.skillsoftcompliance.com,cdnlibrary.qual.skillport.com,cdnlibrary.skillport.com,cdnlibrary.skillport.eu,cdnlibrary-otls.skillport.com,st-cdn01.net-perform.com,assets.yandycdn.com,cdn.nexternal.com,www.workcred.org,img.sedoparking.com,www.standardsboostbusiness.org,cdn.sparklingsociety.net,smartupdate1.centralpointnow.com,cdn.edgeuno.com,downloads.pdf-xchange.com,cachefly.kinematics.com,cachefly.discoverinspire.com,static.volotea.com,*.cachefly.com,*.pluralsight.com,*.cdn.overdrive.com,*.contentreserve.com,*.listen.overdrivechina.cn,*.od-cdn.com,*.overdrivechina.cn,*.read.overdrivechina.cn,*.rbxcdn.com,*.books24x7.com,*.ansi.org,*.livee.com,cachefly.net][JA3S: 8d2a028aa94425f76ced7826b1f39039][Issuer: C=BE, O=GlobalSign nv-sa, CN=GlobalSign RSA OV SSL CA 2018][Subject: C=US, ST=Illinois, L=Chicago, O=Cachenetworks, LLC, CN=*.cachefly.net][Certificate SHA-1: 14:84:4F:1F:E8:A1:78:8A:12:27:36:B8:42:AB:42:52:FC:3B:C4:BA][Chrome][Validity: 2021-10-18 20:21:03 - 2022-11-19 20:21:03][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,0,0,0,0,60,0,0,0,0,0]