diff --git a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf index 9d3957837..ca8905b0d 100644 --- a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf @@ -33,4 +33,4 @@ # 111.111.111.111 0; -35.193.14.140 0; +35.193.7.13 0; diff --git a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf index 31d75b8fa..494a0ba46 100644 --- a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1628 -### Updated: Wed Jun 26 14:46:12 SAST 2019 +### Version: V4.2019.06.1629 +### Updated: Wed Jun 26 14:52:13 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf index 9d3957837..ca8905b0d 100644 --- a/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf @@ -33,4 +33,4 @@ # 111.111.111.111 0; -35.193.14.140 0; +35.193.7.13 0; diff --git a/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf index f93637b4a..22cb3f271 100644 --- a/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf @@ -10,8 +10,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1628 -### Updated: Wed Jun 26 14:46:13 SAST 2019 +### Version: V3.2019.06.1629 +### Updated: Wed Jun 26 14:52:14 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/_conf_files_testing_changes_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_testing_changes_ip_whitelist/conf.d/globalblacklist.conf index 9219403a9..31d75b8fa 100644 --- a/.dev-tools/_conf_files_testing_changes_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_testing_changes_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1627 -### Updated: Wed Jun 26 12:53:58 SAST 2019 +### Version: V4.2019.06.1628 +### Updated: Wed Jun 26 14:46:12 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/blacklist-ips.conf index 966492772..98e259caa 100644 --- a/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1 @@ -35.184.96.71 1; 127.0.0.1 1; diff --git a/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/blockbots.conf b/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/blockbots.conf index 837b4fbe7..0476714db 100644 --- a/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/blockbots.conf +++ b/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/blockbots.conf @@ -31,6 +31,14 @@ ####################################################################### +# ENABLE WHITELISTING TO WORK +# This should over rule any other rule +if ($remote_addr ~ "(127.0.0.1)" ) { + #set $bad_bot '0'; + set $validate_client '0'; +} + + # BOTS # **** #limit_conn bot1_connlimit 100; diff --git a/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/whitelist-ips.conf index 2f692b069..3930c5e1c 100644 --- a/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1 @@ -35.184.96.71 0; 127.0.0.1 0; diff --git a/.dev-tools/_conf_files_testing_changes_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_testing_changes_ratelimiting/conf.d/globalblacklist.conf index b41ed8d1c..31d75b8fa 100644 --- a/.dev-tools/_conf_files_testing_changes_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_testing_changes_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1626 -### Updated: Wed Jun 26 12:28:35 SAST 2019 +### Version: V4.2019.06.1628 +### Updated: Wed Jun 26 14:46:12 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/blacklist-ips.conf index 966492772..98e259caa 100644 --- a/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1 @@ -35.184.96.71 1; 127.0.0.1 1; diff --git a/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/blockbots.conf b/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/blockbots.conf index 837b4fbe7..0476714db 100644 --- a/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/blockbots.conf +++ b/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/blockbots.conf @@ -31,6 +31,14 @@ ####################################################################### +# ENABLE WHITELISTING TO WORK +# This should over rule any other rule +if ($remote_addr ~ "(127.0.0.1)" ) { + #set $bad_bot '0'; + set $validate_client '0'; +} + + # BOTS # **** #limit_conn bot1_connlimit 100; diff --git a/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/whitelist-ips.conf index 2f692b069..3930c5e1c 100644 --- a/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1 @@ -35.184.96.71 0; 127.0.0.1 0; diff --git a/.dev-tools/_conf_files_testing_changes_whitelist/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_testing_changes_whitelist/conf.d/globalblacklist.conf index b41ed8d1c..31d75b8fa 100644 --- a/.dev-tools/_conf_files_testing_changes_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_testing_changes_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1626 -### Updated: Wed Jun 26 12:28:35 SAST 2019 +### Version: V4.2019.06.1628 +### Updated: Wed Jun 26 14:46:12 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index f93637b4a..22cb3f271 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -10,8 +10,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1628 -### Updated: Wed Jun 26 14:46:13 SAST 2019 +### Version: V3.2019.06.1629 +### Updated: Wed Jun 26 14:52:14 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 31d75b8fa..494a0ba46 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1628 -### Updated: Wed Jun 26 14:46:12 SAST 2019 +### Version: V4.2019.06.1629 +### Updated: Wed Jun 26 14:52:13 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 95bbdc1fe..bf016a0c7 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,50 +1,50 @@ -Abonti -Bigfoot -BuiltBotTough -CherryPicker -Cliqzbot -Collector -crawl.sogou.com -Digincore -eCatch -EroCrawler -FDM -GrapeFX +Aipbot +AlphaBot +BatchFTP +Bullseye +Copyscape +DataCha0s +Demon +DnyzBot +Dragonfly +Ebingbong +Getintent +Gigablast +G-i-g-a-b-o-t +GT::WWW HybridBot -IDBot -IRLbot -Iskanie -Jbrofuzz -Lanshanbot -LinqiaRSSBot -Microsoft Data Access -Mojeek -NetMechanic -NetZIP -Nikto -Openfind -OutfoxBot -Pavuk -PECL::HTTP -Picsearch -plumanalytics +InfoNaviRobot +LinkextractorPro +lwp-request +MarkMonitor +meanpathbot +MJ12bot +MS Web Services Client Protocol +Navroad +NearSite +Netvibes +NICErsPRO +oBot +Openvas +Probethenet RankingBot2 -RankurBot -Rogerbot -SemrushBot -SEOstats -Snoopy -spyfu -Telesphoreo -Toata -Turingos -UnisterBot -Virusdie -WallpapersHD -WEBDAV -WebImageCollector +Scrapy +Semrush +sexsearcher +SiteExplorer +SocialRankIOBot +sp_auditbot +TeleportPro +Telesphorep +The Intraformant +Thumbor +True_Robot +Vagabondo +VeriCiteCrawler +VidibleScraper +WBSearchBot +WebLeacher Webshag -WebStripper -WinHTTrack -Woobot +WiseGuys Robot +x22Mozilla Zeus diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 31b00183e..76cce7a4e 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -404checker +360Spider 404enemy -80legs Abonti -Aboundex Acunetix ADmantX -Aipbot -Alligator -Apexoo +AhrefsBot +AllSubmitter +Anarchie ASPSeek +Asterias autoemailspider BackDoorBot backlink-check -BackStreet +BacklinkCrawler BackWeb +Bandit BatchFTP BDCbot -Bitacle +BDFetch Blackboard +Black Hole BlackWidow BLEXBot Blow -Boardreader BuiltWith +Bullseye BunnySlippers -CATExplorador -CCBot -Cegbfeieh -CheeseBot -ChinaClaw -Chlooe -Cliqzbot +BuzzSumo +CherryPicker Cloud mapping -coccocbot-web +Cogentbot Collector com.plumanalytics -Craftbot +CopyRightCheck +Cosmos crawler.feedback crawl.sogou.com -CrazyWebCrawler -CrunchBot -Curious -demandbase-bot +Crescent +CSHttp +Custo +DBLBot Demon -Devil -Digincore -DigitalPebble -DIIbot -Disco -Discoverybot -DnyzBot +Deusu +Dispatch +DittoSpyder +DomainAppender DomainSigmaCrawler +DomainStatsBot Dotbot +Drip DSearch DTS Agent +EasyDL +Ebingbong +eCatch +ECCP/1.0 Ecxi -EMail Siphon EMail Wolf -EroCrawler evc-batch -Exabot -Express WebPictures -ExtLinksBot +Evil Extractor -Firefox/7.0 -FyberSpider -Fyrebot -GalaxyBot -Genieo -Getintent +Extreme Picture Finder +FemtosearchBot +FHscan +Fimap +Foobot +FrontPage GetRight -GetWeb -G-i-g-a-b-o-t -GoZilla -Go!Zilla -GrapeFX +Gigablast +Gigabot +Gotit +Grabber +GrabNet +Grafula GridBot HaosouSpider Harvest +Havij Heritrix HMView -HTMLparser HTTP::Lite -IDBot +HTTrack +Iblog Id-search -IlseBot -IndeedBot +Image Fetch +Image Sucker +Intelliseek +InterGET Internet Ninja -internetVista monitor +InternetSeer ips-agent -Iria IRLbot +Iskanie IstellaBot +JamesBOT +JetCar +Jetty +JOC Web Spider Jorgee -Jyxobot -Kenjin Spider Kozmosbot -Lanshanbot Larbin +LeechFTP LeechGet -LibWeb -Lightspeedsystems +Libwhisker Likse Linkdexbot +LinkextractorPro +LinkScan LinksManager -LinkWalker LinqiaMetadataDownloaderBot -LinqiaRSSBot -Litemage_walker -Ltx71 +LinqiaScrapeBot +Lmspider lwp-request -LWP::Simple -Magnet -MarkMonitor -Masscan +lwp-trivial +Majestic12 +Majestic SEO +Majestic-SEO +Mata Hari MauiBot -meanpathbot -MeanPath Bot -Mediatoolkitbot +mediawords MegaIndex.ru Metauri -Microsoft Data Access -MIDown tool +MFC_Tear_Sample +MIIxpc +Mister PiX MJ12bot -Mojeek -MSIECrawler -muhstik-scan -Musobot +Name Intelligence Nameprotect +Navroad Nessus -netEstate NE Crawler -Nettrack +NetAnts +Netcraft +NetLyzer +NetMechanic +Net Vampire Netvibes -Nibbler -Niki-bot +NetZIP +Nikto +NimbleCrawler +Nimbostratus +Ninja Nmap -Nutch Offline Explorer -Offline Navigator -OnCrawl OpenLinkProfiler -OrangeBot -OutclicksBot PageAnalyzer Page Analyzer Pandalytics -Pavuk -PHPCrawl +Panscient +PECL::HTTP Picscout Pi-Monster Pixray PleaseCrawl plumanalytics -Pockey +Probethenet +ProPowerBot +ProWebWalker Psbot Pump -PyCurl +PxBroker +QueryN Metasearch Quick-Crawler -RankFlex RankingBot RealDownload Recorder -RedesScrapy +ReGet Ripper -RocketCrawler +s1z.ru SalesIntelligent -ScanAlert -scan.lol -ScoutJet +Scanbot +Screaming Semrush +SemrushBot SEOkicks SEOkicks-Robot SEOlyticsCrawler -Seomoz -Siphon +SEOprofiler +seoscanners +SEOstats SISTRIX -SiteExplorer -SiteSnagger Site Sucker SlySearch SmartDownload -Snake +Snapbot Snoopy -Sogou web spider -Sosospider +SocialRankIOBot +Sociscraper +sogouspider Sottopop -SpaceBison -Spammen -Spbot -SputnikBot -Sqlmap +SpankBot +sp_auditbot +Sqlworm Sqworm -Steeler -SuperBot -SuperHTTP -SurveyBot -Swiftbot -sysscan +Sucuri +Suzuran +Szukacz T0PHackTeam -T8Abot -Telesphorep Thumbor -TightTwatBot +Titan +Toweyabot Trendiction Trendictionbot +trendiction.de +True_Robot +TwengaBot +UnisterBot +Upflow +URLy Warning Vacuum Vagabondo -VeriCiteCrawler +VB Project +Virusdie +VoidEYE +Voil Voltron -Wallpapers/3.0 WASALive-Bot Webalta WebAuto Web Auto WebBandit WebCollage -Web Collage WebCopier -WEBDAV WebEnhancer Web Enhancer -WebLeacher +WebFetch +WebFuck +WebGo IS +WebmasterWorldForumBot webmeup-crawler WebPix -Website Quester -WebSucker +Webshag +WebsiteExtractor +Webster +WebStripper +Web Sucker +WebZIP WeSEE Whacker -Who.is Bot -Widow WinHTTrack -WISENutbot -Wonderbot -Wotbox +Woobot Wprecon -WPScan -WWW-Collector-E WWW-Mechanize -WWW::Mechanize -WWWOFFLE x09Mozilla x22Mozilla Xaldon WebSpider -Xenu -YoudaoBot +Xaldon_WebSpider Zauba +zauba.io +Zeus zgrab -Zitebot -ZmEu diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 1c659abad..82b90bfe3 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -404checker 404enemy 80legs -Abonti +Acunetix ADmantX +AhrefsBot AiHitBot -Aipbot -Alexibot -AlphaBot +AllSubmitter +Anarchie Apexoo -ASPSeek -autoemailspider +Asterias +BackDoorBot +Backlink-Ceck +backlink-check BackStreet -Badass -Barkrowler +BackWeb BatchFTP Battleztar Bazinga -BetaBot +BDCbot +Bigfoot Blackboard Black Hole BlackWidow -BLEXBot -Boardreader -Bolt -Brandprotect -Buddy +Blow BuiltWith -Bullseye +BunnySlippers Calculon -CATExplorador -Cegbfeieh +CherryPicker +Chlooe Claritybot -coccocbot-web +Cloud mapping Cogentbot cognitiveseo -Collector -com.plumanalytics -Copier -Copyscape -crawler4j -crawl.sogou.com -CSHttp -DBLBot -demandbase-bot +Cosmos +CrazyWebCrawler +Crescent +DatabaseDriverMysqli +DataCha0s +Deusu Digincore DigitalPebble -Dirbuster +DIIbot Discobot Discoverybot DittoSpyder -DomainCrawler -Dotbot +DomainAppender +DomainSigmaCrawler +Dragonfly Drip +DSearch +EasyDL +Ebingbong ECCP/1.0 -EMail Wolf +Ecxi Evil +Exabot +Express WebPictures ExtLinksBot -Extractor -Extreme Picture Finder -Ezooms -facebookscraper -Genieo +EyeNetIE +FDM +FemtosearchBot +FHscan +FlashGet +Flunky +Foobot +FyberSpider +Fyrebot +GalaxyBot GermCrawler -Getintent -Gigabot -Go-Ahead-Got-It +GetWeb +Gigablast Gotit GoZilla Go!Zilla -Grabber Grafula GrapeFX -GrapeshotCrawler -Heritrix -Hloader +GridBot +HaosouSpider HMView +HTMLparser +HTTP::Lite HTTrack Humanlinks -HybridBot -Iblog -Id-search +Image Fetch Image Sucker -InfoNaviRobot +Indy Library instabid InterGET Internet Ninja -InternetSeer ips-agent Iria -Iskanie +IRLbot JamesBOT +Jbrofuzz JennyBot -JetCar -JOC Web Spider -Joomla -Jorgee +JikeSpider JustView -Jyxobot -Kenjin Spider +Keyword Density Lanshanbot -Lftp -Lightspeedsystems +LeechFTP +LeechGet +LibWeb +Libwhisker +Likse +Linkdexbot LinkextractorPro -LinkpadBot -LinkScan -LinkWalker +LinqiaMetadataDownloaderBot LinqiaScrapeBot -Lipperhey Spider +Litemage_walker +Lmspider +LNSpiderguy +Ltx71 lwp-request -lwp-trivial -Mail.RU_Bot -Majestic SEO +LWP::Simple +Mag-Net +magpie-crawler +Majestic-SEO +MarkMonitor MarkWatch -Mata Hari -MauiBot -Meanpathbot -MeanPath Bot -Mediatoolkitbot -mediawords -MegaIndex.ru +Masscan +meanpathbot +MFC_Tear_Sample Microsoft URL Control -MIDown tool -MIIxpc Mojeek -MSFrontPage +Morfeus Fucking Scanner +Mr.4x3 +MSIECrawler Msrabot muhstik-scan -Name Intelligence -Navroad -netEstate NE Crawler +Musobot +Nameprotect +NearSite +Nessus +NetAnts NetSpider -Nettrack +Net Vampire +Netvibes NetZIP Nibbler NICErsPRO +Niki-bot NPbot -Nutch -oBot -Octopus -Offline Explorer Offline Navigator -OnCrawl -Openfind -Openvas -OpenVAS -OrangeBot -OutfoxBot +OrangeSpider +OutclicksBot +PageAnalyzer Page Analyzer -page scorer -Pandalytics Panscient +Papa Foto +Pavuk PeoplePal -PHPCrawl -Picscout -Pixray -PleaseCrawl -Pockey +Pimonster +Pi-Monster Probethenet -Psbot +ProWebWalker +Pump PxBroker -RankActive RankActiveLinkBot -RankFlex -RankurBot -RealDownload -RebelMouse -Recorder -ReGet +RankingBot +Ripper RocketCrawler -s1z.ru -SBIder -ScanAlert -ScreenerBot -Searchestate -Semrush +Rogerbot +Scanbot +ScoutJet +Screaming SemrushBot -SEOkicks-Robot Seomoz SEOprofiler seoscanners -SeoSiteCheckup serpstatbot -sexsearcher Shodan +Siphon +Sitebeam +SiteExplorer +Siteimprove SiteLockSpider SiteSnagger -SiteSucker SlySearch SmartDownload +Snapbot +Snoopy +SocialRankIOBot Sociscraper sogouspider -Sogou web spider Sottopop -Spanner -sp_auditbot Spbot spyfu +Sqlworm Sqworm +Steeler +Stripper +Sucker Sucuri +SuperBot +SurveyBot +Suzuran Swiftbot -sysscan -T0PHackTeam +Szukacz T8Abot +tAkeOut Teleport TeleportPro Telesoft -Telesphoreo -Titan -Toata -Tracemyfile -Trendiction +Telesphorep +The Intraformant +Thumbor +Toweyabot Trendictionbot True_Robot -Turingos Turnitin +TurnitinBot Twice Typhoeus -UnisterBot -URLy.Warning +Upflow +URLy Warning Vacuum Vagabondo VB Project -VCI VeriCiteCrawler -VidibleScraper Virusdie -VoidEYE -Voltron WASALive-Bot -Webalta -WebBandit -WebCollage -Web Collage -WebCopier -WebFetch -WebFuck -Web Fuck -WebPix -Web Sauger +WebAuto +WEBDAV +Web Enhancer +Web Fetch +WebImageCollector +WebmasterWorldForumBot +webmeup-crawler +Web Pix +WebReaper +WebSauger Webshag -Website Quester -WebZIP +WebStripper +WebWhacker +WeSEE +Whack +Whacker Whatweb Widow WiseGuys Robot WPScan WWW-Collector-E -WWW-Mechanize +WWW::Mechanize +x09Mozilla Xaldon_WebSpider -Xenu +xpymep1.exe +YoudaoBot +Zade +Zermelo +ZmEu +ZumBot diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index b70bea28b..c7645f851 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,100 +1,100 @@ -01apple.com -24-ak.ru -3dracergames.com -51unlim.ru -5i2.net -academia-nsk.org -ads-cool.pro -amateurgalls.com -anapa-inns.ru -angigreene.com -animalia-life.club -apibrowsesmartne-a.akamaihd.net -apiitsthirteende-a.akamaihd.net -apper.de -asia-forum.ru -balook.com -bestwebsitesawards.com -bezcmexa.ru -bidbuy.co.kr -blobar.org -candy-glam-hp.com -cleansearch.net -com-onlinesupport.site -cottageofgrace.com -dafatiri.com -digilander.libero.it -dirtpics.pw -doeco.ru -dogsrun.net -donvito.unas.cz -edwinkonijn.com.au -electricwheelchairsarea.com -electrik-avenue.com -exhibitionplus.eu -extlinks.com -fanrto.com -fatmaelgarny.com -fickkontakte.org -fotosfotos.eu -gay-site.store -gearcraft.us -getmyads24.com -gok-kasten.net -gratuitbaise.com -greatfind-a.akamaihd.net -hd720kino.ru -healbio.ru -hotenergy.ru -idealtits.net -id-forex.com -ipornox.xxx -istizanidineacontrolledsubstance.blogspot.com -keyhantercume.com -kimcurlrvsms.com -kountrylife.com -littleberry.ru -loftdigital.eu -maglid.ru -marmotstore.online -maslenka.kz -mazda-roadsters.com -meccadumps.net -medtherapy.ru -michaelkorssaleoutletonline.net -mokrayakiska.com -mygameplus.ru -myseoconsultant.com -nagdak.ru -nikitsyringedrivelg.pen.io -oynat.info -portside.cc -prefersurvey.net -remontbiz.ru -rostov.xrus.org -royalvegascasino.com -rukino.org -runofilms.ru -ruspdd.com -scanner-barak.top -seeresultshub-a.akamaihd.net -seoboxes.com -shopwme.ru -simon3.ru -stauga.altervista.org -telegraf.by -thelottosecrets.com -thruport.com -tupper-posuda.ru -webix.me -winwotgold.pl -word-vorlagen.xyz -xaijo.com -x-diesel.info -xn--80ab4aa2g.xn--p1ai -xportvusbdriver8i.snack.ws -xxxguitars.com -yurgorod.ru -zog.link -zoodrawings.com -zryydi.com +2000k.ru +2daytrendingnews.com +45en.ru +4webmasters.org +7zap.com +abwa.tk +adexprts.com +admitad.com +all4wap.ru +appfixing.space +arenanews.com.ua +arenda-avtoprokat-krasnodar.ru +asacopaco.tk +ayeartoforget.com +baoxaydung.com.vn +bolezniorganov.ru +brains2.biz +breastaugmentation.co.za +buypanicdisorderpill.com +cacheimages.com +citetick.com +classiquebijoux.ru +compliance-margo.top +cookielawblog.wordpress.com +daretodonate.co +djekxa.ru +domain-tracker.com +dood.live +ecookna.com.ua +eloxal.ru +extstat.com +fotoxxxru.com +free-traffic.xyz +freza-sverlo.ru +fullfileaccess.com +fun2cell.net +funnel.co.za +gabeshop.ru +gambarxkata.co +gembird.com +gigapeta.com +gwhwpxbw.bloger.index.hr +hermesreplica.win +homegardenlova.com +igadgetsworld.com +incest-ru.com +indetiske.ya.ru +internet-apteka.ru +iqoption.com +jabimgo.pw +javatex.co.id +jeremyeaton.co +jerseysbizwholesalecheap.com +job.icivil.ir +kerwinandcariza.com +kiwe-analytics.com +kongruan.com +konkursowo-24.pl +linkarena.com +live-sexcam.tk +muzaporn.com +my-aladin.com +nordstar.pro +onlinebay.ru +pateaswing.com +pathwhelp.org +picture-group.com +pinwallpaper.top +pio.polytopesexempt.com +pomoc-drogowa.cba.pl +postclass.com +proposal-engine.com +quelle.ru +rfd-split.hr +ryetaw.com +scanner-andrew.top +sell-fb-group-here.com +serialsway.ucoz.ru +sexsearch.com +s-forum.biz +sharebutton.to +skanninge.se +sport-video-obzor.ru +sps-shop.com +squidoo.com +theporndude.com +timecrimea.ru +titan-ads.life +tizanidinestreetprice.blogspot.com +tnctrx.com +topquality.cf +tubeline.biz +u-cheats.ru +viberdownload10.com +voltrknc1.com +x-diesel.org +yubikk.info +zaimhelp.ru +zmoda.hostreo.com +zootravel.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 287552e43..aa77f257e 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,1000 +1,1000 @@ +00author.com 08800.top -0c47f8422d3f.com -1001watch.com.ua -101raccoon.ru -123any.com -12bet.com -13tabs.com +0n-line.tv +1000n1.ru +101billion.com +101flag.ru +123movies.love 14b.info -1j7740kd.website +1688.com +178evakuator178.ru +1kinobig.ru +1stat.ru 2000k.ru -2020iscoming.info 256bit.by -2drittel.de -2kata.ru -2nt.ru -2rich4bitches.com -3weekdiet.com +2728fb936f0.com +2daytrendingnews.com 40cg.com -4kepics.com -4webmasters.com -57883.net +45en.ru +4kplayer.pl +4pp13.com +5-steps-to-start-business.com 5tey7463.weebly.com -5ws.dating-app.ru -6128786.com +66cpwgln.space 76brighton.co.uk -7xc4n.com -7zap.com -888.com -aab94f698f36684c5a852a2ef272e031bb.com -aac52006ec82a24e08b665f4db2b5013f7.com -academia-nsk.org +7fon.ru +8gold.com +999webdesign.com +9icmzvn6.website +a96527gi.beget.tech +aa8b68101d388c446389283820863176e7.com +aaa38852e886ac4af1a3cff9b47cab6272.com +abcdefh.xyz +abclauncher.com +absugars.com +abwa.tk acmebtn.ml -adloads.com -adobereader-free.ru -adpremium.org -adrenalin-stk.ru -adsloads.com -adssafeprotected.com -adtech.de -adtiger.tk -adtology.com -advancedsoftwaresupport.com -advokat-grodno.by -ad-words.ru -affordablewebsitesandmobileapps.com -afslankpillen2017nl.eu -agreda.pluto.ro +adamoads.com +adanih.com +add-add.men +adexprts.com +adinterax.com +admanaerofoil.com +adrunnr.com +ads-seo.men +advancedcleaner.com +advancedmassagebysara.com +adverster.com +aerogo.com +affiliate-fr.com +afmuseum.com +agapovdg.ru +ahrefs.com +akita.kz akuhni.by -albamargroup.com -alert-fdm.xyz +akusherok.ru +akvamaster.dp.ua +alfa9.com +algarveglobal.com +alibestsale.com all4invest.ru -all4wap.ru allbizne.co.ua -alldownload.pw -allesohnegirls.net -allforminecraft.ru -all-number.com +allknow.info +allmarketsnewdayli.gdn +all-streaming-media.com +alltheviews.com +aloofly.com +alpharma.net alta-realestate.com +alvtank.se alyeskaresort.com -amazingpic.net amehdaily.com +am-se.com +amung.us analytics-ads.xyz -anapa-inns.ru -androids-store.com -android-systems.ru -angkortours.vn -animal36.com -anonymous-redirect.com -apessay.com -apiappenableinfo-a.akamaihd.net -apibrowsesmartne-a.akamaihd.net +animaltoplist.com +anticrawler.org +anti-virus-removal.info +aosexkontakte.net +aosheng-tech.com +apartmentratings.com +apelsinnik.website apidiamondatanet-a.akamaihd.net apiitsthirteende-a.akamaihd.net -apioasisspacenet-a.akamaihd.net -apishops.ru -appartement-stumm.at -appiq.mobi -applicationg29.com +approvedlocal.co.za +apptester.tk ap.senai.br -apxeo.info -arate.ru -arewater.com -arius.tech +architecturebest.com +arcteryxstore.online +arenanews.com.ua +arenda-avtoprokat-krasnodar.ru +arenda-yeisk.ru +arendovalka.xyz arpelsreplica.xyz -arraty.altervista.org -aruplighting.com -asociatia-tipografilor-transilvania.ro -atelielembrancaqueencanta.com.br -ausmepa.org.au -autorn.ru -avitocars.ru -aviva-limoux.com -avtocredit-legko.ru +arquapetrarca.info +artdestshop.eu +asscenihotosidea.blogspot.com +ausergrubhof.info +aussie-prizes.com +autochoixspinelli.com +autolombard-krasnodar.ru +auto-zapchasti.org +avirasecureserver.com +avtointeres.ru awaybird.ru -ayanaspa.com +awency.com +axbocz.net ayeartoforget.com -babespcs.com -back.dog -backlink4u.net +azartmix.com +azazaporn.com +azbaseimages.net +azte.ch +babyguns.ru backuperwebcam.weebly.com +baikaleminer.com baixar-musicas-gratis.com -balois.worldbreak.com -bang-hotties.com -bankiem.pl -bannertesting.com -bashtime.ru +balkanfarma.ru +bankcrediti.ru +banki76.ru +banksrf.ru batanga.net -bdsmgalls.net +bcmp.org beachtoday.ru -beatifullandscape.co -beauby.ru +bear.gotcher.us +beauty-bracelet.com +bedandbreakfast.com beetpics.pw -benchmarkcommunications.co.uk -bensbargains.net -berrymall.ru -bestdooz.com -best-games.today -bestserials.com -bestssaker.com -best-way.men -bestwaystogetpaid.us +belinka.com.ua +belinvestdom.by +berdasovivan.ru +besthomemadepornsites.com +bestmobilityscooterstoday.com +best-seo-software.xyz +betonka.pro bet-prognoz.com +betterhdporn.com betteroffers.review -betune.onlinewebshop.net bezsporno.ru -bibys.com -bidbuy.co.kr -biglistofwebsites.com -bigpenisguide.com -bigpictures.club -bikini-image.com -bin-brokers.com -bioca.org -bio-japan.net -bio-market.kz -bio-optomarket.ru -biteg.xyz -bizcheapjerseyswholesalechina.com -biznesluxe.ru -bjetjt.com +bigcareer.info +bimatoprost-careprost.com +bitcoinpile.com +bitnote.co +bizlist.com.de +bjorkbacken.nu blackcurranthumidifiers.site -blacktwhite.com +blackle.com +blackwitchcraft.ru blagovest-med.ru -blog.f00kclan.de -blogsfunky672.weebly.com +blog.koorg.ru +bmwhighperformers.com bodybuilding-shop.biz -bonanza-fortune.men -bookmaker-bet.com -brainxs.us -brandehk.dk -brandov.ru +bonky.biz +boo-arts.com +bookmark4you.com +bookmark4you.com.biz +brains2.biz +brakehawk.com +brandbucket.com break-the-chains.com -browsepulse-a.akamaihd.net -browserprotecter.com -bryansk.zrus.org -budpost.com.ua -buketeg.xyz -bum.com.ru -burger-imperia.com -bus-offer.com +briomotor.co +brus.city +btvn.ru +bumskontakte.org +buron.pw buttons-for-website.com -buyessay3.blogspot.ru +buttons-for-your-website.com +buyclomidonlaine.com buy-loft.ru -buy-meds24.com -buypanicdisorderpill.com -buytizanidineonlinenoprescription.blogspot.com -buzzonclick.com -buzz-porno.info -buzzurl.jp -bvlgariring.xyz -bycontext.com -cactussoft.biz -call-of-duty.info -calstaterealty.us -camsex.xxx-cam.webcam -candlewooddentalcentre.com.au +buyparajumpers.online +buyscabiescream.com +bwlx.prepedu.cn +bytimedance.ru +camdolls.com +candelluxsklep.pl +candypeople.se candytiner.com -carina-sy.de +canvas.gloverid.site +cardsdumps.com +cardul.ru carmuffler.net -cartierlovebraceletreplica.xyz +cars-modification.net +carson.getenjoyment.net +cartechnic.ru +cartierbracelet.xyz +cartierloveringreplica.xyz cartierlove.xyz -cartierreplica.top cartujano-pre.de -casite-513049.cloudaccess.net -cbozhe.com -cctva.tv -cellfun.mobi -centraletermice.us -chcu.net -cheapestjerseysonwholesale.com -cheapmarmot.online +ccbill.com +cdnnetwok.xyz +ceotrk.com +cerev.info +cete.ru +cfjump.com +c-gainsbourg.com +cheapjerseysbizwholesale.us +cheapmoncler.xyz chee-by.biz -chiblackhawks-jerseys.com +chelnytruck.ru chinajerseyswholesalecoupons.com -chocolateslim-original.com -chocoslim.pro -christianlouboutinreplica.win -cialischmrx.com -cibpenonptib.flu.cc -citysecurity.nu -ci.ua -clash-clans.ru -classiquebijoux.ru -cleanersoft.com -clickaider.com -clickintext.net +chocolateslim-u-srbiji.com +choosecuisine.com +cialischsrx.com +cineacademy.ru +cinemaflix.website +clarithromycin500mg.com +cleanallvirus.com +cleanmypc.com clickprotects.com -clicksor.net -clodo.ru +clicksor.com +clips.ua.ac.be club-musics.ru -club-samodelkin.ru -cmd.kz -coinsspb.com +cobaltpro.ru +collegeessay19.blogspot.ru +com-cleaner.systems comissionka.net -compliance-brian.top -compliance-brian.xyz +compliance-barak.xyz compliance-fred.top -compliance-margo.top -conboy.us -confib.ifmo.ru -conocer-sanabria.com +compliance-john.top +compliance-josh.top +compliance-julia.top +compliance-olga.top +compucelunlock.net +concordexoticrentals.com continent-e.tv -cookie-law-enforcement-aa.xyz -cookie-law-enforcement-hh.xyz -coprofam.org -copro.pw +copypaste.traderzplanet.in +copyrightclaims.org +cornomase.win +counter.bloke.com country-chic.ru +cpabegins.ru cpajunkies.com -crackzplanet.com -cranly.net -creams.makeforum.eu -crest3d.ru -csgo4.win -cubs-jerseys.us -culmimg.pw -customcollegeessays.net -customerguru.in -cyse.tk -daisye.top -dangphoto.trade -datract.com -dawlenie.com -ddlmega.net -ddsoldes.fr -dealitright.click +creatives.adbetclickin.pink +creditmoney.com.ua +creditnation.ru +cryptoswap.biz +cs-passion.pl +customergrowthsystems.com +custom-product-labels.com +cvety24.by +danceuniverse.ru +dataloading.net +db.speedup.gdn +ddospanel.com dear-diary.ru -dentalpearls.com.au -descargar-musica-gratis.net -deshevo-nedorogo.ru +decoratinghomes.ga +delayreferat.ru +demian.kz designdevise.com -design-ideas.info -dfiles.me -dichvuvesinhhanoi.com -dikx.gdn -dimkino.ru -dinkolove.ya.ru -discountbarbour.online -divisioncore.com -divku.ru -dlya-android.org -dmmspy.com +destinationrealestate.com +dev78.dioniqlabb.se +dieswaene.com +diffbot.com +digest-project.ru +dignitasdata.se +diminishedvalueoforegon.com +discovertreasure-a.akamaihd.net +discovertreasurenow.com +dispo.de +diusyawiga.tk +divci-hry.info +djekxa.ru +djihispano.com +dnepropetrovsk.mistr-x.org documentbase.net -dogclothing.store -dojki-hd.com -domaineaneblanc.com +dogbreedspicture.net +domik-derevne.ru +dominateforex.ml domoysshop.ru -dorratex.tn -dostavka-v-ukrainu.ru -download-of-the-warez.blogspot.com -dustyorate.com +domznaniy.ru +downloadeer.net +downloadmefiranaratb1972.xpg.com.br +dpihatinh.gov.vn +drillsaw.ru +drivotracker.com +drugspowerstore.com +dverimegapolis.ru dyshagi.ru -earthmagic.info -e-biznes.info -e-buyeasy.com -ecommerce-seo.org -editors.choice6912650.hulfingtonpost.com -eduinfosite.com -eladkarako.github.io -eldiariodeguadalajara.com -elektrischezigarette2.onsugar.com -elektroniksigaraankara.info -elektrozigaretten2.yn.lt -elidelcream.weebly.com -enbersoft.com -englishdictionaryfree.com -eng-lyrics.com +easyshoppermac.com +ednorxmed.com +ed-shop01.ru +educatemetv.com +efnor-ac.com +e-kwiaciarz.pl +eladkarako.com +elektrozigarette2.webs.com +elektrozigarettekaufen2.blogse.nl +elektrozigaretten1.tumblr.com +eloconcream.blogspot.com +emailaccountlogin.co +englishgamer.com +enhand.se +en.home-task.com eorogo.top -erot.co -esfchat.tk +eralph.tk +eropho.com +erotik-kostenlos.net +estelight.ru etotupo.ru -evehemming.blogspot.com.au -evidencecleanergold.com -evogarage.com -exchangeit.gq -excitacion.info -exmasters.com -expertblog.info -expolicenciaslatam.co -expresstoplivo.ru +eu-cookie-law.blogspot.com +eu-cookie-law.info +euronis-free.com +everypony.ru +exoneration-project.us +exonline.info +expdom.com +extlinks.com extrabot.com -ezigarettekaufen.myblog.de +extractorandburner.com +extremepornos.net +ezigarettekaufen2.blox.pl f1nder.org -fable.in.ua -factorynightclub.com -falllow.gq -farmingworm.com -fatmaelgarny.com +fakehandbags.xyz +fashionavenuegame.com +fashion-stickers.ru favornews.com -feel-planet.com -feeriaclub.ru -femdom.twiclub.in -fenoyl.batcave.net -ferretsoft.com -fetishinside.com -fettefrauen.net -fickenprivat.info -firstsiteguide.com -fitness-video.net -fiuxy.com -flash4fun.com +femmesdenudees.com +fermersovet.ru +ff30236ddef1465f88547e760973d70a.com +fickenbumsen.net +fickkontaktehobbyhuren.com +fickkontakte.org +filesclub.net +filesdatabase.net +filmbokep69.com +filmci.pro +filmfanatic.com +findacheaplawyers.com +findsexguide.com +firesub.pl +fivedwld.cf +flashahead.info +floating-share-buttons.com +flooringinstallation-edmonton.com floridahuntingfishingadventures.com -florida-tourism.net -flowwwers.com -flytourisme.org -fm-upgrade.ru -forexunion.net -formulaantiuban.com -forodvd.com -forpostlock.ru -fotopop.club -fotoxxxru.com -foxjuegos.com +floridamhca.org +focalink.com +fok.nl +forex-indextop20.ru +forex-procto.ru +formseo.com +fortevidyoze.net +forum20.smailik.org +forums.toucharcade.com +fotzen-ficken.com foxtechfpv.com freakycheats.com -freegamesplay.online +freejabs.com freenode.info +freenom.link +free-social-buttons.xyz free-stock-illustration.com -free-today.com -freshdz.com +freeuploader.com +freeuploader.ml +freewebs.com +freewhatsappload.com front.ru -fuel-gas.com +front.to +fun2cell.net +funponsel.com futbolkisales.ru -fx-brokers-review.com -fyxabomiw.ru +fyl.com.ru +gallery.rennlist.com gamebackyard.com -gamedayhouse.com -gamevalue7.weebly.com -gamezblox.com +gamedayassist.com +game-mmorpg.net +gameprimary.com +games.kolossale.ru gateway.zscalertwo.net -gcup.ru -ge0ip.net -geckoandfly.com -geilehausfrauen.net +gaytube.com +gearsadspromo.club +geile-lelly.eu +gelezki.com +generalporn.org gepezz.info -germetiki.com.ua getaclueamerica.com getlaid-xxxhookupdirect.com -getoutofdebtfree.org getpopunder.com -getridofstretchmarks.org -gfaq.ru -ggiaro.com -ghernnqr.skyrock.com -girlporn.ru -girlsatgames.ru -gkvector.ru -glastecfilms.com.my -glogow.pl -glopages.ru -gne8.com -go2album.com -goldandcard.ru -golden-praga.ru -goodnightjournal.com -googlepositions.com -gourcy.altervista.org -gpirate.com -gq-catalog.gq -greenidesign.co +getresponse.com +girls-ufa.ru +gk170.ru +golden-catalog.pro +goodhousekeeping.com +goodhumor24.com +gorodservis.ru +gowreckdiving.com +greatdealshop.com gribokstop.com -gsbs.com.ua -gsou.cf -gstatey.net -gta-club.ru -guardlink.com -guiadeserraazul.com -guildebzh.info -guruofcasino.com -h2monline.com -handicapbathtubarea.com -handsandlegs.ru -hanwei.us +gsasearchenginerankerhelp.com +gsmlab.pl +guge.io +guitar-master.org +gz2.bbsoldes.fr +habermetre.com +hackers-crackers.tk +hao123.com +happy.new.yeartwit.com harmonyglen.us -hasshe.com -hatdc.org -headpharmacy.com -heatpower.ru -hebr.myddns-flir.com -heliko.no -hentaiheroes.com +havepussy.com +hdpixent.com +hdwallpapers-free.com +headpress.ru +heartofbeijing.blogspot.com +help.tpu.ru herehloadibs.cf -highstairs-a.akamaihd.net -hikesearch.net -histock.info -homeart.space -honyaku.yahoofs.jp -hostcritique.com -hosting-tracker.com -hotchatdate.com -hotelcrocenzi.sm -hotsocialz.com -hotxnights.info -houston-vikings.com -hoverboard360.at +hermesreplica.pw +hit-men.men +hjaoopoa.top +hkdiiohi.skyrock.com +hobbyhuren24.net +hobbyhuren-datenbank.com +holidaypics.org +homedecoguide.info +homemade.gq +hornymatches.com +hostnow.men +hotblognetwork.com +housediz.com +housemilan.ru +hoverboard360.es +howmuchdoestizanidinecost.blogspot.com hreade.com -hscsscotland.com -humanorightswatch.org -hydropump.su +huimin764128.com +hulfingtonpost.com +humbmosquina.tk +hunterboots.online +hut1.ru +hystersister.com +idc.com.ua ideashome.id -ie.57883.net -ikearugs.xyz -imadedinner.net -images-graphics-pics.com -images.gyffu.com -imfamous.info -imk.com.ua -immobiliaremassaro.com +ido3.com +ifirestarter.ru +igithab.com +iloveitaly.ru +imagerydatabase.com +imperia31.ru impisr.edunsk.ru -impresagaia.it inclk.com -incolors.club -indianmedicaltourismshop.com infektsii.com -inmate-locator.us -insider.pro -inspiring-desperate.tk -intelhdgraphicsgtdrive6w.metroblog.com -intellego.info -interferencer.ru -intimshop-fantasy.ru -intim-uslugi.info -invitefashion.com -ipornox.xxx -ipostroika.ru -iqoption.com -iron-age.info -isotoner.com -istizanidineanopiate.blogspot.com -istizanidinelikexanax.blogspot.com -itrevolution.cf -j33x.com -japfm.com -javspace.net -javstock.com +insurple.com +inzn.ru +iplusbit.blogspot.co.za +irkutsk.online-podarki.com +irunfar.com +istanbulit.com +istmira.ru +istock-mebel.ru +itis4you.com +iusstf.org +ix20.ru +jacago.com +jasonpartington.com +javrip.net jav-way.site +jbl-charge.info +jennyfire.ru jerseyschinabizwholesale.us -jerseyssportsshop.com -jillepille.com -jmat.cn -joessmogtestonly.com -jolic2.com -jongose.ninja -justdating.online -justkillingti.me -kamorel.com +jerseyswholesalechinalimited.com +jerseywholesaleelitestore.com +jumptap.com +kaidalibor.de +kakadu-interior.com.ua +kalandranis.gr kashubadesign.ru -katjimej.blog.fc2.com -keenoutlet.online -kerch.site -kevblog.top -keyword-suggestions.com -keywordteam.net -kfon.eu -kinocccp.net -kino-fun.ru -kinogonew.ru -kinohall.ru -kinomaniatv.pl -kiprinform.com -kndxbkdx.bloger.index.hr -knogg.net -knowsitall.info -kochanelli.com +katushka.net +kaz.kz +kinoduh.ru +kino-key.info +klosetkitten.com +kollekcioner.ru kolotiloff.ru +komp-pomosch.ru komputers-best.ru -kongoultry.net konica.kz -kopihijau.info -kupiproday.com.ua +krasnodar-avtolombards.ru +krassh.ru +kreativperlen.ch +kredit-pod-zalog-krasnodar.ru +kunstaktien.info +kutikomi.net +kuzinsp.ru +kvartir-remont.biz kw21.org -laapp.com -labplus.ru -lapitec.eu +kwzf.net +lafriore.ru +lakomka.com.ua +lancheck.net +landmania.ru laptoper.net -larutti.ru -laserpen.club +larchik.net +lasvegaslockandsafe.com lawyersinfo.org +laxdrills.com +lazymanyoga.com +ldrtrack.com leadn.pl -leadwayau.com lechtaczka.net -legalrc.biz -legionalpha.com -libertybilisim.com -likrot.com -lineshops.biz -linkdebrideur.xyz -lion.conboy.us -lion.misba.us -liveinternet.ro -lizace.com -lkbennettstore.online -lookingglassemb.com -losangeles-ads.com -louisvuittonoutletstore.net -lsex.xyz +lennatin.info +lesbian.xyz +letmacworkfaster.world +letmacwork.world +letsart.ru +letslowbefast.site +lida-ru.com +light.ifmo.ru +lightinghomes.net +limads.men +link.ac +liumimgx.pw +livetsomudvekslingstudent.bloggersdelight.dk +lmrauction.com +localflirtbuddies.com +locationdesaison.com +locksmith.jp +lollypopgaming.com +lol-smurfs.com +longadventure.com +louboutinshoes.xyz +lovi-moment.com.ua +lsp-awak-perikanan.com luciddiagnostics.in -luckyday.world +luxmagazine.cf +m1media.net m4ever.net -mackeeper-center.club -maglid.ru -mailemedicinals.com -makemoneyonline.com -malls.com -mamasuper.prom.ua -manimpotence.com -manve.info -mapquestz.us -mararoom.ru -marcoislandvacations.net -marcowebonyodziezowe.pl -marktforschung-stuttgart.com -marsgatan.com -matsdale.com -maximilitary.ru -mazda-roadsters.com -mcnamaratech.com -mediaoffers.click -medkletki.ru -meendo-free-traffic.ga -meetingrainstorm.bid -member-quiz.com +m5home.ru +mabdoola.blogspot.com +machicon-akihabara.info +macnewtech.com +mac-shield.com +magicdiet.gq +makedo.ru +mansparskats.com +marmotstore.online +martlinker.com +massage-info.nl +matb3aa.com +max-p.men +m.b00kmarks.com +mcar.in.ua +mebelcomplekt.ru +medicines-choice.com +meds-online24.com +mega-bony2017.pl +mega-polis.biz.ua +menetie.ru message-warning.net -metaxalonevstizanidine.blogspot.com -mexicotravelnet.com -micasainvest.com +mex-annushka.ru +mezaruk.info microstatic.pl -minet.club -miniature.io -mir-limuzinov.ru -mirobuvi.com.ua +mir-business-24.ru missclub.info +missvietnam.org +mixed-wrestling.ru +mixtapetorrent.com mmostrike.ru -mmstat.com -mobilierland.com -moivestiy.biz -moncleroutletonline.pw -monetizationking.net -moroccosurfadventures.com -mosdverka.ru -mp3downloadhq.com -mp3films.ru -mp3ringtone.info -mriyadh.com -mrvideospornogratis.xxx -muizre.ru -musezone.ru -musezone.su -muycerdas.xxx -my-bc.ru +mobot.site +modnie-futbolki.net +mole.pluto.ro +money-for-placing-articles.com +mosrif.ru +mototsikl.org +moviemail-online.co.uk +mowser.com +mrmoneymustache.com +musicstock.me +musicvidz.ru +musirc.com +mustwineblog.com +muznachas-service.ru +mycaf.it mydearest.co -myfreecams.com -myfreemp3.eu -myhitmp3.club -myperiod.club -myshopmatemac.com -nakozhe.com -narkologiya-voronezh.ru -narutonaruto.ru -nash-krym.info -negociosdasha.com -net-profits.xyz -new-post.tk -news-speaker.com -nextlnk12.com -nfljerseysforsalewholesaler.com -ngps1.ru +my-floor.in.ua +myhydros.org +myindospace.com +mylida.org +myliveblog.ru +mypornfree.ru +nastroyke.net +nastydollars.com +naval.jislaaik.com +netallergy.ru +newhairstylesformen2014.com +nextrent-crimea.ru +nfljerseys.online +nfvsz.com +nikitabuch.com nikitsyringedrivelg.pen.io -nootrino.com -normalegal.ru -notasprensa.info +ningessaybe.me +njpalletremoval.com +nlfjjunb5.ru +noclegonline.info novatech.vn -ntic.fr -nzfilecloud.weebly.com -o333o.com -oakridgemo.com -oda.as -odesproperty.com -offf.info -okeinfo.online -old-rock.com -olgacvetmet.com -olympescort.com -oneminutesite.it -oneshotdate.com -online-hd.pl +nowtorrents.com +nudepatch.net +nuit-artisanale.com +nuker.com +obnal.org +ochistka-stokov.ru +offer.camp +offers.bycontext.com +offtime.ru +ohmyrings.com +okmedia.sk +olvanto.ru +onclkads.com +onemactrckr.com +onlinadverts.com +onlinefilmz.net +onlineporno.site online-sbank.ru -onlinetvseries.me +onload.pw +onlywoman.org ons-add.men -onthemarch.co -ooo-olni.ru -openmediasoft.com -oralsexfilme.net -oranga.host.sk +optitrade24.com orenburg-gsm.ru -osagonline.ru -ossmalta.com -otbelivanie-zubov.com -oxford-book.com.ua -paphoselectricianandplumber.com -parajumpersoutlet.online -partner-cdn.men -pavlodar.xkaz.org -pdns.download -pensplan.com -php-market.ru +orgasmus-virtual.com +orion-v.com +outshop.ru +oz-offers.com +paceform.com +panicatack.com +paparazzistudios.com.au +partnerads.men +partners-ship.pro +pathwhelp.org +pcads.ru +pdn-4.com +pearlisland.ru +pekori.to +pensplan4u.com +pepperstyle.ru +percin.biz.ly +petitions.whitehouse.gov +photokitchendesign.com physfunc.ru +picsfair.com pictures-and-images.net -pills24h.com +picturesify.com +pihl.se +pila.pl +pinsdaddy.com pinwallpaper.xyz -pipki.r.acdnpro.com -pisanieprac.info -piski.top -pix-hd.com -platezhka.net -pl-top.pl -pogodnyyeavarii.gq -poiskzakona.ru -pojdelo.weebly.com -pokerniydom.ru -polytopesexempt.com -popmarker.com -porn555.com -pornhive.org -pornokajf.com -pornolook.net +pio.polytopesexempt.com +plastjulgranar.se +play-mp3.com +pochtovyi-index.ru +pokemongooo.ml +pomoc-drogowa.cba.pl +pooleroadmedicalcentre.co.uk +popander.mobi +popcash.net +popup-fdm.xyz +popup.matchmaker.com +porngalleries.top +pornhub-forum.ga +porno-chaman.info +pornoklad.ru +porno-play.net pornoreino.com -pornoslive.net -portal-eu.ru -powc.r.ca.d.sendibm2.com -preparevideosafesystem4unow.space -pricheskaonline.ru +pornosexrolik.com +porno-transsexuals.ru +pornowarp.info +portside.xyz +posible.net +postclass.com +potolokelekor.ru +pozdravleniya-c.ru +pravoholding.ru priora-2.com -privat-girl.net -producm.ru -prodvigator.ua -professionaldieselcare.com +privatefx.all4invest.info +produkto.net profitkode.com -progonrumarket.ru +profitsport.club +proftests.net +progress-upakovka.ru +prointer.net.ua prokotov.com -prospekt-st.ru -prostitutki-tolyatti.org -pukaporn.com -pulse33.ru -pussyfleet.com -putitin.me -puzo2arbuza.ru -pyramidlitho.webs.com -qcstrtvt.bloger.index.hr -qitt.ru -qor360.com -quickcashlimited.com -raavidesigns.com -rabotaetvse.ru -radiogambling.com +promalp-universal.ru +promoforum.ru +pro-poly.ru +prostitutki-rostova.org +pr-ten.de +psoriasis-file.trade +pulseonclick.com +puzzleweb.ru +q-moto.ru +qualitymarketzone.com +questionmarque.ch +quickbuck.com +quickloanbank.com +radiodigital.co ranking2017.ga -rapidsites.pro -raschtextil.com.ua -rebrand.ly -rebuildermedical.com -redhotfreebies.co.uk -redirlock.com -referencemoi.com +rapevideosmovies.com +rasteniya-vs-zombi.ru +razleton.com +razyboard.com +realresultslist.com +registrationdomainsite.com +registry-clean-up.net +registrydomainservices.com reimageplus.com -relax.ru -remont-mobile-phones.ru +remontvsamare.su remorcicomerciale.ro -remybutler.fr -retailwith.com -reversing.cc -ringtonepartner.com +rentaremotecomputer.com +rep-am.com +replicaclub.ru +resellerclub.com +retreatia.com +rewardit.com +reward-survey.net +rfid-locker.co +richinvestmonitor.com +ring4rhino.com robertefuller.com -roleforum.ru -royalcar-ufa.ru -royal-investments.net -ru-dety.ru +roll123.com +romhacking.ru +rossanasaavedra.net +rossmark.ru +rp9.ru +rumamba.com +running-line.ru russian-postindex.ru -russia-tao.ru -russia-today-video.ru -russintv.fr -rvtv.ru -sabaapress.com -saddiechoua.com -salmonfishingsacramentoriver.com -salutmontreal.com -samolet.fr -sampleletters.net -sbornik-zakonov.ru +rus-teh.narod.ru +rvi.biz +s8-nowy-wygraj.comli.com +samara.rosfirm.ru +samoiedo.it +sanjosestartups.com +santechnik.jimdo.com +sarafangel.ru +satellite.maps.ilovevitaly.com +saugatuck.com +saveindex.xyz sbtdesign.co.uk -sbwealthsolutions.ca -scanner-alexa.top -scanner-andrew.top -scanner-jessica.top -scanner-julia.top +scanmarine.info +scanner-brian.top +scanner-julianna.top +scanner-viktor.top score-ads.men -sdelai-prosto.ru -seadragonherbery.com -seansonline24.pl -search.alot.com -searchengineranker.email -search-error.com -searchtooknow-a.akamaihd.net -search.xtconnect.com +scottbywater.com +sc-specialhost.com +searchaddis.com seasaltwithfood.com -secret.xn--oogle-wmc.com -security60-e.com -seeingmeerkat.com -seemoreresultshu-a.akamaihd.net segol.tv -sei80.com -seinterface.com +sell-fb-group-here.com semalt.com -seminarygeorgia59.ga -seoboxes.com -seoexperimenty.ru -seofied.com -seo-prof1.xyz +seocheckupx.com +seonetwizard.com seorankinglinks.xyz -servicecenter.co.ua -serviporno.com -sex-dating.co -sexfreepornoxxx.com -sexgalleries.top -sexkontakteao.info -sex-sex-sex5.com -sextracker.be -sextracker.com -s-forum.biz +serdcenebolit.com +service-core.ru +seryeznie-znakomstva.ru +sevendays.com.ua +sex.hotblog.top +sexiporno.net +sexphoto.site +sexpornotales.com +sexs-foto.com +sextracker.de +sexuria.net +sexyali.com +sexy-pings.com shemale-sex.net -shiksabd.com -shodanhq.com -shopwme.ru -shtora66.ru -shymkent.xkaz.org -sibtest.ru -silvercash.com -similardeals.net -simple-share-buttons.com -simpoed.ufop.br -sispe.com.br -siteonomy.com -s-iwantyou.com -sixcooler.de -skanninge.se -sky-mine.ru +shisha-swag.de +shoesonlinebuy.cn +shoesonlinebuy.xyz +shtaketniki.kz +shtaketniki.ru +sibvitr.ru +sideeffectsoftizanidine.blogspot.com +signal03.ru +siha.de +simple-image.com.ua +sinel.info +sirpornogratis.xxx +sizeplus.work +sk.golden-praga.ru +skincrate.net slavic-magic.ru -slomm.ru -slonechka.ru -small-game.com -smartadserver.com +slayerlife.com +slowmacfaster.trade +sluganarodu.ru smart-scripts.com smokewithrabbits.com -snip.to -snsdeainavi.info -soaksoak.ru -sobecjvuwa.com.ru -social-fun.ru +sneakyboy.com +snip.tw +snomer1.ru +soc-econom-problems.ru +socialbuttons.xyz socialmadesimple.com -social-s-iii.xyz -softomix.com -softomix.net -soft-program.com -solmarket.by -soserfis.com -sovetogorod.ru +socialseet.ru +socialsignals24.com +soc-proof.su +sofit-dmd.ru +softxaker.ru +soheavyblog.com +solution4u.com +sophang8.com +souvenir.cc soviet-portal.do.am -space4update.pw -spammen.de +sovinsteel.ru spasswelt.xyz -speedup-my.site -spidtest.space -spin2016.cf -sport7777.net -srdrvp.com -sr-rekneskap.no -stairliftsarea.com +spb.ru +spidtest.org +spravka-medosmotr.ru +spywarespy.com +srecorder.com +stackthatbucks.com +staff.prairiesouth.ca standardchartered-forex.com stariy-baku.com -statistici.ro -stefanbakosab.se -stjamesschool.info -stockspmb.info -strfls.com -strongholdsb.ru -stroy-portal22.ru -success-seo.com -supers.com.ua -support.nopeas.sk -surintech.ac.th -svolze.com -syvertsen-da.no +startufa.ru +statustroll.com +steamoff.net +stoki.ru +stpicks.com +styro.ru +supervesti.ru +surcentro.com +symbaloo.com +symphonyintegratedhealthcare.com +syndicate.fun +taaaak.com taboola.com -takeprofitsystem.com -tamada69.com +tacbelarus.ru +tagil.zrus.org +taqplayer.info tattomedia.com td-33.ru -tdsing.ru -teastory.co -techart24.com +tds-advert005.info tecspb.ru -teenbbw.yopoint.in -telefonsexsofort.tk +telefonsex-ohne0900.net +telegraf.by +telegramdownload10.com +telsis.com templates.radiodigital.co -texbaza.by -thebestweightlosspills.ovh -thebitcoincode.com -thefarmergame.com -thegalerie.eu -thegameriders.com -theguardlan.com +terrafootwear.us +teslathemes.com +thegioixekhach.com +theheroes.ru thejournal.ru -the-torrent-tracker.blogspot.com -thewebsitetemplate.info -thewomenlife.com -threecolumnblogger.com -tizanidine4mgtablets.blogspot.com +thenews-today.info +thepantonpractice.co.uk +thetoiletpaper.com +the-trader.net +thin.me.pn +timer4web.com +timetorelax.biz +timhost.ru +titan-ads.life +tizanidine4mgprice.blogspot.com +tizanidineandcipro.blogspot.com tizanidineandgabapentin.blogspot.com -tizanidinedruginteractions.blogspot.com -tizanidinedrugtest.blogspot.com -tizanidinehydrochloride4mgstreetvalue.blogspot.com -tizanidinerecreationaluse.blogspot.com -tizanidinesideeffectsweightloss.blogspot.com -tizanidinevsvalium.blogspot.com -tizanidinewithdrawal.blogspot.com -tnaionline.org -todohr.com -tomatis.gospartner.com +tizanidinecapsules.blogspot.com +tizanidinehcl4mgisitanarcotic.blogspot.com +tizanidineonline.blogspot.com +tizanidinestreetvalue.blogspot.com +tmm-kurs.ru tonerbox.kz -toolsky.com +toon-families.com toonfamilies.net -topads.men -topappspro.com topbestgames.com -topdownloads.ru -top-l2.com -topquality.cf -torontoplumbinggroup.com +topshef.ru +topsy.com torrent-newgames.com -torrents-tracker.com -totu.us -tracklead.net -traderzplanet.in -trafaret74.ru -trafficmonetize.org -trahic.ru -trainoffend.ml -truebeauty.cc -ttrraacckkrr.com -tubo360.com -tupper-posuda.ru -turbabitload.weebly.com -twittruth.com -typimga.pw +torrent-to-magnet.com +torture.ml +tourcroatia.co.uk +toursmaps.com +tpu.ru +track112.site +trkur.com +truck-addzilla.life +truck-land.life +trucri.me +trumpetedextremes.com +tsyndicate.com +tubeline.biz +tuckermktg.com +tuckpointingmasonrysystems.com +turizmus.us +turvgori.ru +twiclub.in +twojebook.pl tzritel.tk -ua.tc -ucoz.ru -uggsale.online -ultimatesetnewfreeallsoftupgradesystems.pw -undergroundcityphoto.com -unrealcommander.org -uprour.com -uptimechecker.com -urccvfmc.bloger.index.hr -urengoy.pro +ua-company.ru +uasb.ru +ufa.xrus.org +uhdtv.website +uhodzalijami.ru +umnovocaminho.com +upstore.me +uptime-gamma.net +urlcut.ru +uroki.net +usacasino.com usadacha.net -us-america.ru -usdx.us -v24s.net -v720hd.ru -vacuumscleaner.com -vadimkravtcov.ru -valkiria-tk.ru -vashsvet.com -vektorpress.ru +vapomnoncri.tk +varikoz24.com veles.shop -vellings.info -veselokloun.ru -via-energy-acquistare.com -viagroid.ru -videochat.bz -videochat.life -video-production.com -vikistars.com -vilingstore.net +velobikestock.com +vesnatehno.com +vezuviy.su +viagra-soft.ru +videochat.cafe +video-woman.com villacoloniale.com -vinylvault.co.uk -vitanail.ru -viven.host.sk -vizag.kharkov.ua -vkak.ru -vmnmvzsmn.over-blog.com -vodaodessa.com +visa-pasport.ru +visitcambridge.org +vladimir.xrus.org +volgograd.xrus.org +voloo.ru +volume-pills.biz +voluumtrk.com +vozbujdenie.com +vpnmouse.com +vrnelectro.ru +vseigru.one +vucms.com vykupavto-krasnodar.ru -vzubah.com -wallpapers-best.com -wanker.us +vzlom-na-zakaz.com +w7s.ru +wallpaperstock.net +wareseeker.com warning.or.kr +watch-movies.ru watchmyfb.pl -web-betting.ru -weboptimizes.com -webradiology.ru -website-datenbank.de +waterefficiency.co +wcb.su +webalan.ru +webenlace.com.ar +webextract.profound.net websiteexplorer.info -website-speed-check.site -website-stealer.nufaq.com -weclipart.com -wesharepics.info -whatsupinfoley.com -whereiskentoday.com -wholesalejerseys-cheapest.com -wholesalejerseyscheapjerseys.us.com -wildnatureimages.com -wineration.com -wingsoffury2.com -wladimirpayen.com -womensterritory.ru -wosik-dach.service-for-web.de -woweb.com.ua -wufak.com -www.event-tracking.com -www.kabbalah-red-bracelets.com +website-speed-checker.site +websitevaluebot.com +wedding-salon.net +weekes.biz.tc +wejdz-tu.pl +welck.octopis.com +wellcome2slovenia.ru +wesharepics.com +westen-z.life +wfb.hatedriveapart.com +whiteelephantwellington.com +wholesalejerseychinaoutlet.com +whosonmyserver.com +wieseversa.no +wineitudes.wordpress.com +winterclassichockeyjerseys.com +wnhjavlhezp.gq +wonderfulflowers.biz +worldoffiles.ru +wowas31.ucoz.ru +writersgroup580.web.fc2.com +www.get-free-traffic-now.com wygraj-skiny.win -x5market.ru -xaijo.com -xcc24.pl -x-diesel.com -xfluro.com -xlovecam.com -x-mix.info -xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai -xn------7cdbapdecfd4ak1bn0amjffj7afu3y.xn--p1ai -xn-----7kcabaipgeakzcss7bjdqdwpfnhv.xn--p1ai -xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai -xn----7sbaphztdjeboffeiof6c.xn--p1ai +xblognetwork.com +xchangetrak.com +x-diesel.org +xedserver.com +xingzi-vision.com +xjrul.com +xkaz.org +xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai +xn----7sbbahaq9bb5afgiqfliv4m.xn--p1ai xn--80aaaks3bbhabgbigamdr2h.xn--p1ai -xn--80ahvj9e.xn--p1ai +xn--80ab4aa2g.xn--p1ai +xn--80adaggc5bdhlfamsfdij4p7b.xn--p1ai xn--80ajbshivpvn2i.xn--p1ai -xn--b1agm2d.net -xn----dtbndd4ae7eub.top -xnxx-n.com -xsion.net -x-true.info -xvideospornoru.com -xxx-cam.webcam -xxxguitars.com -yadro.ru -yogamatsexpert.com -yottos.com -yougotanewdomain.com -youjizz.vc -yourads.website -youresponsive.com -yourporngay.com -yourserverisdown.com -youtoner.it -youtube-downloader.savetubevideo.com -youtubedownload.org -yuarra.pluto.ro -yugk.net +xn--80aodinpgi.xn--p1ai +xn--e1aggki3c.xn--80adxhks +xn--h1ahbi.com.ua +xn--lifehacer-1rb.com +xxx-treker.ru +xxxtube69.com +yaminecraft.ru +yaoguangdj.com +yatrk.xyz +yhit.press +youporn-forum.ga +youradexchange.com +yoursearch.me yugo-star.ru -yun56.co -zaidia.xhost.ro -zarabiaj-dzis.pl -zarepta.com +zacreditom.ru +zahvat.ru +zajm-pod-zalog-nedvizhimosti.ru +zambini.ru +zastenchivosti.net zazagames.org -zdesformula.ru -zed21.net -zeroredirect10.com -zigarettenonlinekaufen1.bloog.pl +zeikopay.com +zerocash.msk.ru +zeroredirect8.com zigarettenonlinekaufen2.bloog.pl -znakom.sibtest.ru -zog.link -zona-aqua.ru -zone-kev717.info -zoogdiesney.com -zoomovies.org -zoompegs.com -zrus.org -zscalertwo.net -zzbroya.com.ua -zzlgxh.com +zigzog.ru +znakomstva-moskva77.ru +znaturaloriginal.com +zolotoy-lis.ru +zoodrawings.com +zooggames.com +zoominfo.com +zoosexart.com +zootravel.com diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index 19753826b..4f75a1504 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -03p.info -100dollars-seo.com -1pamm.ru -23kw.ru -999webdesign.com -abcdefh.xyz -adconscious.com -adman.se -adrunnr.com -adultfullhd.com -adverster.com -adviceforum.info -ahrefs.com -alfa-medosmotr.ru -allmarketsnewdayli.gdn -all-streaming-media.com -alphaforum.ru -alpharma.net -animalia-life.club -apple.com-webbrowsing-security.science -arius.tech -asophoto.com -atley.eu.pn -aussie-prizes.com -baksman.com -bankofthewext.com -bestcalovebracelet.cn -bg6s0.com -bird1.ru -blog.koreadaily.com -bpro1.top -braintobe.win -brandedleadgeneration.com -browserprotecter.com -cctva.tv -cdncash.org -christianlouboutinshoes.xyz -cienum.fr -classiquebijoux.ru -clippingphotoindia.com -compliance-mary.top -cupday.com -cvety24.by -dailyrank.net -dairyindia.in -dalavia.ru -dataloading.net -dating-app.ru -davebestdeals.com +aa8b68101d388c446389283820863176e7.com +abwa.tk +adbetclickin.pink +adclickservice.com +adprotect.net +advertex.info +alert-fjg.xyz +alert.scansafe.net +alfa9.com +alibestsale.com +all4invest.info +aludecor.info +amehdaily.com +amospalla.es +an-donut.com +animali.deagostinipassion.it +apiholdingmypage-a.akamaihd.net +apimountainbikei-a.akamaihd.net +apioasisspacenet-a.akamaihd.net +appiq.mobi +applyneedy.xyz +ap.senai.br +architecturebest.com +arvut.org +asacopaco.tk +asiengirls.net +asrvrep-a.akamaihd.net +aviav.ru.com +avkzarabotok.info +avramstroy.ru +avto-oligarh.ru +back.dog +backuperwebcam.weebly.com +bank.uz +becuo.com +beetpics.pw +best-businessman.ru +bildsuche.ru +bitcoins-live.ru +bonkers.name +brendbutik.ru +budilneg.xyz +buyparajumpers.online +cactussoft.biz +cardul.ru +cdn.walkme.com +cialischmrx.com +citysecurity.nu +clarithromycin500mg.com +classicakuhni.ru +clickbanksites.info +cornerstone-countertops.com +cornomase.win +cracksplanet.com dayibiao.com -ddospanel.com -dengi-pod-zalog-nedvizhimosti.ru -detsky-nabytek.info -dev78.dioniqlabb.se -dinosaurus.site -dividendo.ru -domznaniy.ru -donna7753191.ru -downloaddy.net -downtuptv.gq -druzhininevgeniy63.blogspot.com -ekaterinburg.xrus.org -elektronischezi.livejournal.com -elvel.com.ua -ero-advertising.com -essay-edu.biz -estibot.com -eu-cookie-law.info -exonline.info -expertnaya-ocenka.ru -fashionavenuegame.com -feel-planet.com -fickenprivat.info -filmbokep69.com -filmidivx.com -filmi-v.online -flirt4free.com +dealitright.click +de.zapmeta.com +djstools.com +downloadeer.net +edge.sharethis.com +ednorxmed.com +efnor-ac.com +eshop4u.jp +extads.net +ezigarettenshop2.myblog.de +femmesdenudees.com +financeloan.us +fitfloponline.store +florida-tourism.net fm-upgrade.ru -fok.nl -formularz-konkurs.tk -forodvd.com -fotopop.club -foxtechfpv.com -freenom.link -free-social-buttons.com -free-stock-illustration.com -fun-mobi.pl -funnel.co.za -g33.org -galaxyflowers.ru -galleryawesome.com -game300.ru -genericviagrasildenafiled.net -gentamicineyedrops.blogspot.com -global-ics.co.za -go2album.com -gojiberriess.apishops.ru -goldenggames.com -google-liar.ru -googlemare.com -guildebzh.info -hanink.biz.ly -hasfun.com -havepussy.com -hmywwogw.bloger.index.hr -homemature.net -hotgirlhdwallpaper.com +frighteningremain.cf +front.to +gallerily.com +gamevalue7.weebly.com +gayxperience.com +ggiaro.com +go2mike.ru +gogalleryawesome.com +gogps.me +goodwinmetals.co +gopixdatabase.com +goroda-vsego-mira.ru +gov.yanao.ru +graphics8.info +gta-club.ru +hamptonoaks.ca +handicapvantoday.com +hd-filmy.net +helvetia.com.ua +hidemyass.com +hobbyhuren-datenbank.com hotshoppymac.com -hoverboard360.nl -htmlcorner.com -hunthillfarmtrust.org -ideawheel.com -inmate-locator.us +howmuchdoestizanidinecost.blogspot.com +howopen.ru +hscsscotland.com +img.wallpaperstock.net inome.com.ua -instabid.tech -intellekt21.ru -iqupdatetmz.win -ivearchenceinflu.cf -jackpotchances.com -javatex.co.id -kabbalah-red-bracelets.com +internet-apteka.ru +internetartfair.com +itch.io +javitas.info +jongose.ninja +jpcycles.com +jyvopys.com +kakadu-interior.com.ua +kalb.ru kamorel.com -kazan.xrus.org -kino-ecran.ru -kmgamex.cf -krasnodar.ru -kriokomora.info -kumuk.info -laapp.com -lalalove.ru -landreferat.ru -livefixer.com -long-beach-air-conditioning.com -losangeles-ads.com +kerwinandcariza.com +keywordblocks.com +khafre.us +kinobaks.com +ks1234.com +kvartira-sutochno.com +kvartir-remont.biz +kvartiry-remont.ucoz.ru +labelwater.se +lacave.ntic.fr +lacloop.info +lanadelreyfans.us +levaquin750mg.blogspot.com +lexaprogeneric.link +lion.playtap.us +liupis.com +localflirtbuddies.com +lombia.com low-format.ru -mackeeper-center.club -makemoneyonline.com -matchpal-a.akamaihd.net -maxthon.com -monclerboots.xyz -moonci.ru -mountainstream.ms -mrsdalloways.com -muycerdas.xxx -mydeathspace.com -mypornfree.ru -nalogovyy-kodeks.ru -next-dentists.tk -noscrapleftbehind.co -nucia.biz.ly -oballergiya.ru +ltvperf.com +lyngdalhudterapi.no +m292.info +mafcards.ru +mailemedicinals.com +marketland.ml +mere.host.sk +mezaruk.info +mokrayakiska.com +moneyteam24.com +muznachas-service.ru +negral.pluto.ro +nfljerseyscheapchinabiz.com +nhl09.ru +notasprensa.info +oecnhs.info onefilms.net -openmediasoft.com -ostroike.org -ozoz.it -paceform.com -pc4download.co -pdamods.ru +paintingplanet.ru +parfusale.se pearlisland.ru -perfection-pleasure.ru -photorepair.ru -piccshare.com -pinsdaddy.com -plusnetwork.com -pornokajf.com -pornoxxx.com.mx -privatefx-in.ru -prizesbook.online -prohoster.info -prosmibank.ru -prostitutki-novgoroda.org -prostitutki-tyumeni.org -proxyelite.biz -pulseonclick.com -purplesphere.in -quebec-bin.com -realting-moscow.ru +picurams.pw +pills24h.com +pirateday.ru +pizda.lol +pk-services.ru +pornoblood.com +pornoelita.info +porno-play.net +porno-video-chati.ru +pozdrawleniya.com +prizeestates.cricket +pr-ten.de +ptr.ruvds.com +quickloanbank.com +randki-sex.com +realtytimes.com redhotfreebies.co.uk -responsive-test.net -roleforum.ru -rus-pornuha.com -saletool.ru -scanner-barak.top -se.bnt-team.com -seksvideoonlain.com -sendearnings.com -service.adtech.us -serviporno.com -sexcamamateurchat.com -sexgalleries.top -sexy.babes.frontend-stack.top -shoesonlinebuy.xyz -shoppingjequiti.com.br -sisi-go.ru -sitevaluation.org -skanninge.se -socialmonkee.com -social-s-hhh.xyz +reelheroes.net +regionshop.biz +reklama-i-rabota.ru +restaurantlescampi.com +rossanasaavedra.net +ruscopybook.com +sammyweaver.com +sawin.beth.webd.pl +sbetodiodnye-lampy.ru +scanner-elena.top +scanner-george.top +scanner-jack.top +scanner-jessica.top +scanner-nelson.top +search-goo.com +searchinterneat-a.akamaihd.net +searchtooknow-a.akamaihd.net +seo-prof1.xyz +servicecenter.co.ua +sex-foto.pw +sexiporno.net +sexspornotub.com +sharebutton.org +shemale-sex.net +shoppingmiracles.co.uk +shopsellcardsdumps.com +sideeffectsoftizanidine.blogspot.com +simul.co +site3.free-share-buttons.com +sitiz.club +slavia.info +sluganarodu.ru +smart-balancewheel.com +snjatie-geroinovoy-lomki.ru +socialmadesimple.com +socialsignals24.com softwaretrend.net -soserfis.com -spasswelt.net -stackthatbucks.com +spammen.de +speedup-my.site +spywarespy.com +statoutlook.info +store-rx.com stromerrealty.com -sudexpert66.ru -supernew.org -svetodiodoff.ru -taboola.com -tastyfoodideas.com -taximytishi.ru -thegameriders.com -time-japan.ru -timetorelax.biz -tizanidinedrugtest.blogspot.com -tizanidineshowupondrugtest.blogspot.com -top10-way.com -top1-seo-service.com -topkarkas.com -topsiteminecraft.com -track2.shop -tracklead.net -truck-addzilla.life +suhanpacktech.com +tattoo33.ru +templates.franklinfire.co +theautoprofit.ml +tizanidinehcl2mgsideeffects.blogspot.com +tizanidinestreetvalue.blogspot.com +tmm-kurs.ru +tobeyouday.win +toonfamilies.net +topcar-krasnodar.ru +toproadrunner5.info +traxdom.ru +trucri.me trudogolik.net -upproar.com -urldelivery.com -via-gra.webstarts.com -videochat.ph -vip-dom.in -vucms.com -wallpapersist.com -warningzscaler.heraeus.com -website-speed-up.top -weightatraining.com -welck.octopis.com -whatistizanidine2mg.blogspot.com -whatistizanidinehclusedfor.blogspot.com -wieseversa.no -world-mmo.com -wowas31.ucoz.ru -x69ty.ru -xedserver.com -xitjw.info -xn--24-glceagatoq7c2a6ioc.xn--p1ai -xn----7sbbahaq9bb5afgiqfliv4m.xn--p1ai -xn--80aanaardaperhcem4a6i.com -xn--80aebbcbcdemfkhba4byaehoejh8dza3v.xn--p1ai -xn----btbdvdh4aafrfciljm6k.xn--p1ai -xn--d1aifoe0a9a.top -xxxtube69.com -zaapplesales.blogspot.com -zigarettenonlinekaufen.tumblr.com +uchil.net +umityangin.net +uptime-as.net +uvozdeckych.info +vandrie-ict.nl +veles.shop +vesnatehno.com +via-energy-acquistare.com +vkontaktemusic.ru +wait3sec.org +wallpapersdesk.info +warningwar.ru +webextract.profound.net +weddingdresses.xyz +wholesalejerseysgaa.com +win-spy.com +witherrom55.eklablog.fr +wnoz.de +wonderfulflowers.biz +xlolitka.com +xmlinde.com +xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai +xn----7sbbagbq7bd5aheftfllo4m.xn--p1ai +xn--80ahdheogk5l.xn--p1ai +xn--q1a.xn--b1aube0e.xn--c1acygb.xn--p1ai +xxxrus.org +y8games-free.com +yaaknaa.info +ypmuseum.ru +zigzog.ru diff --git a/.dev-tools/test_units/ratelimittest.txt b/.dev-tools/test_units/ratelimittest.txt index e69de29bb..0809108aa 100644 --- a/.dev-tools/test_units/ratelimittest.txt +++ b/.dev-tools/test_units/ratelimittest.txt @@ -0,0 +1,129 @@ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.14.2
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.14.2
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.14.2
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.14.2
+ + + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to the Nginx + + +
+

Welcome to the Nginx Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + \ No newline at end of file diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index cd0a005e8..86992fcfa 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index 773cb25da..439a47c51 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index a1d79cfa0..dc965bcae 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2019.06.1628 +#### Version: V4.2019.06.1629 #### Bad Referrer Count: 6713 #### Bad Bot Count: 556 ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index b50ddd0a1..ac51ff7fb 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2019.06.1628 +#### Version: V4.2019.06.1629 #### Bad Referrer Count: 6713 #### Bad Bot Count: 556 ____________________ diff --git a/README.md b/README.md index 5bc624827..a79b6732b 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2019.06.1628 +#### Version: V4.2019.06.1629 #### Bad Referrer Count: 6713 #### Bad Bot Count: 556 ____________________ diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 929119842..ae9eb3c8e 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2019.06.1628 +#### Version: V4.2019.06.1629 #### Bad Referrer Count: 6713 #### Bad Bot Count: 556 ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 31d75b8fa..494a0ba46 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1628 -### Updated: Wed Jun 26 14:46:12 SAST 2019 +### Version: V4.2019.06.1629 +### Updated: Wed Jun 26 14:52:13 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index f93637b4a..22cb3f271 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -10,8 +10,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1628 -### Updated: Wed Jun 26 14:46:13 SAST 2019 +### Version: V3.2019.06.1629 +### Updated: Wed Jun 26 14:52:14 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 31d75b8fa..494a0ba46 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1628 -### Updated: Wed Jun 26 14:46:12 SAST 2019 +### Version: V4.2019.06.1629 +### Updated: Wed Jun 26 14:52:13 SAST 2019 ### Bad Referrer Count: 6713 ### Bad Bot Count: 556 ################################################### diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 39dc8af7f..6991fb0b2 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,8 +6,8 @@ ### Version Information # ################################################### -### Version: V4.2019.06.1628 -### Updated: Wed Jun 26 14:46:14 SAST 2019 +### Version: V4.2019.06.1629 +### Updated: Wed Jun 26 14:52:15 SAST 2019 ### Bad Bot Count: 556 ################################################### ### Version Information ##