diff --git a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf index ca8905b0d..1af51490b 100644 --- a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf @@ -33,4 +33,4 @@ # 111.111.111.111 0; -35.193.7.13 0; +35.184.96.71 0; diff --git a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf index 9aef5ccdc..d356b01cb 100644 --- a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1597 -### Updated: Tue Jun 25 15:25:42 SAST 2019 +### Version: V4.2019.06.1598 +### Updated: Tue Jun 25 15:42:43 SAST 2019 ### Bad Referrer Count: 6711 ### Bad Bot Count: 556 ################################################### @@ -693,38 +693,38 @@ map $http_user_agent $bad_bot { # *********************************************** # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*\badidxbot\b" 0; - "~*\bAdsBot-Google\b" 0; - "~*\baolbuild\b" 0; - "~*\bbingbot\b" 0; - "~*\bbingpreview\b" 0; - "~*\bDoCoMo\b" 0; - "~*\bduckduckgo\b" 0; - "~*\bfacebookexternalhit\b" 0; - "~*\bFeedfetcher-Google\b" 0; - "~*\bGooglebot\b" 0; - "~*\bGooglebot-Image\b" 0; - "~*\bGooglebot-Mobile\b" 0; - "~*\bGooglebot-News\b" 0; - "~*\bGooglebot/Test\b" 0; - "~*\bGooglebot-Video\b" 0; - "~*\bGoogle-HTTP-Java-Client\b" 0; - "~*\bGravityscan\b" 0; - "~*\bgsa-crawler\b" 0; - "~*\bJakarta\ Commons\b" 0; - "~*\bKraken/0.1\b" 0; - "~*\bLinkedInBot\b" 0; - "~*\bMediapartners-Google\b" 0; - "~*\bmsnbot\b" 0; - "~*\bmsnbot-media\b" 0; - "~*\bSAMSUNG\b" 0; - "~*\bSlackbot\b" 0; - "~*\bSlackbot-LinkExpanding\b" 0; - "~*\bslurp\b" 0; - "~*\bteoma\b" 0; - "~*\bTwitterBot\b" 0; - "~*\bWordpress\b" 0; - "~*\byahoo\b" 0; + "~*(?:\b)adidxbot(?:\b|)" 0; + "~*(?:\b)AdsBot-Google(?:\b|)" 0; + "~*(?:\b)aolbuild(?:\b|)" 0; + "~*(?:\b)bingbot(?:\b|)" 0; + "~*(?:\b)bingpreview(?:\b|)" 0; + "~*(?:\b)DoCoMo(?:\b|)" 0; + "~*(?:\b)duckduckgo(?:\b|)" 0; + "~*(?:\b)facebookexternalhit(?:\b|)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; + "~*(?:\b)Googlebot(?:\b|)" 0; + "~*(?:\b)Googlebot-Image(?:\b|)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; + "~*(?:\b)Googlebot-News(?:\b|)" 0; + "~*(?:\b)Googlebot/Test(?:\b|)" 0; + "~*(?:\b)Googlebot-Video(?:\b|)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; + "~*(?:\b)Gravityscan(?:\b|)" 0; + "~*(?:\b)gsa-crawler(?:\b|)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; + "~*(?:\b)Kraken/0.1(?:\b|)" 0; + "~*(?:\b)LinkedInBot(?:\b|)" 0; + "~*(?:\b)Mediapartners-Google(?:\b|)" 0; + "~*(?:\b)msnbot(?:\b|)" 0; + "~*(?:\b)msnbot-media(?:\b|)" 0; + "~*(?:\b)SAMSUNG(?:\b|)" 0; + "~*(?:\b)Slackbot(?:\b|)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; + "~*(?:\b)slurp(?:\b|)" 0; + "~*(?:\b)teoma(?:\b|)" 0; + "~*(?:\b)TwitterBot(?:\b|)" 0; + "~*(?:\b)Wordpress(?:\b|)" 0; + "~*(?:\b)yahoo(?:\b|)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # *************************************************** @@ -735,12 +735,12 @@ map $http_user_agent $bad_bot { # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*\bjetmon\b" 1; - "~*\blibwww-perl\b" 1; - "~*\bLynx\b" 1; - "~*\bmunin\b" 1; - "~*\bPresto\b" 1; - "~*\bWget/1.15\b" 1; + "~*(?:\b)jetmon(?:\b|)" 1; + "~*(?:\b)libwww-perl(?:\b|)" 1; + "~*(?:\b)Lynx(?:\b|)" 1; + "~*(?:\b)munin(?:\b|)" 1; + "~*(?:\b)Presto(?:\b|)" 1; + "~*(?:\b)Wget/1.15(?:\b|)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************************** @@ -748,23 +748,23 @@ map $http_user_agent $bad_bot { # ************************************************************** # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*\bAlexa\b" 2; - "~*\barchive.org\b" 2; - "~*\bBaidu\b" 2; - "~*\bBUbiNG\b" 2; - "~*\bFlipboardProxy\b" 2; - "~*\bia_archiver\b" 2; - "~*\bMSIE\ 7.0\b" 2; - "~*\bProximic\b" 2; - "~*\bR6_CommentReader\b" 2; - "~*\bR6_FeedFetcher\b" 2; - "~*\bRED/1\b" 2; - "~*\bRPT-HTTPClient\b" 2; - "~*\bsfFeedReader/0.9\b" 2; - "~*\bSpaidu\b" 2; - "~*\bUptimeRobot/2.0\b" 2; - "~*\bYandexBot\b" 2; - "~*\bYandexImages\b" 2; + "~*(?:\b)Alexa(?:\b|)" 2; + "~*(?:\b)archive.org(?:\b|)" 2; + "~*(?:\b)Baidu(?:\b|)" 2; + "~*(?:\b)BUbiNG(?:\b|)" 2; + "~*(?:\b)FlipboardProxy(?:\b|)" 2; + "~*(?:\b)ia_archiver(?:\b|)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; + "~*(?:\b)Proximic(?:\b|)" 2; + "~*(?:\b)R6_CommentReader(?:\b|)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; + "~*(?:\b)RED/1(?:\b|)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; + "~*(?:\b)Spaidu(?:\b|)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; + "~*(?:\b)YandexBot(?:\b|)" 2; + "~*(?:\b)YandexImages(?:\b|)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf index ca8905b0d..1af51490b 100644 --- a/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf @@ -33,4 +33,4 @@ # 111.111.111.111 0; -35.193.7.13 0; +35.184.96.71 0; diff --git a/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf index cebe1bfdc..8e0f74838 100644 --- a/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf @@ -10,8 +10,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1597 -### Updated: Tue Jun 25 15:25:43 SAST 2019 +### Version: V3.2019.06.1598 +### Updated: Tue Jun 25 15:42:44 SAST 2019 ### Bad Referrer Count: 6711 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index cebe1bfdc..8e0f74838 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -10,8 +10,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1597 -### Updated: Tue Jun 25 15:25:43 SAST 2019 +### Version: V3.2019.06.1598 +### Updated: Tue Jun 25 15:42:44 SAST 2019 ### Bad Referrer Count: 6711 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 9aef5ccdc..d356b01cb 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1597 -### Updated: Tue Jun 25 15:25:42 SAST 2019 +### Version: V4.2019.06.1598 +### Updated: Tue Jun 25 15:42:43 SAST 2019 ### Bad Referrer Count: 6711 ### Bad Bot Count: 556 ################################################### @@ -693,38 +693,38 @@ map $http_user_agent $bad_bot { # *********************************************** # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*\badidxbot\b" 0; - "~*\bAdsBot-Google\b" 0; - "~*\baolbuild\b" 0; - "~*\bbingbot\b" 0; - "~*\bbingpreview\b" 0; - "~*\bDoCoMo\b" 0; - "~*\bduckduckgo\b" 0; - "~*\bfacebookexternalhit\b" 0; - "~*\bFeedfetcher-Google\b" 0; - "~*\bGooglebot\b" 0; - "~*\bGooglebot-Image\b" 0; - "~*\bGooglebot-Mobile\b" 0; - "~*\bGooglebot-News\b" 0; - "~*\bGooglebot/Test\b" 0; - "~*\bGooglebot-Video\b" 0; - "~*\bGoogle-HTTP-Java-Client\b" 0; - "~*\bGravityscan\b" 0; - "~*\bgsa-crawler\b" 0; - "~*\bJakarta\ Commons\b" 0; - "~*\bKraken/0.1\b" 0; - "~*\bLinkedInBot\b" 0; - "~*\bMediapartners-Google\b" 0; - "~*\bmsnbot\b" 0; - "~*\bmsnbot-media\b" 0; - "~*\bSAMSUNG\b" 0; - "~*\bSlackbot\b" 0; - "~*\bSlackbot-LinkExpanding\b" 0; - "~*\bslurp\b" 0; - "~*\bteoma\b" 0; - "~*\bTwitterBot\b" 0; - "~*\bWordpress\b" 0; - "~*\byahoo\b" 0; + "~*(?:\b)adidxbot(?:\b|)" 0; + "~*(?:\b)AdsBot-Google(?:\b|)" 0; + "~*(?:\b)aolbuild(?:\b|)" 0; + "~*(?:\b)bingbot(?:\b|)" 0; + "~*(?:\b)bingpreview(?:\b|)" 0; + "~*(?:\b)DoCoMo(?:\b|)" 0; + "~*(?:\b)duckduckgo(?:\b|)" 0; + "~*(?:\b)facebookexternalhit(?:\b|)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; + "~*(?:\b)Googlebot(?:\b|)" 0; + "~*(?:\b)Googlebot-Image(?:\b|)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; + "~*(?:\b)Googlebot-News(?:\b|)" 0; + "~*(?:\b)Googlebot/Test(?:\b|)" 0; + "~*(?:\b)Googlebot-Video(?:\b|)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; + "~*(?:\b)Gravityscan(?:\b|)" 0; + "~*(?:\b)gsa-crawler(?:\b|)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; + "~*(?:\b)Kraken/0.1(?:\b|)" 0; + "~*(?:\b)LinkedInBot(?:\b|)" 0; + "~*(?:\b)Mediapartners-Google(?:\b|)" 0; + "~*(?:\b)msnbot(?:\b|)" 0; + "~*(?:\b)msnbot-media(?:\b|)" 0; + "~*(?:\b)SAMSUNG(?:\b|)" 0; + "~*(?:\b)Slackbot(?:\b|)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; + "~*(?:\b)slurp(?:\b|)" 0; + "~*(?:\b)teoma(?:\b|)" 0; + "~*(?:\b)TwitterBot(?:\b|)" 0; + "~*(?:\b)Wordpress(?:\b|)" 0; + "~*(?:\b)yahoo(?:\b|)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # *************************************************** @@ -735,12 +735,12 @@ map $http_user_agent $bad_bot { # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*\bjetmon\b" 1; - "~*\blibwww-perl\b" 1; - "~*\bLynx\b" 1; - "~*\bmunin\b" 1; - "~*\bPresto\b" 1; - "~*\bWget/1.15\b" 1; + "~*(?:\b)jetmon(?:\b|)" 1; + "~*(?:\b)libwww-perl(?:\b|)" 1; + "~*(?:\b)Lynx(?:\b|)" 1; + "~*(?:\b)munin(?:\b|)" 1; + "~*(?:\b)Presto(?:\b|)" 1; + "~*(?:\b)Wget/1.15(?:\b|)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************************** @@ -748,23 +748,23 @@ map $http_user_agent $bad_bot { # ************************************************************** # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*\bAlexa\b" 2; - "~*\barchive.org\b" 2; - "~*\bBaidu\b" 2; - "~*\bBUbiNG\b" 2; - "~*\bFlipboardProxy\b" 2; - "~*\bia_archiver\b" 2; - "~*\bMSIE\ 7.0\b" 2; - "~*\bProximic\b" 2; - "~*\bR6_CommentReader\b" 2; - "~*\bR6_FeedFetcher\b" 2; - "~*\bRED/1\b" 2; - "~*\bRPT-HTTPClient\b" 2; - "~*\bsfFeedReader/0.9\b" 2; - "~*\bSpaidu\b" 2; - "~*\bUptimeRobot/2.0\b" 2; - "~*\bYandexBot\b" 2; - "~*\bYandexImages\b" 2; + "~*(?:\b)Alexa(?:\b|)" 2; + "~*(?:\b)archive.org(?:\b|)" 2; + "~*(?:\b)Baidu(?:\b|)" 2; + "~*(?:\b)BUbiNG(?:\b|)" 2; + "~*(?:\b)FlipboardProxy(?:\b|)" 2; + "~*(?:\b)ia_archiver(?:\b|)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; + "~*(?:\b)Proximic(?:\b|)" 2; + "~*(?:\b)R6_CommentReader(?:\b|)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; + "~*(?:\b)RED/1(?:\b|)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; + "~*(?:\b)Spaidu(?:\b|)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; + "~*(?:\b)YandexBot(?:\b|)" 2; + "~*(?:\b)YandexImages(?:\b|)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 5e81d3ae5..9f5d346a6 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,50 +1,50 @@ -360Spider -ADmantX -Asterias -Bigfoot -BLEXBot -Blow -Bullseye -ChinaClaw -Chlooe -coccocbot-web -cognitiveseo -DataCha0s -DomainCrawler -ExtLinksBot -Extreme Picture Finder -HTTP::Lite -InfoTekies -InternetSeer -internetVista monitor +Aboundex +AiHitBot +BetaBot +BlackWidow +Brandwatch +Discoverybot +DnyzBot +EasyDL +Ecxi +EirGrabber +Flunky +Fyrebot +GridBot JennyBot -JustView -Kenjin Spider -Lanshanbot -Likse -LinqiaMetadataDownloaderBot -Lipperhey -Mail.RU_Bot -Metauri -Name Intelligence -Nameprotect -netEstate NE Crawler -NetZIP -PeoplePal -Picsearch -PxBroker -RealDownload -Screaming -SearchmetricsBot -SocialRankIOBot -Sosospider -sysscan -TightTwatBot -Voltron -WallpapersHD -WebAuto -Web Auto -Web Enhancer -Web Pix -Web Sucker +JetCar +JikeSpider +Joomla +Jyxobot +Keyword Density +LibWeb +lwp-trivial +Magnet +Mata Hari +MauiBot +Microsoft Data Access +Microsoft URL Control +Nmap +OpenVAS +RankurBot +SEOlyticsCrawler +seoscanners +SEOstats +sexsearcher +Siteimprove +Sucker +tAkeOut +Teleport +Trendictionbot +Turingos +VCI +Webalta +WebCopier +WebmasterWorldForumBot +Webshag +Webster +WWW-Mechanize Xenu +Zauba +Zeus +ZmEu diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 6ad939846..999f69d23 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ +360Spider 404checker 404enemy 80legs -Abonti +Aboundex Aboundexbot AfD-Verbotsverfahren AIBOT AiHitBot +Aipbot Alligator +AllSubmitter +AlphaBot +Apexoo +archive.org_bot ASPSeek +Asterias +autoemailspider +Backlink-Ceck backlink-check BacklinkCrawler -BackStreet -BatchFTP -BBBike -Blackboard -Black Hole +BackWeb +Badass +Bandit +Barkrowler +Battleztar Bazinga +BDCbot +BetaBot +Bitacle BlackWidow -BLEXBot -Bolt -Brandprotect -Buddy -BunnySlippers -CazoodleBot +BlowFish +Boardreader +BotALot +Brandwatch +BuiltWith +Cegbfeieh CheeseBot -CherryPicker -CheTeam ChinaClaw -Chlooe -Claritybot +Cloud mapping +coccocbot-web Cogentbot -CopyRightCheck -Copyscape +cognitiveseo +com.plumanalytics +Copier +crawler4j +crawler.feedback +crawl.sogou.com Crescent -Custo -DatabaseDriverMysqli +DBLBot +demandbase-bot +Demon Deusu -DigitalPebble -Dirbuster -Disco -DnyzBot -DomainCrawler -DomainSigmaCrawler -DomainStatsBot +Devil +Digincore +DIIbot +Discoverybot Dragonfly +EasyDL eCatch -ECCP/1.0 -EMail Siphon -EMail Wolf +Ecxi +EroCrawler +evc-batch Evil +Express WebPictures ExtLinksBot Extractor -ExtractorPro EyeNetIE -facebookscraper -FDM -FemtosearchBot -FlashGet -FrontPage -Genieo -GoZilla +Ezooms +GalaxyBot +GermCrawler +Getintent +GetRight +GetWeb +Gigablast +Gigabot +Go-Ahead-Got-It Go!Zilla -Grabber +Grafula GrapeFX -GridBot +GT::WWW HaosouSpider -Harvest -Havij Heritrix Hloader +HMView +HTMLparser HTTP::Lite HTTrack Humanlinks -HybridBot IDBot -Id-search -IlseBot -Image Fetch Image Sucker -Intelliseek -InterGET -internetVista monitor -Iria -Iskanie +Indy Library +InfoNaviRobot +InfoTekies +InternetSeer +ips-agent JennyBot Joomla Jyxobot -Kenjin Spider -Lanshanbot -Lftp -LinkWalker +Kozmosbot +Larbin +LeechFTP +Libwhisker +Lightspeedsystems +LinkextractorPro +LinqiaMetadataDownloaderBot LinqiaRSSBot -Lipperhey -Lipperhey Spider +LinqiaScrapeBot Litemage_walker Lmspider -LWP::Simple -Mag-Net -magpie-crawler -Majestic12 -MarkMonitor -MarkWatch -MauiBot -MeanPath Bot +Ltx71 +Masscan +Mass Downloader +meanpathbot +Meanpathbot Mediatoolkitbot +mediawords MegaIndex.ru -Metauri Microsoft Data Access -Microsoft URL Control MIDown tool MIIxpc -Mister PiX -Mr.4x3 -MSFrontPage -Msrabot +Morfeus Fucking Scanner +MSIECrawler MS Web Services Client Protocol -Name Intelligence -Nameprotect +Musobot +Navroad NearSite -Needle Nessus Netcraft -NetSpider -Nettrack -Net Vampire +NetMechanic Netvibes +NetZIP +Nibbler +NICErsPRO +Niki-bot +Nikto NimbleCrawler -Nimbostratus Nmap -Octopus -Offline Explorer +NPbot +Offline Navigator OnCrawl -Openfind -OpenLinkProfiler Openvas OpenVAS -OrangeSpider -OutclicksBot -PageGrabber +OutfoxBot +PageAnalyzer page scorer -PageScorer -Pandalytics Panscient pcBrowser -PECL::HTTP -PHPCrawl -PictureFinder +PeoplePal +Picsearch Pi-Monster +PleaseCrawl plumanalytics Pockey +POE-Component-Client-HTTP Probethenet -ProWebWalker -Psbot +ProPowerBot Pump PyCurl -QueryN Metasearch -RankActive +Quick-Crawler +RankActiveLinkBot Rankivabot RankurBot -RealDownload +Reaper RebelMouse -ReGet -Ripper -Rogerbot -ScanAlert +RocketCrawler +s1z.ru +SalesIntelligent +SBIder Scanbot -ScoutJet Scrapy -Screaming -Searchestate +ScreenerBot +Semrush SEOkicks -SEOkicks-Robot +SEOlyticsCrawler SEOprofiler -seoscanners -SeoSiteCheckup SEOstats -Shodan +serpstatbot Siphon -Sitebeam -SiteExplorer -Siteimprove SiteLockSpider -SiteSnagger -Site Sucker +SiteSucker SlySearch SmartDownload Snake -Snoopy -SocialRankIOBot -Sociscraper -sogouspider +Sogou web spider Sosospider Sottopop -SpaceBison +Spammen SpankBot Spanner sp_auditbot -SputnikBot +Spinn3r spyfu +Sqlworm Sqworm -Steeler +Stripper +Sucker Sucuri SuperBot SuperHTTP -Surfbot -sysscan -Teleport +Swiftbot +T0PHackTeam +tAkeOut TeleportPro +Telesoft +The Intraformant TheNomad -Thumbor TightTwatBot -trendiction.com -trendiction.de -Turnitin +Titan +Trendiction +Turingos TwengaBot Upflow -URLy Warning URLy.Warning Vagabondo VB Project -VCI -Virusdie -Wallpapers/3.0 -Webalta -WebAuto -WebBandit -WebEnhancer +VidibleScraper +WallpapersHD +WASALive-Bot +WBSearchBot +WebCollage +WEBDAV WebFuck -WebImageCollector +WebLeacher WebPix +Web Pix +WebReaper WebSauger Web Sauger -WebsiteExtractor WebsiteQuester Website Quester -WebStripper +Webster WebSucker -Whatweb -Who.is Bot -Widow -Woobot +Web Sucker +WebZIP +Whack +WinHTTrack +WiseGuys Robot Wotbox -WWW-Mechanize -WWW::Mechanize -x09Mozilla -Xenu -zauba.io +Wprecon +WWW-Collector-E +WWWOFFLE +Xaldon_WebSpider +xpymep1.exe +Zade +Zauba +Zermelo +Zeus ZmEu ZumBot ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 256064ca4..442cd484a 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ +360Spider 80legs -Abonti Aboundex -Acunetix -AhrefsBot +Aboundexbot +AfD-Verbotsverfahren AIBOT -Anarchie -Attach +Alligator +AllSubmitter +AlphaBot autoemailspider BackDoorBot +Backlink-Ceck +backlink-check BacklinkCrawler -Badass +BackWeb +Bandit +Barkrowler BatchFTP Battleztar Bazinga -BBBike -Blackboard +BlackWidow BLEXBot BlowFish Boardreader -BotALot -Brandwatch +Bolt +Brandprotect BuiltBotTough Bullseye BunnySlippers BuzzSumo +Calculon CATExplorador CazoodleBot -CCBot Cegbfeieh -Chlooe +CheeseBot +CheTeam +ChinaClaw +Claritybot Cliqzbot -coccocbot-web cognitiveseo -Collector -com.plumanalytics -crawler4j +CopyRightCheck +Cosmos crawler.feedback CrazyWebCrawler Crescent CrunchBot +Curious Custo -DataCha0s +DatabaseDriverMysqli +DBLBot demandbase-bot Demon -Digincore -DigitalPebble +Deusu +DIIbot Dirbuster +Discoverybot +Dispatch DittoSpyder DomainAppender DomainSigmaCrawler +DomainStatsBot +Dotbot +Dragonfly +Drip +Ebingbong eCatch EirGrabber -EMail Wolf -EroCrawler -evc-batch -Evil -ExtLinksBot -Extractor -ExtractorPro -EyeNetIE -facebookscraper -FDM +FemtosearchBot +FHscan Fimap -FlashGet +Flunky FrontPage +FyberSpider +Fyrebot +GalaxyBot Genieo -Gigablast -Gigabot +GermCrawler +Getintent +GetRight G-i-g-a-b-o-t Go!Zilla -Grabber -GrabNet -Grafula -HaosouSpider -HEADMasterSEO -Heritrix -HTTP::Lite +GrapeFX +Haansoft +Harvest +HMView +HTTrack HybridBot +Iblog Id-search -IlseBot Image Fetch -instabid +IndeedBot +InfoTekies Intelliseek -JamesBOT -JetCar +InternetSeer +internetVista monitor +IRLbot Jetty +JikeSpider JOC Web Spider -Keyword Density -Kozmosbot +Kenjin Spider Lanshanbot -Lftp +LeechFTP +LeechGet +LexiBot LibWeb -Likse -Linkdexbot -LinkextractorPro +Lightspeedsystems LinkpadBot LinksManager +LinqiaMetadataDownloaderBot +LinqiaRSSBot LinqiaScrapeBot -Lipperhey +Litemage_walker Lmspider lwp-request -LWP::Simple Magnet -Mag-Net -Majestic12 -Majestic SEO +magpie-crawler +Mail.RU_Bot MarkMonitor -Masscan +MarkWatch Mass Downloader Mata Hari MauiBot +Meanpathbot +Mediatoolkitbot +mediawords +Metauri MFC_Tear_Sample -Microsoft Data Access -Mojeek +MIDown tool +Mister PiX +Morfeus Fucking Scanner MSIECrawler -Msrabot -muhstik-scan -Navroad +Musobot +Name Intelligence +Nameprotect +Nessus +Netcraft netEstate NE Crawler -NetLyzer -NetMechanic +Nettrack NextGenSearchBot -Niki-bot -Nikto -Nimbostratus +NICErsPRO Ninja -Nmap -NPbot Nutch -Octopus +oBot Offline Explorer -Offline Navigator -OpenLinkProfiler +Openfind OpenVAS OrangeBot -OutfoxBot -PageAnalyzer +OrangeSpider +OutclicksBot PageGrabber -page scorer Pandalytics -pcBrowser +Panscient +Pavuk PeoplePal -Picscout -Picsearch +PictureFinder Pimonster -Pi-Monster PleaseCrawl -plumanalytics -Pockey POE-Component-Client-HTTP -ProPowerBot -Psbot +Probethenet Pump +PxBroker +PyCurl +RankActive +RankActiveLinkBot +RankingBot +RankingBot2 RealDownload -RebelMouse +Reaper +Recorder RedesScrapy -Ripper +ReGet +RepoMonkey +RocketCrawler +Rogerbot s1z.ru -SBIder -SEOkicks -SEOkicks-Robot +Scrapy +Screaming +ScreenerBot +Searchestate +SearchmetricsBot +Semrush +SEOlyticsCrawler SEOprofiler -seoscanners SeoSiteCheckup -SEOstats -serpstatbot sexsearcher -Siphon -Sitebeam -SiteExplorer -Siteimprove +SISTRIX SiteSnagger SiteSucker Site Sucker -SlySearch -SMTBot -Snapbot -Snoopy +SocialRankIOBot +Sociscraper sogouspider -Sosospider +Sogou web spider Sottopop SpaceBison Spammen -Spanner sp_auditbot -Spbot -SputnikBot -Sqlworm -Steeler -Sucker -Sucuri +Spinn3r +spyfu +Stripper SuperBot -SuperHTTP +SurveyBot sysscan -T0PHackTeam -Telesoft -Telesphoreo -Telesphorep -Toata +tAkeOut +Teleport +TeleportPro +The Intraformant +TheNomad +Thumbor +TightTwatBot +Titan Toweyabot Tracemyfile -Trendiction Trendictionbot -trendiction.com -trendiction.de -True_Robot -Turnitin +Turingos TwengaBot +Twice +UnisterBot Upflow +URLy Warning URLy.Warning -Vacuum Vagabondo -VCI -Virusdie -Voil +VB Project +VeriCiteCrawler +VidibleScraper +WallpapersHD +WBSearchBot Webalta -Web Auto Web Collage -WEBDAV +WebCopier WebEnhancer +Web Enhancer WebFetch Web Fetch -WebFuck -Web Fuck WebGo IS -Web Pix -WebSauger +WebReaper Web Sauger -Webshag -WebsiteQuester -Webster +Website Quester WebStripper -WebSucker -WebWhacker -WebZIP +Whack Whatweb WinHTTrack WiseGuys Robot WISENutbot -Wonderbot -Wotbox -Wprecon -WWW::Mechanize -WWWOFFLE +x09Mozilla +Xaldon WebSpider xpymep1.exe -YoudaoBot Zauba -Zermelo -zgrab +zauba.io +Zeus ZumBot diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index c4883917f..50115086f 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,100 +1,100 @@ -100searchengines.com -3rbseyes.com -activecampaign.dreamhosters.com -adult-shop.com.ua -airlandsea.info -angry-fermi-7633.arukascloud.io -apartmentratings.com -apisecretsaucebi-a.akamaihd.net -artefakct.com -autobrennero.it -autonew.biz -bankiem.pl -bannerads.de -beauty-bracelet.com -bezcmexa.ru -bitporno.sx -bjanshee.ru -bonky.biz -break-the-chains.com -btvn.ru -compliance-alex.top -compliance-josh.top -conocer-sanabria.com -cpajunkies.com -darmebel.com.ua -detsky-nabytek.info -dtnlyss.com -ekto.ee -elektrozigarette1.dreamwidth.org -eloconcream.blogspot.com -exoneration-project.us -fitness-video.net -floating-share-buttons.com -focalink.com -foodcrafts.website -freakycheats.com -froggytube.com -gameprimary.com -gamewrath.com -gayxperience.com -gemgrab-a.akamaihd.net -girlgamerdaily.com -gkvector.ru -glavtral.ru -grand-chlen.ru -greatfind-a.akamaihd.net -gsasearchengineranker.pw -hackers-crackers.tk -hanink.biz.ly -holidaypics.org -hotxnights.info -hoztorg-opt.ru -images.gyffu.com -imperia31.ru -ip-guide.com -istizanidineanopiate.blogspot.com -jav-fetish.com -kuzinsp.ru -laptop-4-less.com -likesuccess.com -mantramusic.ru -maridan.com.ua -meltwater.com -missclub.info -mobplayer.ru +000free.us +4pp13.com +accessoires-mode.in +anapa-inns.ru +android-vsem.org +aruplighting.com +autodo.info +ayeartoforget.com +battle.net +betteroffers.review +bizlist.com.de +brandedleadgeneration.com +brk-rti.ru +cartechnic.ru +cbox.ws +cheapbarbour.online +cheapjerseysa.com +clips.ua.ac.be +compliance-john.top +cranly.net +druzhininevgeniy63.blogspot.com +esfchat.tk +filerockstar298.weebly.com +findclan.org +fitfloponline.store +fivedwld.cf +freshwallpapers.info +getmiro.com +gnuetella.com +graphics8.info +gratuitbaise.com +gwebtools.com +hit-men.men +hotblognetwork.com +infazavr.ru +istizanidineanarcoticdrug.blogspot.com +jewelryandfiligree.com +kabbalah-red-bracelets.com +kerch.site +kinobaks.com +letmacworkfaster.site +letslowbefast.site +liky.co.ua +lineshops.biz +luxmagazine.cf +makedo.ru +monclerjacketsoutlet.pw +montredemarque.nl myiptest.com -narkologiya-penza.ru -niroo.info -pandarastore.top -partner-cdn.men -phpdevops.com -privatamateure.com -pts163.ru -purchasepillsnorx.com -rednise.com -samo-soznanie.ru -scat.porn -security60-e.com -service.adtech.us -sfd-chess.ru -shoppytoolmac.com -space-worry.ml -spy-sts.com -stuff-about-money.com -suchenindeutschland.com -tattooha.com -tiens2010.ru -tizanidinehcl2mg.blogspot.com -top250movies.ru -touchmods.fr -tubo360.com -videospornogratisx.net -vilingstore.net -warezaccess.com -website-speed-up.site -w-journal.ru -ww1943.ru -youporn-ru.com -zeikopay.com -zophim.me +myonlinepayday.co +na-telefon.biz +novodigs.com +offf.info +onlinewritingjobs17.blogspot.ru +o-o-8-o-o.ru +openmultipleurl.com +orgasmatrix.com +osnova3.ru +oxford-book.com.ua +piccdata.com +porno.simple-image.com.ua +prostitutki-odessa.org +proxyradar.com +quality-traffic.com +radiodigital.co +razyboard.com +refads.pro +sa-live.com +sankt-peterburg.nodup.ru +sdrescher.net +selectads.men +seminarygeorgia59.ga +seocheki.net +seogadget.ru +seokicks.de +sexcamamateurchat.com +sharpchallenge.com +simple-image.com.ua +sindragosa.comxa.com +social-buttons.com +spammen.de +svetlotorg.ru +tabakur77.com +tagil.zrus.org +tech4master.com +thewebsitetemplate.info +tmearegion26.com +tuckermktg.com +ukrobstep.com +us-america.ru +vacances-voyages.info +vladimir.zrus.org +vote-up.ru +vysigy.su +wikes.20fr.com +xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai +xn--h1ahbi.com.ua +y8games-free.com +zoogdisany.com +zrus.org diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 9e06a9085..f49cfdb70 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,1000 +1,1000 @@ -0daymusic.org -1001watch.com.ua -101flag.ru -11235813.webzdarma.cz -1688.com +00go.com +0912701309f8ce.com +0n-line.tv +123any.com +12bet.com 1flag.co.za -1kdailyprofit.me -1webmaster.ml -202ch.com -21h2o.com -256bit.by -2x2fan.ru -300richmond.co.nz -34.gs -3rbseyes.com -45en.ru -4inn.ru -4k-player.pl -4ur.click -66cpwgln.space +1kinobig.ru +1qingdao.com +2000k.ru +214jaluwobapef.cf +2345.com +24x7-server-support.site +273-fz.ru +4kplayer.pl +5kstore.com +777-club.ru 7minuteworkout.com -7zap.com -8xv8.com -91abcw.com -aaa38852e886ac4af1a3cff9b47cab6272.com -aab94f698f36684c5a852a2ef272e031bb.com +7xc4n.com +883zy.com +8gold.com +9med.net +a96527gi.beget.tech +aa08daf7e13b6345e09e92f771507fa5f4.com +aarbur.com abctoppictures.net -academia-nsk.org -acmebtn.ml -adclickthru.net -addtoadd.men -adktrailmap.com -admatic.com.tr -admeasures.com -adsref.men -adultfriendfinder.com +abiente.ru +abusora.com +acc.eu.org +acheterviagrafr24.com +acortarurl.es +adclickservice.com +adf.ly +adhome.biz +adnotbad.com +adrenalin-stk.ru +adrunnr.com +ads-seo.men +adtology.com adultfullhd.com -adultmeetonline.info -advancedcleaner.com -advocatemsk.ru -advokateg.xyz -aero2.ru +advokateg.ru +afslankpillen2017nl.eu agadelha.com.br -agysya.ru -ahmedabadwebs.com -ahrntal.verymes.xyz +agahinameh.com +agreda.pluto.ro aihelen.net -alarmobninsk.ru -alert-jdh.xyz -alfa-img.com -allboard.xobor.de +algarveglobal.com allcredits.su -allforminecraft.ru -allkrim.com -allmarketsnewdayli.gdn -allsilver925.co.il -allstatesugarbowl.org -alphacarolinas.com -alveris.ru +alldownload.pw +allnews24.in +alot.com +alphaforum.ru +alta-realestate.com alyeskaresort.com -amazon-adsystem.com -amoi.tn -anaksma.info +amanda-porn.ga +am-se.com anal-acrobats.com +anal-acrobats.hol.es an-donut.com -angel1777.kz -angry-fermi-7633.arukascloud.io -animali.deagostinipassion.it -anime.dougasouko.com +androids-store.com +android-vsem.org +angkortours.vn +animebox.com.ua animenime.ru -apartment.ru -apessay.com -apiadanaknet-a.akamaihd.net -apimossnetinfo-a.akamaihd.net -apimountainbikei-a.akamaihd.net -apioasisspacenet-a.akamaihd.net -apiportalmorecom-a.akamaihd.net -apisecretsaucebi-a.akamaihd.net +anjalika.co.in +anlimebel.kiev.ua +anmysite.com +apartamentwroclaw.eu +apiallgeniusinfo-a.akamaihd.net +apiappenableinfo-a.akamaihd.net +apibrowsesmartne-a.akamaihd.net app5.letmacworkfaster.world -appearance-cool.com -appfixing.space -apps-analytics.net +apper.de +apple.com-webbrowsing-security.science ap.senai.br -arabsexxxtube.com -arabseyes.com +apxeo.info +arcadepages.com arcteryxstore.online arendas.net -arendatora.ru -artavenuegardenstudios.com -artel116.ru +arius.tech articlesdirectoryme.info -artpicso.com -aruplighting.com -asophoto.com -asseenontvonline.ru -atlant-auto.info -atout-energie-69.com -atovh.local-finders.com -autochoixspinelli.com -autovideobroadcast.com +asacopaco.tk +asmxsatadriverin.aircus.com +asseenontv.ru +atlasvkusov.ru +autobrennero.it +automatic-seo.com +autorn.ru +avek.ru +aviav.eu aviav.ru -avkzarabotok.com +avitocars.ru avkzarabotok.info -avtolombard-krasnodar.com -awaybird.ru -azartclub.org +avtoarenda.by +avtointeres.ru +awency.com +azazaporn.com +azbuka-mo.ru +azlex.uz +babespcs.com +babieca.com backuperwebcam.weebly.com -baersaratov.ru -bala.getenjoyment.net -balitouroffice.com -bambi.ck.ua -bamo.xsl.pt -bankmib.ru -bannerspace.com -baoxaydung.com.vn -battlecarnival.su -battleforupdating.site -bbsoldes.fr -begalka.xyz -belinka.com.ua +bag77.ru +baixar-musicas-gratis.com +balkanfarma.org +bankcrediti.ru +bannerbank.ru +beclean-nn.ru +belinvestdom.by benchmarkcommunications.co.uk -bestbuy.ca -bestfortraders.com -best-games.today -bestkfiledxd.cf -bestplacetobuyeriacta.jw.lt -bestprofits-there.com -best-seo-software.xyz -bestwrinklecreamnow.com -betterscooter.com -betwinservice.com -bidbuy.co.kr -big-boards.info -biketank.ga -bio-optomarket.ru -bjanshee.ru -bkgr.se -blacktwhite.com -blavia.00author.com -blogqpot.com -blogrankers.com -blogsfunky672.weebly.com +bensbargains.net +best-businessman.ru +besthoro.ru +besttorrentknifta.weebly.com +bet365.com +bgviagrachrx.com +bigpictures.club +billiard-classic.com.ua +blackwitchcraft.ru +bloggerads.net +blog.koreadaily.com bloke.com -bluejays-jerseys.us -board.f00d.de -bochemit.com.ua -bodybuilding-shop.biz -bonzbuddy.com -bookhome.info -bookmaker-bet.com -break-the-chains.com -breastaugmentation.co.za -brokergid.ru -btvn.ru -bupropion-sr-150-mg.us -buyhoverboard.com -byme.se -cacheimages.com -cactussoft.biz -callcafe.info -call-of-duty.info -callstevens.com -candcstuccoandstone.com -candlehandmade.com +blue-square.biz +bongacams.com +bongiornos.info +bookmark4you.com.biz +brainsandeggs.blogspot.com +brandbucket.com +brandov.ru +bristolhostel.com +brus-vsem.ru +buckspar.gq +buqayy0.livejournal.com +burger-imperia.com +burger-tycoon.com +burkesales.com +businescoop.men +buttons-for-website.com +buttons-for-your-website.com +buydissertation.net +buyessayonline19.blogspot.ru +buy-loft.ru +buy-meds24.com +buypuppies.ca +buytizanidineonlinenoprescription.blogspot.com +buzzzg.men +bwlx.prepedu.cn +calc-for-credit.ru +camsex.xxx-cam.webcam candy-glam-hp.com -carapics.com -cardsharp1.ru +canyougethighofftizanidine.blogspot.com +cardiosport.com.ua +carfax.com.ua carina-sy.de -cartierlovebraceletreplica.xyz -cartierlovebracelet.xyz -casinorewards.info -castingbank.ru -cbcseward.com +carmuffler.net +carsplate.com +cartierbraceletsreplica.pw +cartierreplica.top +cayado.snn.gr celebsopics.com -centre-indigo.org.ua -chatseo.com -cheapjerseysap.com -cheapjerseysfootballshop.com +cenokos.ru +centrumcoachingu.com +ceotrk.com +cete.ru +c-gainsbourg.com +changde.58.com +cheapbarbour.online +cheapjerseysbizwholesale.us checkhit.com -check-host.net checkm8.com -checktext.org -cherubinimobili.it -children-learningreading.info -chinajerseyswholesalecoupons.com -chinese-amezon.com -cialgenisrx.com -cialis-samples.com -ci.ua -civilwartheater.com +chiblackhawks-jerseys.com +chlooe.com +chocolateslim-en-espana.com +chocolateslim-en-france.com +chocolateslim-original.com +choosecuisine.com +chorus.terakeet.com +cialischsrx.com +cienum.fr +cigarpass.com +cindymatches.com +cineacademy.ru +cinemaenergy-hd.ru +citysecurity.nu +cleanallvirus.com +cleanmypc.com +cleanpcnow.com +clickbanksites.info +clickintext.net +clickprotects.com +clicksor.com clicksor.net -clipartnew.com +clickzzs.nl clothesforcash.com -club-samodelkin.ru -coindirect.io -companies-catalog.com +coin-hive.com +cometorussia.net +comissionka.net compiko.info -compliance-alex.xyz -compliance-don.top +compliance-barak.xyz compliance-fred.top -compliance-ivan.top -compliance-jessica.top -compliance-julia.top -compliance-nelson.top -confib.ifmo.ru -converse.ddsoldes.fr +compliance-josh.top +compliance-julianna.top +com-secure.download +comsysnet.com +conciergegroup.org +connectingsingles.com +consorzioilmosaico.org +construmac.com.mx +contextualyield.com +coolwallpapers-hd.com +coop-gamers.ru +copenergo.ru coslab.club +cougfan.info +countbertwistdisp26.soup.io counter.bloke.com -counterbot.com -courtshipgift.com -covetnica.com -cranly.net +counter.yadro.ru +country-chic.ru +craftburg.ru creditnation.ru -crest-poloski.ru -cryptoswap.biz -culturevie.info -customboxes4less.com +creditwell.ru +crirussian.ru +crystalslot.com customcatchcan.com -cypernhuset.se -czeshop.info -dailyrank.net -danschawbel.com -daretodonate.co +custom-product-labels.com +cyse.tk +d2jsp.org +dafatiri.com +daisye.top +darkbooks.org +data1.scopich.com +dataloading.net +dawlenie.com d-black.bz -deanmoore.ie -decorationspcs.com -dellalimov.com -delta-line.men -deniven.1bb.ru -deshevo-nedorogo.ru -detskie-konstruktory.ru -dev-seo.blog -diarioaconcagua.com -diegolopezcastan.com +db.speedup.gdn +dcj-nn.ru +ddlmega.net +dealitright.click +dealwifi.com +delitime.info +dengi-pod-zalog-nedvizhimosti.ru +detmebel.su +dev.citetick.com +dickssportinggoods.com dieta-personalna.pl -diffbot.com +digilander.libero.it digitalfaq.com -diminishedvalueoforegon.com -directrev.com -dispo.de -dividendo.ru -djonwatch.ru -dlya-android.org -dogclothing.store -dogoimage.com -dood.live -download-of-the-warez.blogspot.com -downvids.net -dpihatinh.gov.vn -drev.biz -druzhbany.ru -duplicashapp.com -dvd-famille.com -dvervmoskvu.ru -dyshagi.ru -easytuningshop.ru -easyukraine.com -ec-file.info -e-collantes.com -edudocs.net -ee77ee.com -egvar.net -ekaterinburg.xrus.org -eladkarako.com -elektrischezi.canalblog.com +digital-sale.su +dimkino.ru +diusyawiga.tk +djihispano.com +dneprsvet.com.ua +docstoc.com +doctissimo.fr +dokfilms.net +domcran.net +domznaniy.ru +donvito.unas.cz +doska-vsem.ru +dostavka-v-krym.com +downloaddy.net +downloads-whatsapp.com +dprtb.com +dptaughtme.com +driving.kiev.ua +droidlook.net +drpornogratisx.xxx +druzhininevgeniy63.blogspot.com +dustyorate.com +dverimegapolis.ru +eandsgallery.com +easyshoppermac.com +ebooknovel.club +ecxtracking.com +edelstahlschornstein-123.de +ed-shop01.ru +education-cz.ru +efkt.jp +egovaleo.it +ekspertmed.com +ekspresihati.info +eleimgo.pw +elektrischezigarette2.onsugar.com elektrischezigaretten2.beeplog.com -elektronischezigarettekaufen1.beeplog.com -elektronischezigarettekaufen1.myblog.de -elektronischezigarettekaufen2.tumblr.com -elitepcgames.com -embedle.com -enbersoft.com +elektrozigarette2.wordpressy.pl +elektrozigaretten1.postbit.com +elidelcream.weebly.com +energydiet-info.ru enge-fotzen.info -engines-usa.com -englishdictionaryfree.com -en.home-task.com -enskedesquashclub.se -envaseslotusama.com -ereko.ru +englishgamer.com +enternet.ee +escortslet.net +eshop4u.jp +essaypro.com este-line.com.ua -etur.ru -euromasterclass.ru -euronis-free.com -evening-dating-club.info -everypony.ru +etenininrade.ga +eu-cookie-law.info +eugenevaultstorage.com +eupornstar.info +evogarage.com +exct.net exdocsfiles.com -exonline.info +expresstoplivo.ru extlinks.com -extractorandburner.com -eywords-monitoring-your-success.com -ezaz.info -ezigarettenkaufen1.pagina.gr -facecup.top -fandlr.com -fashionindeed.ml +extrabot.com +extstat.com +ezigarettekaufen2.mpbloggar.se +ezigarettekaufen.myblog.de +ezigarettenshop1.yolasite.com +ezigarettenshop2.myblog.de +f012.de +f0815.de +farm26.ru +farmingworm.com +farmprofi.net fastfixing.tech -fast-wordpress-start.com -favornews.com -fbdownloader.com -fellowshipoftheminds.com -fetlifeblog.com -filmci.pro -filmfanatic.com -financehint.eu -financeloan.us +fatmaelgarny.com +ferieboligkbh.dk +fifa-coins.online +fikasound.tk +filefilter.weebly.com +filesmonster.porn +filmi-onlain.info +filunika.com.ru find1friend.com -findthe.pet -fitness-video.net -flashlarevista.com -florida-tourism.net -flowwwers.com -f-online.de -foodcrafts.website -forensicpsychiatry.ru -forex21.ru -forexgb.ru -formseo.com -foto-basa.com -fotoxxxru.com -free-fbook-traffic.com +findclan.org +fitnesspiks.com +flauntyoursite.com +flex4launch.ru +flirt4free.com +flooringinstallation-edmonton.com +floridahuntingfishingadventures.com +fodelsedagspresenter.nu +food.dtu.dk +footbalive.org +forex-indextop20.ru +formulaantiuban.com +forpostlock.ru +fortwosmartcar.pw +foxtechfpv.com +free-social-buttons.com free-today.com -freeuploader.com -freeuploader.ml -freewebs.com +free-traffic.xyz +freewlan.info +frequiry.com freshberry.com.ua -frighteningremain.cf -frvo.alptandem.ru +froggytube.com +frustrated-favorable.gq ftns.ru -fuck-paid-share-buttons.xyz fun-mobi.pl -fyl.com.ru -g33.org -gagrasector.ru -galaxys6manual.info -gallerylisting.com -gambarkatabaru.com -gambarxkata.co -gamedayhouse.com -gameonasia.com -gameplexcity.com -gamesprite.me -gaming-journal.com +funnel.co.za +gael-s.ru +gallictures.com +gamblingnerd.com +gamebackyard.com +games.kolossale.ru +gamingspark.com gay.adultgalls.com -gaytube.com -gembird.com -generalporn.org -get-free-social-traffic.com -getresponse.com -get-seo-domain.com +gazobeton-p.com.ua +gazporno.com +gdebestkupit.ru +generic-pills-online.com +getaclueamerica.com +getridofstretchmarks.org get-your-social-buttons.info -given2.com -gktt.ru -glavprofit.ru -glcomputers.ru -globalscam.ga +gkvector.ru +glopages.ru gloverid.site -go2jump.org -goldandcard.ru -gombita.info +go2album.com +goatse.ru +golmau.host.sk gomusix.com -gorodservis.ru -gosarhivrt.ru -gotcher.us -gowreckdiving.com +goodhumor24.com +good-mummy.ru +googlefeud.com +gq-catalog.gq gratis-sexkontakte.com -gratuitbaise.com -greatzip.com -gsasearchenginerankerdiscount.com -g.starmoe.xyz -gta-top.ru +gratuitxblcodes.com +greendream.com.ua +gribkovye-zabolevaniya.com +gsasearchenginerankerhelp.com +gsasearchengineranker.top +gsmlab.pl +gsou.cf +gtopstats.com +guitar-master.org +guod.me +gwebtools.com.br +gwhwpxbw.bloger.index.hr gyffu.com -hackers-crackers.tk -hard-porn.mobi -hayate.biz -hazardky.net -hdfreeporno.net +hahashka.ru +hamilton.ca +hanink.biz.ly +harmonyglen.us +hauleddes.com +haveinc.xyz +hccoder.info +hchha.com +hdimagelib.com hdwalls.xyz -healbio.ru -healing-dysplasia.ru -hebr.myddns-flir.com -helicalpile.us -hermesbirkinhandbagoutlets.com -heygidday.biz -himazin.info -historichometeam.com -hkdiiohi.skyrock.com -hobbyhuren24.net -homegardenlova.com -homeinns.com -hopeonthestreet.co.uk -host-tracker.com -hotchatdate.com +headpress.ru +hifidesign.ru +hikesearch.net +histock.info +hitcpm.com +hit-kino.ru +homedecorpicture.us +homemature.net +hoporno.com hotel-mkad.ru hotgirlhdwallpaper.com -hotloans.ru -housediz.com -housemilan.ru -hoverboard360.at -huimin764128.com -hully.altervista.org -hut1.ru -iamsport.org -iccornacircri.cf -ico.re -ideoworld.org -ilmen.net +hotshoppymac.com +hoverboard360.nl +husky-shop.cz +hustoon.over-blog.com +hvd-store.com +hydropump.su +hyipmanager.in +idc.com.ua +idegenvezeto.eu +ideibiznesa2015.ru +id-forex.com +ihc2015.info +i-hobot.ru +iklysha.ml +ilbe.club imagecoolpub.com -imagui.eu imgata.com -imitex-plus.ru -imperiafilm.ru -impisr.ru -inboxdollars.com -incanto.in.ua +imicrovision.com +immobiliaremassaro.com incest-ru.com incolors.club -inet-traffic.com -infektsii.com -infostatsvc.com -inmate-locator.us -inome.com.ua -instakink.com -in-tandem.co -intellektmedia.at -interfucks.net -intervsem.ru -investzalog.ru +indo-export.ru +infowarcraft.ru +innoslicon.com +insomniagamingfestival.com +instabid.tech +interesnie-faktu.ru +int.search.mywebsearch.com +invest-pamm.ru +investyb.com invitefashion.com -invivo.hu -iopeninghours.co.uk -iqoption.pro +inzn.ru +io9.com +iqoption-bin.com +iqs.biz.ua iqupdatetmz.win -islandminingsupply.wordpress.com -isotoner.com -istizanidineacontrolledsubstance.blogspot.com -itch.io -itronics.ca -itservicesthatworkforyou.com +ivanovo.zrus.org ivanstroi.ru -ivoiretechnocom.ci -iweblist.info izhevsk.zrus.org -j33x.com -jabimgo.pw -javrip.net -jennyfire.ru -jerseywholesalechinabiz.com -jerseywholesaleelitestore.com -joingames.org -jurajskie.info -justprofit.xyz -kaliningrad.zrus.org -kamagragelusa.net -kamorel.com -karaganda.xkaz.org -karpun-iris.ru -karting196.ru -kashubadesign.ru +izoll.ru +jackwolfskinoutlet.online +javitas.info +javlibrary.cc +javstock.com +jerseyswholesalechinalimited.com +jestr.org +jetsli.de +jillepille.com +johnrobertsoninc.com +jongose.ninja +juliaworld.net +jumptap.com +kadashihotel.com +kakablog.net +kam-dom.ru +kamen-e.ru +katjimej.blog.fc2.com katushka.net -kaz.kz -keenoutlet.online -kevblog.top -keywordsuggests.com -kidskunst.info -kiinomaniak.pl -kinocccp.net -kino-filmi.com -kino-rating.ru -kinosed.net -kiwi237au.tk -kletkimehan.ru +kchaxton.com +kenaba.su +ketrzyn.pl +keywordhut.com +keyword-suggestions.com +khafre.us +kino-fun.ru +kinogonew.ru +kinoplen.ru +kinostorm.net +kinotorka.ru +kiskinhouse.com +kiwe-analytics.com +kladrus.ru klin3952.ru -knowsitall.info -kochanelli.com -koleso24.com.ua -kollesa.ru -kolotiloff.ru -konkursowo-24.pl -konoplisemena.com -kopihijau.info +kostenloser-sex.com kosynka-games.ru -kotaku.com -koversite.info +kozhniebolezni.com krafte.ru -krasivoe-hd.net -krasnodar-avtolombards.ru -krasnodar.zrus.org -ktotut.net -kurwa.win -lacapilla.info -ladov.ru -law-check-eight.xyz -law-check-seven.xyz -lawyers.cafe -laxdrills.com -leadn.pl -legalrc.biz +krasula.pp.ua +krynica.info +ku6.com +labafydjxa.su +lacasamorett.com +laparfumotec.com +laulini.soclog.se +ldrtrack.com +lechenie-gemorroya.com +lernur.net +lerporn.info lesbian.xyz -letsart.ru letsrepair.in -libertybilisim.com +levaquin750mg.blogspot.com lifehacĸer.com likesuccess.com -lineshops.biz -linkbux.ru -linkdebrideur.xyz -linkrr.com -lisque.batcave.net -littleberry.ru -liumimgx.pw -liverpool.gsofootball.com -lomb.co -lomza.info -lost-alpha.ru -louboutinreplica.pw -louisvuittonoutletstore.net +link.ac +lion.conboy.us +lion.playtap.us +liveinternet.ru +localflirtbuddies.com +locatellicorretor.com.br +lombardfinder.ru +lonely-mature.com +longadventure.com +louboutinreplica.xyz lowephotos.info -lutherstable.org -m3gadownload.pl -mactechinfo.info -madot.onlinewebshop.net -mafcards.ru -magicalfind-a.akamaihd.net -mahnwachen-helfen.info -marcoislandvacations.net -marketland.ml -marwer.info -maslenka.kz -master-muznachas.ru -matb3aa.com -maxthon.com -mazda-roadsters.com -mebelcomplekt.ru +lsex.xyz +luciddiagnostics.in +luckyday.world +lunamedia.co +lyrster.com +m4ever.net +mabdoola.blogspot.com +machicon-akihabara.info +mac-shield.com +makedo.ru +managerpak204.weebly.com +mastershef.club +masthopehomes.com +maximpartnerspr.com mediaoffers.click +medicinacom.ru +medicines-choice.com +medizinreisen.de +medkletki.ru medkritika.ru -medosmotr-ufa.ru -meds-online24.com -med-zdorovie.com.ua -menhealed.net -metabar.ru -metallosajding.ru -metarip.ru -mexicosleevegastrectomy.com -milblueprint.com -minecraft-neo.ru -mini.7zap.com -miniads.ca -minneapoliscopiers.com -mirtorrent.net -missclub.info -mjchamonix.org -mlvc4zzw.space -mmostrike.ru -mncrftpcs.com -mnogabukaff.net -mobot.site -mobplayer.ru +medtherapy.ru +meduza-consult.ru +meetlocalchicks.com +meget.co.za +metascephe.com +mezaruk.info +micasainvest.com +microsearch.ru +microstatic.pl +mielec.pl +migente.com +mikrobiologies.ru +militarysale.pro +mir-betting.ru +mirobuvi.com.ua +misswell.net +mixed-wrestling.ru +mmoguider.ru +mobioffertrck.com modenamebel.ru -mojaocena.com -monclerboots.xyz -monclercoats.xyz +moesen-ficken.com +moi-glazki.ru +moje-recenze.cz +monarhs.info +monclerjacketsoutlet.pw moncleronline.xyz -moneyviking-a.akamaihd.net -moneyzzz.ru -monsterdivx.tv -montazhnic.ru +morefastermac.trade +morlat.altervista.org +morocco-nomad-excursions.com moscow-clining.ru -mountainstream.ms -mouselink.co -moviezbonkerssk.cf -mowser.com -mp3films.ru -mrbojikobi4.biz -mrmoneymustache.com -msk-diplomat.com -mtmtv.info -mulberryoutletonlineeu.com -mundoaberrante.com -musicas.baixar-musicas-gratis.com -mustwineblog.com -muz-shoes.ru -mvpicton.co.uk -my-bc.ru -mycaf.it +most.gov.iq +mturkcontent.com +muizre.ru mycouponizemac.com -myftpupload.com -myhydros.org -myindospace.com -mylesosibirsk.ru -myshopmatemac.com +myfreecams.com +mykings.pw +mylida.org +myplaycity.com +nagdak.ru nailsimg.com -nalogovyy-kodeks.ru naperehresti.info -narco24.me -narutonaruto.ru -naturtreenspicerx.pw -navalwiki.info -nerudlogistik.ru -netallergy.ru -netfacet.net -nevansk.ru -newtechspb.ru -nextlnk12.com -nhl17coinsforps3.gratisblog.biz -nibbler.silktide.com -nodding-passion.tk -no-rx.info -notaria-desalas.com -november-lax.com +nardulan.com +nationalbreakdown.com +netcheckcdn.xyz +net-profits.xyz +new7ob.com +nfljerseyscheapchinabiz.com +nfljerseys.online +nikkiewart.ru +norththeface.store +notasprensa.info +nottyu.xyz +novodigs.com novosibirsk.xrus.org -nuit-artisanale.com -nyfinance.ml -obnal.org -offer.camp -offers.bycontext.com -offer.wpsecurity.website -office2web.com -officedocuments.net -olvanto.ru +nrjmobile.fr +nsatc.net +obesidadealgarve.com +oecnhs.info +oil-td.ru +okayimage.com +okonich.com.ua +ok.ru +olgacvetmet.com +onclickpredictiv.com +onemactrckr.com onescreen.cc -onetravelguides.com -onlainbesplatno.ru -onlinadverts.com -onlinebay.ru -online-hit.info +online247.ml +onlinedomains.ru +onlinefilmz.net +onlinemegax.com +onlineserialy.ru online-templatestore.com -onlinewritingjobs17.blogspot.ru -ooomeru.ru -ooo-olni.ru -openstat.com +optikremont.ru +optom-deshevo.ru +ordernorxx.com +orenburg-gsm.ru orhonit.com -ourville.info -ownshop.cf -oxford-book.com.ua +ororodnik.goodbb.ru +otbelivanie-zubov.com +ourtherapy.ru +ownshop.win oyster-green.com +paceform.com page2rss.com -panchro.xyz -pandarastore.top -panicatack.com -parajumpersoutlet.online -partnerads.men -pavlodar.xkaz.org -pc-virus-d0l92j2.pw -pdamods.ru +panchro.co.uk +pandroid.co +par-fallen.ga +pateaswing.com +patterntrader-en.com +paydayloanslocal.com +pc4download.co +pc-services.ru pechikamini.ru -pewit.pw -pflexads.com -phobia.us +pensplan.com +pepperstyle.ru +perso.wanadoo.es +petitions.whitehouse.gov +phormchina.com php-market.ru -phuketscreen.com -picturesfrom.com -pinstake.com -pinwallpaper.top -pinwallpaper.xyz -pipki.r.acdnpro.com -pisanieprac.info -piulatte.cz -pix-hd.com +pictures-and-images.com +picturesmania.com +piluli.info +piski.top +pixell.club pk-pomosch.ru -plastgran.com -play.leadzupc.com -playlott.com -pl-top.pl -poppen-nw.net -popugauka.ru -popup-hgd.xyz -porndroids.com -pornofeuer.com +pkr1hand.com +plastgranar.nu +playfortuna-play.ru +play-movie.pl +playtap.us +plusnetwork.com +poem-paying.gq +poems.com.ua +polimga.pw +polybuild.ru +popcash.net +popunder.ru +porn555.com +porngalleries.top +pornhive.org +pornhub-forum.ga pornofiljmi.com -pornogratisdiario.com -pornokajf.com -pornonik.com -pornosemki.info -portadd.men -pp-budpostach.com.ua -pr0fit-b0x.com -pravoholding.ru -presleycollectibles.com -preventheadacheguide.info -privatamateure.com -privat-girl.net -privetsochi.ru -prizefestival.mobi -produkto.net -prodvigator.ua -progonrumarket.ru +pornogig.com +pornophoto.xyz +pornosexrolik.com +pornotubexxx.name +pozdrawleniya.com +pregnant.guru +priceg.com +pricheskaonline.ru +printingpeach.com +privatefx.all4invest.info +proekt-mos.ru +profitwithalex.info progress-upakovka.ru -propranolol40mg.blogspot.com -prostitutki-odessa.org -pst2017.onlinewebshop.net +projectforte.ru +promgirldresses.xyz +proprostatit.com +prostitutki-astana.org +prostitutki-kharkova.org +proxyradar.com pukaporn.com -pulse33.ru -pulseonclick.com +pussyfleet.com putevka24.ru -puzzleweb.ru pwwysydh.com -pxhdwsm.com +pyrodesigns.com.au +qexyfu.bugs3.com q-moto.ru -qpypcx.com +qualitymarketzone.com +queerspace.com quickbuck.com -quick-offer.com +quickloanbank.com +quit-smoking.ga +qwarckoine.com radiogambling.com -raisedseo.com rangjued.com rangoman.date -rankia.com -ranksonic.org -ratemodels.net -razamicroelectronics.com +realitykings.com +rebuildermedical.com +reckonstat.info redirlock.com -r-e-f-e-r-e-r.com +regdefense.com registry-clean-up.net -registrydomainservices.com -reimageplus.com -relayblog.com +reining.lovasszovetseg.hu +reklama1.ru +remont-komputerov-notebook.ru remont-mobile-phones.ru -remont-ustanovka-tehniki.ru renecaovilla.online -replicaclub.ru -research.ifmo.ru -responsinator.com -revistaindustria.com -richinvestmonitor.com -ric.info -riders.ro +renecaovillasale.online +rentehno.ru +respublica-otel.ru +retreatia.com +rialp.getenjoyment.net +rightenergysolutions.com.au +rixpix.ru +rock-cafe.info rockma.se -royal-betting.net -royal-investments.net -rp9.ru -rubbed.us -running-line.ru -rusexy.xyz -rus-pornuha.com -russian-postindex.ru -rustic-quiver.win -s1z.ru -s8-nowy-wygraj.comli.com -sabaapress.com -saecsa.co +rootandroid.org +royalcar-ufa.ru +rozalli.com +safe-app.net sajatvelemeny.com -sammyweaver.com -samolet.fr -sanjosestartups.com sape.top -satellite.maps.ilovevitaly.com -saveindex.xyz -sbornik-zakonov.ru -scanner-fred.top -scanner-jane.top -scanner-jessica.top -scanner-john.top -scanner-olga.top -schoolfiles.net -scrnet.biz.ua -seansonline24.pl -search-error.com -search-goo.com -searchtooknow-a.akamaihd.net -seasonvar.ru -se.bnt-team.com -seemoreresultshu-a.akamaihd.net -segol.tv -seksotur.ru -seoholding.com +sardinie.us +sarf3omlat.com +sax-sex.com +sayyoethe.blogspot.co.za +sbtdesign.co.uk +scanner-irvin.top +scanner-josh.top +scanner-viktor.top +scanner-walt.top +scanspyware.net +scenicmissouri.us +scopich.com +score-ads.men +scottbywater.com +searchinquire.com +see-your-website-here.com +seinterface.com +seminarykansas904.ml +semprofile.com +seocheki.net +seofirmreviewsus.info +seolab.top +seomarketings.online seorankinglinks.us -seo-traffic-ranking.info -service.adtech.fr +servingnotice.com serviporno.com +seryeznie-znakomstva.ru sexad.net sexcamamateurchat.com -sexflirtbook.com -sexpornotales.com -sexreliz.com +sex-dating.co +sexgalleries.top sexs-foto.top -sextracker.com -sex-tracker.de sextracker.de -sexvideo-sex.com -sexyali.com -sexyteens.hol.es sfd-chess.ru -sharebutton.to shiksabd.com -shinikiev.com.ua -shlyahten.ru -shopfishing.com.ua +shmetall.com.ua +shoesonlinebuy.xyz +shopcheermakeup.info shoppingjequiti.com.br shopvilleroyboch.com.ua -shtaketniki.kz -sibecoprom.ru -sideeffectsoftizanidine.blogspot.com -silktide.com -simpoed.ufop.br -site-analyzer.com -siteripz.net -slavic-magic.ru -sledstvie-veli.net -sluganarodu.ru -slutloadlive.com -smsactivator.ru -sneakyboy.com -snip.tw -snjatie-geroinovoy-lomki.ru -snomer1.ru -snowplanes.com -snworks.com -soaksoak.ru +signoredom.com +simon3.ru +simply.net +sims-sims.ru +sirpornogratis.xxx +sisiynas.ru +site5.com +site-speed-checker.site +site-speed-check.site +sitevaluation.com +sixcooler.de +skanninge.se +skylta.com +smartshoppymac.com +smokewithrabbits.com +smstraf.ru +snyatie-lomki-v-stacionare.ru soblaznu.net -social-buttons.com socialbuttons.xyz -social-button.xyz -softomix.net -softonicads.com -solinf.co -sophang8.com +social-search.me +softlinesolutions.me +solmarket.by +solution4u.com +sortthemesitesby.com +sosdepotdebilan.com spamnuker.com -spasswelt.net -spidtest.org -sportbetfair.com -sports-supplements.us +spb.afora.ru +spin2016.cf +spravka-medosmotr.ru +sprttrack.com +sptslmtrafms.com +spyfu.com spywarebegone.com -stardevine.com -stpolice.com -sugvant.ru -sukarame.net -suralink.com -sureone.pro -svtrd.com -swinginwithme.ru -tagil.zrus.org +spywarespy.com +ssconstruction.co +staff.prairiesouth.ca +startufa.ru +statustroll.com +sterva.cc +sticken.co +stretchmate.net +stroymonolit.su +students-cheapskate.ml +studiofaca.com +success-seo.com +superkanpo.com +supermesta.ru +supernew.org +support.nopeas.sk +surffoundation.nl +suture.co +svensk-poesi.com +svetodiodoff.ru +svolze.com +sweepstakes.rewardit.com +sygraem.com +symbaloo.com +taboola.com +tamada69.com tarad.com -taxi-v-eisk.ru -tds-advert002.info -telsis.com -tetracsaudi.com +techart24.com +tecspb.ru +tedy.su +telefonsexi.com +telegramdownload10.com +templates.radiodigital.co +tfxiq.com +thaisamkok.com thebitcoincode.com -thecoolimages.net -thecounter.com -thedownloadfromwarez.blogspot.com -thefarmergame.com +thefds.net +thefotosgratis.eu thegalerie.eu -thenetinfo.com -thesmartsearch.net -thiegs.reco.ws -thin.me.pn -tilido.com -titslove.yopoint.in -tizanidine4mgprice.blogspot.com -tizanidine4mgstreetprice.blogspot.com -tizanidineandcipro.blogspot.com -tizanidineandhydrocodone.blogspot.com -tizanidinecapsules.blogspot.com -tizanidinedrugtest.blogspot.com -tizanidineduringpregnancy.blogspot.com -tizanidinehcl2mgtablet.blogspot.com +thepantonpractice.co.uk +thetoiletpaper.com +thewebsitetemplate.info +timeallnews.ru +tizanidineforopiatewithdrawal.blogspot.com +tizanidinehcl2mg.blogspot.com +tizanidinehcl2mgsideeffects.blogspot.com tizanidinehcl4mgtab.blogspot.com -tizanidinehclsideeffects.blogspot.com -tizanidineinteractions.blogspot.com -toloka.hurtom.com -top10registrycleaners.com -topappspro.com +tizanidineinfo.blogspot.com +tizanidineingredients.blogspot.com +tizanidinemusclerelaxant.blogspot.com +tizanidinewithdrawal.blogspot.com +tkanorganizma.ru +tksn.ru +tmm-kurs.ru +toolsky.com +top250movies.ru +topclickguru.com top-deal.com.pl -topkarkas.com top-karkas.ru -toproadrunner5.info -totu.us +torontoplumbinggroup.com toursmaps.com -tracxn.com -traderzplanet.in -trafficmonetize.org -traffictrade.life -trion.od.ua +toy-shop.top +trafficjunky.com +trafficreceiver.club +truck-land.life +tubeline.biz tuberkulezanet.ru -tubo360.com -tula.howotorg.ru -turist-strani.ru -twitlinks.com -uac.net.au +tuckermktg.com +tula.mdverey.ru +turbabitload.weebly.com +turizm.bz +turvgori.ru +twiclub.in +twodollarshows.com +twojebook.pl +txxx.com +tzritel.tk uamtrk.com -u-cheats.ru -ufa.xrus.org -ugogo.info -umityangin.net -unblocksit.es -unece.org -unimodemhalfduplefw.pen.io -unitygame3d.com +ucoz.ru +udsgame.online +ugguk.online +uhdtv.website +underthesite.com +unionmarkt.de +univerfiles.com +unpredictable.ga +upstore.me +uptime-delta.net +uptime-gamma.net +upupa.net urdoot.win -url2image.com -uroki.net -usbggettwku.ga -usdx.us -usiad.net +urlcut.ru +uroffer.link ussearche.cf -validccseller.com -vapsy.com +uvozdeckych.info +v720hd.ru +vadimkravtcov.ru +valkiria-tk.ru +varbergsvind.se varikozdok.ru -vedomstvo.net -veles.shop -vellings.info -venerologiya.com -vepad.com -vgoloveboli.net -via-energy-cumpara.com -via-energy-order.com -viagengrarx.com +veerotech.com +ventelnos.com +vesnatehno.com +vesnatehno.ru viagra-soft.ru -via-gra.webstarts.com -viandpet.com -viberdownload10.com -videochat.cafe -videogamesecrets.com -videosbox.ru +videochat.tv.br +video-hollywood.ru +videojam.tv videos-for-your-business.com -videotuber.ru +videospornogratisx.net video-woman.com -videtubs.pl -vielporno.net -vip-dom.in -virtuagirl.com -visa-china.ru -visionwell.com.cn -viteonlusarezzo.it -vizag.kharkov.ua +vietimgy.pw +vintontech.info +vipms.ru +vipsexfinders.com +visa-pasport.ru vladimir.xrus.org -volgograd.xrus.org -voluumtracker1.com -vostoktrade.info -vykup-avto-krasnodar.ru -vykupavto-krasnodar.ru +voloo.ru +voloslove.ru +volume-pills.biz +vut.com.ru +w3data.co +wait3sec.org +wallpapersdesk.info warningwar.ru -watchinf.com -waterefficiency.co waterpurifier.club -watsonrealtycorp.com -wdrake.com -webadvance.club +waysbetter.cn +webcamdevochka.com +webix.me +webpromotion.ae webscouter.net -website-analytics.online -wemedinc.com -westen-v.life -wetgames.ru +website-datenbank.de +website-speed-check.site +weburlopener.com +weddingdresses.xyz +wejdz-tu.pl wfb.hatedriveapart.com -wheelchairliftsarea.com -whereverdesperate.gq -while.cheapwebsitehoster.com -wholesalejerseyscheapjerseys.us.com +whatistizanidine2mg.blogspot.com +whengirlsgowild.com +wholesalejerseychinaoutlet.com +wholesalejerseychinashop.com +wildcattube.com +wildnatureimages.com wildworld.site -wma-x.com +win-spy.com +winterclassichockeyjerseys.com +winwotgold.pl +wnoz.de +womens-journal.net +wordpresscore.com +works.if.ua worldis.me worldlovers.ru -wstroika.ru -wttavern.com +wowas31.ucoz.ru +wtsindia.in +wufak.com www.888.com -wygraj-skiny.win -wyniki-lista.pl -x5market.ru -x69ty.ru -x-diesel.com -xfire.com -xgames-04.com -xjlottery.com -xlovecam.com +wzgyyq.com +x-diesel.info +xerox-douglas.cf +xfluro.com +xkaz.org +xlolitka.com +x-musics.com xn--1-8sbcpb0bdm8k6a.xn--p1ai -xn--80adaggc5bdhlfamsfdij4p7b.xn--p1ai -xn--80ajbshivpvn2i.xn--p1ai -xn--80ak6aa92e.com -xn----8sbdbjgb1ap7a9c4czbh.xn--p1acf +xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai +xn-----7kcabaipgeakzcss7bjdqdwpfnhv.xn--p1ai +xn----7sbahjd3btneuw1joc.xn--p1ai +xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks xn--b1ag5cfn.xn--p1ai -xn----itbkqkfiq.xn--p1ai -xn--q1a.xn--b1aube0e.xn--c1acygb.xn--p1ai -x-porno.video -xvideospornoru.com +xn--h1ahbi.com.ua +xtube.com +xxlargepop.com +xxxrus.org +xxxtubesafari.com xzlive.com -yaderenergy.ru +yachts-cruise.info yadro.ru -yaoguangdj.com -yellowstonesafaritours.com -youjizz.vc -youporn-ru.com +yellocloud.be +yellowfootprints.com +yes-com.com +yopoint.in +yoshkarola.zrus.org +youandcredit.ru +youbrainboost.asia youradulthosting.com -yourporn.com -yousense.info +yoursite.com +you-shall-not-pass.is74.ru +youtubedownload.org yugk.net -yunque.pluto.ro -yurgorod.ru -zacreditom.ru -zaimite.ru -zaim-pod-zalog-krasnodar.ru -zaobao.com.sg -zeg-distribution.com -zeleznobeton.ru +zaapplesales.blogspot.com +za-fun-offer.com +zaimhelp.ru +zakazfutbolki.com +zarenica.net +zatjmuzu.info +zebramart.ru +zero1.it zerocash.msk.ru -zeroredirect11.com -zhongwenlink.com -zigarettenonlinekaufen1.bloog.pl -znaniyapolza.ru -zog.link +zetmaster.ru +zkjovpdgxivg.ga +z-master.ru +zmoda.hostreo.com +zona-aqua.ru +zoogdinsney.com zoolubimets.ru +zoominfo.com +zoomovies.org zrizvtrnpale.tk +zynax.ua diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index 161fc95dc..de8040a13 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ 007angels.com -20pascals.nl -24-ak.ru -2ads.co.uk -7makemoneyonline.com -aasoldes.fr -academia-nsk.org -acunetix-referrer.com -admitad.com -adultfriendfinder.com -afmuseum.com -agreda.pluto.ro -ahrefs.com -alldezservice.kz -all-streaming-media.com -apibetweenlinesn-a.akamaihd.net -arate.ru -arvut.org -autobudpostach.club -autotop.com.ua -avtointeres.ru -awency.com -ayanaspa.com -bablonow.ru -balans.shahterworld.org -besthomemadepornsites.com -bif-ru.info -biznesluxe.ru -bloggerads.net -brainboosting.club -brakehawk.com -buchananshardware.com -bum.com.ru -buy-cheap-pills-order-online.com -cashkitten-a.akamaihd.net -cctva.tv -celejihad.info -cellfun.mobi -cheapcigarettesc.info -chelnytruck.ru -cherubinimobili.it -chinacheapelitejerseys.com -chlooe.com -ciceros.co -ci.ua -clknsee.com -club-lukojl.ru -colehaanoutlet.store -compliance-brian.top -concordexoticrentals.com -conocer-sanabria.com -cordstrap.cc -countercrazy.com -crd.clan.su -dealitright.click -dichvuvesinhhanoi.com -dlya-android.org -dnepr-avtospar.com.ua -docstoc.com -domainsphoto.com -domashneeruporno.com -donvito.unas.cz -dorratex.tn -download-wallpaper.net -dumpsmania24.com -elektrozigarette2.wordpressy.pl -erectile.bid -escortplius.com -esnm.ru -eyelike.com.ua -fable.in.ua +1adult.com +2015god.org +8lufu.com +98oi.ru +aa14ab57a3339c4064bd9ae6fad7495b5f.com +aa8b68101d388c446389283820863176e7.com +aac52006ec82a24e08b665f4db2b5013f7.com +aad1f4acb0a373420d9b0c4202d38d94fa.com +adidas.frwebs.fr +ads-seo.men +advancedmassagebysara.com +afftrack001.com +agardomains.com +agecheckadult.com +allsilver925.co.il +ampower.me +animebox.com.ua +anime.dougasouko.com +aquarium-pro.ru +arendatora.ru +audiofree.ru +aussie-prizes.com +autochoixspinelli.com +avirasecureserver.com +avon-severozapad.ru +babyguns.ru +baladur.ru +balla-boo.se +bannerspace.com +best-games.today +bio-japan.net +bioscorp.ru +bio.trade-jp.net +biteg.xyz +budilneg.xyz +buron.pw +canacopegdl.com +canrole.com +canzoni.ru +carabela.com.do +cartierbracelet.xyz +cartierlovestore.xyz +caulderwoodkennels.com +cheapestjerseys-wholesale.com +ciekawinki.pl +clickcash.com +clubfashionista.com +compliance-julianna.top +conboy.us +cool-wedding.net +cosmetic.donna7753191.ru +counter.bloke.com +creativehutindia.com +data-mining.tk +ddospanel.com +deanmoore.ie +directrev.com +djonwatch.ru +domainroam.win +domik-derevne.ru +dtnlyss.com +ecookna.com.ua +ed-shop01.ru +ekto.ee +eng-lyrics.com +escort-russian.com +eurosamodelki.ru +everflora.ru +evidencecleanergold.com +exct.net +extlabs.io +familienzahnaerzte.com +farmingworm.com fast-wordpress-start.com -fillmewithhappiness.com -flowersbazar.com -forum.tvmir.org -frameimage.org +fatmaelgarny.com +feeriaclub.ru +filmci.pro +firma-legion.ru +foxweber.com free-fbook-traffic.com -freemaintenancesysforpcandmac.top -free-traffic.xyz -geckoandfly.com -gg.zzyjxs.com +free-share-buttons.top +frequiry.com +fr.netlog.com +fyxabomiw.ru +g33.org +gateway.zscalerone.net +ghazel.ru girlporn.ru -gnuetella.com -go2album.com -goatse.ru -golden-praga.ru -googlsucks.com -gotwebsite1.com -gsasearchengineranker.top -hazardky.net -hd-filmy.net -hdimagegallery.net -hidemyass.com -hobbyhuren24.net -hollywoodactress.info -homemade.gq -hostcritique.com -hotchatdate.com -hotkeys.com -hoverboard360.at -iccornacircri.cf -igrovyeavtomaty777.ru -ilbe.club -immobiliaremassaro.com -irkutsk.zrus.org -iskalko.ru -isotoner.com -istizanidineanopiate.blogspot.com -jewelryandfiligree.com -johannesburgsingles.co.za -kam-dom.ru -kareliatobacco.ru -kazinogames.lv -keywordsdoctor.com -keywords-monitoring-success.com -khafre.us -kino2018.club -kinogonew.ru -krasivoe-hd.net -labvis.host.sk -laptop-4-less.com +gomusix.com +goodprotein.ru +goodwriterssales.com +gosreg.amchs.ru +grtyi.com +gungamesz.com +gz2.bbsoldes.fr +hannasolution.ru +hetmanship.xyz +hmywwogw.bloger.index.hr +homeart.space +homedecorpicture.us +huhn.altervista.org +ideawheel.com +iinstalll-fii1leis.jus0wil.pp.ua +imadedinner.net +intelhdgraphicsgtdrive6w.metroblog.com +interferencer.ru +iopeninghours.co.uk +jabimgo.pw +jackpotchances.com +javitas.info +jerseywholesalebizchina.com +jikoman.info +jpcycles.com +jus0wil.pp.ua +keywordhouse.com +kiinomaniak.pl +kino-key.info +konteiner24.com +koptims.tiu.ru +kraljeva-sutjeska.com +krasula.pp.ua +laserpen.club lecbter-relationships.ga -lmrauction.com -loginduepunti.it +ledis.top +life-instyle.com +live-sexchat.ru +liveu.infoteka.hu +loadingpages.me +locimge.pw lombardfinder.ru -louboutinreplica.pw -louboutinreplica.xyz -maju.bluesalt.co -marcoislandvacations.net -michaelkorsoutletstore.net -mindbox.co.za -minecraft-rus.org -missvietnam.org -mixx.com -mmostrike.ru -mobstarr.com -morlat.altervista.org -moscow-clining.ru -moscow.online-podarki.com -myonlinepayday.co -myxdate.info -nacap.ru -narutonaruto.ru -net-profits.xyz -normalegal.ru -of-ireland.info -ok-ua.info -online-sbank.ru -ontargetseo.us -oops-cinema.ru -pekori.to -pensplan4u.com -pensplan.com -photosaga.info -piccshare.com -pizda.lol -pl.aasoldes.fr -poptool.net +lovelycraftyhome.com +mail.allnews24.in +manipulyator-peterburg.ru +manve.info +maranbrinfo.com.br +marblestyle.ru +mcadamssupplyco.com +medispainstitute.com.au +meet-flirt-dating.com +minneapoliscopiers.com +mirtorrent.net +moneyzzz.ru +mosdverka.ru +myblogregistercm.tk +mypets.by +na-telefon.biz +njpalletremoval.com +nodup.ru +novosti-hi-tech.ru +oconto.ru +oktube.ru +o-o-6-o-o.ru +openmediasoft.com +orsonet.ru +otbelivanie-zubov.com +page2rss.com +paperwritingservice17.blogspot.ru +parlament.biz +pastaleads.com +pechikamini.ru +percin.biz.ly +picsforkeywordsuggestion.com +picturesboss.com +pizza-imperia.com +playlott.com +ploenjitmedia.azurewebsites.net +plohaya-kreditnaya-istoriya.ru +pl-top.pl +pontiacsolstice.info +popup-hgd.xyz +porno-play.net +positive2b.ru +preg.marketingvici.com +presleycollectibles.com pricheski-video.com priscilarodrigues.com.br -prodvigator.ua -prombudpostach.com.ua -promodj.com -q-moto.ru -quangcaons.com -raschtextil.com.ua -real-time-analytics.com -reining.lovasszovetseg.hu -remedyotc.com -replicalouboutin.xyz -ritlweb.com -ruscoininvest.company -sabizonline.com -saltspray.ru -sandhillsonline.com -sape.top -scanner-julianna.top -scanner-mark.top -sdsjweb.com -searchkut.com -seeingmeerkat.com +prohoster.info +pron.pro +proposal-engine.com +pro-tec.kz +pussysaga.com +qor360.com +rapevideosmovies.com +redbottomheels.xyz +redirectme.net +registrydomainservices.com +rfd-split.hr +robot-forex.biz +royalcar-ufa.ru +rusprostitute.com +russkoe-zdorovie.ru +sanjosestartups.com sei80.com -semalt.com -seoboxes.com -sex.hotblog.top -sexpornotales.com -sexvideo-sex.com -shell-pmr.ru -ship-marvel.co.ua -shitmovs.com -sildenafil-tadalafil.info -simple-share-buttons.com -s-iwantyou.com -skincrate.net -skyway24.ru -snowplanes.com -socialmadesimple.com -socialmonkee.com -start.myplaycity.com -stockspmb.info -strv.se -swingerseiten.com -takethatad.com -tdsing.ru -templates.franklinfire.co -thebluffs.com -thegamerznetwork.com -theguardlan.com -timecrimea.ru -tizanidinehclsideeffects.blogspot.com -tizanidineoral.blogspot.com -tizanidinevsvalium.blogspot.com -top-l2.com -tourismvictoria.com -tozup.com -tpu.ru -tracksz.co -turbabitload.weebly.com -twu.com.ua -tyumen.xrus.org -validccseller.com -vegascosmetics.ru -video-chat.love -viel.su +seinterface.com +serpstat.com +shop-electron.ru +signx.info +siteexpress.co.il +softomix.net +songplanet.ru +spacash.com +spb-plitka.ru +sprttrack.com +starwars.wikia.com +stop-gepatit.te.ua +studiofaca.com +sugarlyflex.pw +survival.betteroffers.review +svensk-poesi.com +svtrd.com +telefonsexsofort.tk +titangel-vietnam.com +torrnada.ru +toursmaps.com +trahvid.com +traxdom.ru +typer.one +u17795.netangels.ru +undergroundcityphoto.com +underthesite.com +univerfiles.com +uroffer.link +urzedowski.eu +varikozdok.ru +venta-prom.ru +vereo.eu +video-production.com +videotuber.ru vip.51.la -visionwell.com.cn vkonche.com -voloomoney.com -voprosotvet24.ru -wemedinc.com -whatsappdownload10.com -wildcattube.com -wladimirpayen.com -womama.ru -woman-h.ru -womensplay.net -wosik-dach.service-for-web.de -wpsecurity.website -xgftnlrt.bloger.index.hr -xn--80ahdheogk5l.xn--p1ai -xuki.us -yellowads.men -yginekologa.com -your-bearings.com -zakonobosago.ru -zerocash.msk.ru -zigzog.ru -zoomovies.org -zrizvtrnpale.tk -zx6.ru +vote-up.ru +w7s.ru +wait3sec.org +webads.co.nz +website-analytics.online +where-toget.com +wholinkstome.com +wiosenny-bon-1500.pl +wstroika.ru +www.event-tracking.com +www.kabbalah-red-bracelets.com +xingzi-vision.com +x-lime.net +xn----7sbaphztdjeboffeiof6c.xn--p1ai +xn--oogle-wmc.com +zahvat.ru +zaim-pod-zalog-krasnodar.ru +zarepta.com +zeg-distribution.com diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index ca06dc6a4..6fc620641 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index 56ad07748..28104f0df 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index c4feadc50..18941791a 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2019.06.1597 +#### Version: V4.2019.06.1598 #### Bad Referrer Count: 6711 #### Bad Bot Count: 556 ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 0284669b8..0e04406e0 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2019.06.1597 +#### Version: V4.2019.06.1598 #### Bad Referrer Count: 6711 #### Bad Bot Count: 556 ____________________ diff --git a/README.md b/README.md index c8b246f0c..84ca7f507 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2019.06.1597 +#### Version: V4.2019.06.1598 #### Bad Referrer Count: 6711 #### Bad Bot Count: 556 ____________________ diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 444b16987..563ddd60f 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2019.06.1597 +#### Version: V4.2019.06.1598 #### Bad Referrer Count: 6711 #### Bad Bot Count: 556 ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 9aef5ccdc..d356b01cb 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1597 -### Updated: Tue Jun 25 15:25:42 SAST 2019 +### Version: V4.2019.06.1598 +### Updated: Tue Jun 25 15:42:43 SAST 2019 ### Bad Referrer Count: 6711 ### Bad Bot Count: 556 ################################################### @@ -693,38 +693,38 @@ map $http_user_agent $bad_bot { # *********************************************** # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*\badidxbot\b" 0; - "~*\bAdsBot-Google\b" 0; - "~*\baolbuild\b" 0; - "~*\bbingbot\b" 0; - "~*\bbingpreview\b" 0; - "~*\bDoCoMo\b" 0; - "~*\bduckduckgo\b" 0; - "~*\bfacebookexternalhit\b" 0; - "~*\bFeedfetcher-Google\b" 0; - "~*\bGooglebot\b" 0; - "~*\bGooglebot-Image\b" 0; - "~*\bGooglebot-Mobile\b" 0; - "~*\bGooglebot-News\b" 0; - "~*\bGooglebot/Test\b" 0; - "~*\bGooglebot-Video\b" 0; - "~*\bGoogle-HTTP-Java-Client\b" 0; - "~*\bGravityscan\b" 0; - "~*\bgsa-crawler\b" 0; - "~*\bJakarta\ Commons\b" 0; - "~*\bKraken/0.1\b" 0; - "~*\bLinkedInBot\b" 0; - "~*\bMediapartners-Google\b" 0; - "~*\bmsnbot\b" 0; - "~*\bmsnbot-media\b" 0; - "~*\bSAMSUNG\b" 0; - "~*\bSlackbot\b" 0; - "~*\bSlackbot-LinkExpanding\b" 0; - "~*\bslurp\b" 0; - "~*\bteoma\b" 0; - "~*\bTwitterBot\b" 0; - "~*\bWordpress\b" 0; - "~*\byahoo\b" 0; + "~*(?:\b)adidxbot(?:\b|)" 0; + "~*(?:\b)AdsBot-Google(?:\b|)" 0; + "~*(?:\b)aolbuild(?:\b|)" 0; + "~*(?:\b)bingbot(?:\b|)" 0; + "~*(?:\b)bingpreview(?:\b|)" 0; + "~*(?:\b)DoCoMo(?:\b|)" 0; + "~*(?:\b)duckduckgo(?:\b|)" 0; + "~*(?:\b)facebookexternalhit(?:\b|)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; + "~*(?:\b)Googlebot(?:\b|)" 0; + "~*(?:\b)Googlebot-Image(?:\b|)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; + "~*(?:\b)Googlebot-News(?:\b|)" 0; + "~*(?:\b)Googlebot/Test(?:\b|)" 0; + "~*(?:\b)Googlebot-Video(?:\b|)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; + "~*(?:\b)Gravityscan(?:\b|)" 0; + "~*(?:\b)gsa-crawler(?:\b|)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; + "~*(?:\b)Kraken/0.1(?:\b|)" 0; + "~*(?:\b)LinkedInBot(?:\b|)" 0; + "~*(?:\b)Mediapartners-Google(?:\b|)" 0; + "~*(?:\b)msnbot(?:\b|)" 0; + "~*(?:\b)msnbot-media(?:\b|)" 0; + "~*(?:\b)SAMSUNG(?:\b|)" 0; + "~*(?:\b)Slackbot(?:\b|)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; + "~*(?:\b)slurp(?:\b|)" 0; + "~*(?:\b)teoma(?:\b|)" 0; + "~*(?:\b)TwitterBot(?:\b|)" 0; + "~*(?:\b)Wordpress(?:\b|)" 0; + "~*(?:\b)yahoo(?:\b|)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # *************************************************** @@ -735,12 +735,12 @@ map $http_user_agent $bad_bot { # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*\bjetmon\b" 1; - "~*\blibwww-perl\b" 1; - "~*\bLynx\b" 1; - "~*\bmunin\b" 1; - "~*\bPresto\b" 1; - "~*\bWget/1.15\b" 1; + "~*(?:\b)jetmon(?:\b|)" 1; + "~*(?:\b)libwww-perl(?:\b|)" 1; + "~*(?:\b)Lynx(?:\b|)" 1; + "~*(?:\b)munin(?:\b|)" 1; + "~*(?:\b)Presto(?:\b|)" 1; + "~*(?:\b)Wget/1.15(?:\b|)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************************** @@ -748,23 +748,23 @@ map $http_user_agent $bad_bot { # ************************************************************** # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*\bAlexa\b" 2; - "~*\barchive.org\b" 2; - "~*\bBaidu\b" 2; - "~*\bBUbiNG\b" 2; - "~*\bFlipboardProxy\b" 2; - "~*\bia_archiver\b" 2; - "~*\bMSIE\ 7.0\b" 2; - "~*\bProximic\b" 2; - "~*\bR6_CommentReader\b" 2; - "~*\bR6_FeedFetcher\b" 2; - "~*\bRED/1\b" 2; - "~*\bRPT-HTTPClient\b" 2; - "~*\bsfFeedReader/0.9\b" 2; - "~*\bSpaidu\b" 2; - "~*\bUptimeRobot/2.0\b" 2; - "~*\bYandexBot\b" 2; - "~*\bYandexImages\b" 2; + "~*(?:\b)Alexa(?:\b|)" 2; + "~*(?:\b)archive.org(?:\b|)" 2; + "~*(?:\b)Baidu(?:\b|)" 2; + "~*(?:\b)BUbiNG(?:\b|)" 2; + "~*(?:\b)FlipboardProxy(?:\b|)" 2; + "~*(?:\b)ia_archiver(?:\b|)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; + "~*(?:\b)Proximic(?:\b|)" 2; + "~*(?:\b)R6_CommentReader(?:\b|)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; + "~*(?:\b)RED/1(?:\b|)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; + "~*(?:\b)Spaidu(?:\b|)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; + "~*(?:\b)YandexBot(?:\b|)" 2; + "~*(?:\b)YandexImages(?:\b|)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index cebe1bfdc..8e0f74838 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -10,8 +10,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1597 -### Updated: Tue Jun 25 15:25:43 SAST 2019 +### Version: V3.2019.06.1598 +### Updated: Tue Jun 25 15:42:44 SAST 2019 ### Bad Referrer Count: 6711 ### Bad Bot Count: 556 ################################################### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 9aef5ccdc..d356b01cb 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1597 -### Updated: Tue Jun 25 15:25:42 SAST 2019 +### Version: V4.2019.06.1598 +### Updated: Tue Jun 25 15:42:43 SAST 2019 ### Bad Referrer Count: 6711 ### Bad Bot Count: 556 ################################################### @@ -693,38 +693,38 @@ map $http_user_agent $bad_bot { # *********************************************** # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*\badidxbot\b" 0; - "~*\bAdsBot-Google\b" 0; - "~*\baolbuild\b" 0; - "~*\bbingbot\b" 0; - "~*\bbingpreview\b" 0; - "~*\bDoCoMo\b" 0; - "~*\bduckduckgo\b" 0; - "~*\bfacebookexternalhit\b" 0; - "~*\bFeedfetcher-Google\b" 0; - "~*\bGooglebot\b" 0; - "~*\bGooglebot-Image\b" 0; - "~*\bGooglebot-Mobile\b" 0; - "~*\bGooglebot-News\b" 0; - "~*\bGooglebot/Test\b" 0; - "~*\bGooglebot-Video\b" 0; - "~*\bGoogle-HTTP-Java-Client\b" 0; - "~*\bGravityscan\b" 0; - "~*\bgsa-crawler\b" 0; - "~*\bJakarta\ Commons\b" 0; - "~*\bKraken/0.1\b" 0; - "~*\bLinkedInBot\b" 0; - "~*\bMediapartners-Google\b" 0; - "~*\bmsnbot\b" 0; - "~*\bmsnbot-media\b" 0; - "~*\bSAMSUNG\b" 0; - "~*\bSlackbot\b" 0; - "~*\bSlackbot-LinkExpanding\b" 0; - "~*\bslurp\b" 0; - "~*\bteoma\b" 0; - "~*\bTwitterBot\b" 0; - "~*\bWordpress\b" 0; - "~*\byahoo\b" 0; + "~*(?:\b)adidxbot(?:\b|)" 0; + "~*(?:\b)AdsBot-Google(?:\b|)" 0; + "~*(?:\b)aolbuild(?:\b|)" 0; + "~*(?:\b)bingbot(?:\b|)" 0; + "~*(?:\b)bingpreview(?:\b|)" 0; + "~*(?:\b)DoCoMo(?:\b|)" 0; + "~*(?:\b)duckduckgo(?:\b|)" 0; + "~*(?:\b)facebookexternalhit(?:\b|)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; + "~*(?:\b)Googlebot(?:\b|)" 0; + "~*(?:\b)Googlebot-Image(?:\b|)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; + "~*(?:\b)Googlebot-News(?:\b|)" 0; + "~*(?:\b)Googlebot/Test(?:\b|)" 0; + "~*(?:\b)Googlebot-Video(?:\b|)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; + "~*(?:\b)Gravityscan(?:\b|)" 0; + "~*(?:\b)gsa-crawler(?:\b|)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; + "~*(?:\b)Kraken/0.1(?:\b|)" 0; + "~*(?:\b)LinkedInBot(?:\b|)" 0; + "~*(?:\b)Mediapartners-Google(?:\b|)" 0; + "~*(?:\b)msnbot(?:\b|)" 0; + "~*(?:\b)msnbot-media(?:\b|)" 0; + "~*(?:\b)SAMSUNG(?:\b|)" 0; + "~*(?:\b)Slackbot(?:\b|)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; + "~*(?:\b)slurp(?:\b|)" 0; + "~*(?:\b)teoma(?:\b|)" 0; + "~*(?:\b)TwitterBot(?:\b|)" 0; + "~*(?:\b)Wordpress(?:\b|)" 0; + "~*(?:\b)yahoo(?:\b|)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # *************************************************** @@ -735,12 +735,12 @@ map $http_user_agent $bad_bot { # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*\bjetmon\b" 1; - "~*\blibwww-perl\b" 1; - "~*\bLynx\b" 1; - "~*\bmunin\b" 1; - "~*\bPresto\b" 1; - "~*\bWget/1.15\b" 1; + "~*(?:\b)jetmon(?:\b|)" 1; + "~*(?:\b)libwww-perl(?:\b|)" 1; + "~*(?:\b)Lynx(?:\b|)" 1; + "~*(?:\b)munin(?:\b|)" 1; + "~*(?:\b)Presto(?:\b|)" 1; + "~*(?:\b)Wget/1.15(?:\b|)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************************** @@ -748,23 +748,23 @@ map $http_user_agent $bad_bot { # ************************************************************** # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*\bAlexa\b" 2; - "~*\barchive.org\b" 2; - "~*\bBaidu\b" 2; - "~*\bBUbiNG\b" 2; - "~*\bFlipboardProxy\b" 2; - "~*\bia_archiver\b" 2; - "~*\bMSIE\ 7.0\b" 2; - "~*\bProximic\b" 2; - "~*\bR6_CommentReader\b" 2; - "~*\bR6_FeedFetcher\b" 2; - "~*\bRED/1\b" 2; - "~*\bRPT-HTTPClient\b" 2; - "~*\bsfFeedReader/0.9\b" 2; - "~*\bSpaidu\b" 2; - "~*\bUptimeRobot/2.0\b" 2; - "~*\bYandexBot\b" 2; - "~*\bYandexImages\b" 2; + "~*(?:\b)Alexa(?:\b|)" 2; + "~*(?:\b)archive.org(?:\b|)" 2; + "~*(?:\b)Baidu(?:\b|)" 2; + "~*(?:\b)BUbiNG(?:\b|)" 2; + "~*(?:\b)FlipboardProxy(?:\b|)" 2; + "~*(?:\b)ia_archiver(?:\b|)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; + "~*(?:\b)Proximic(?:\b|)" 2; + "~*(?:\b)R6_CommentReader(?:\b|)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; + "~*(?:\b)RED/1(?:\b|)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; + "~*(?:\b)Spaidu(?:\b|)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; + "~*(?:\b)YandexBot(?:\b|)" 2; + "~*(?:\b)YandexImages(?:\b|)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 57715a733..8541682e0 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,8 +6,8 @@ ### Version Information # ################################################### -### Version: V4.2019.06.1597 -### Updated: Tue Jun 25 15:25:44 SAST 2019 +### Version: V4.2019.06.1598 +### Updated: Tue Jun 25 15:42:45 SAST 2019 ### Bad Bot Count: 556 ################################################### ### Version Information ##