diff --git a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf index ac0f7c816..c9e3281f2 100644 --- a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf @@ -26,4 +26,4 @@ # Only add actual IP addresses and ranges here 111.111.111.111 0; -35.202.124.16 0; +35.202.33.153 0; diff --git a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf index de14c35d5..875cb9d35 100644 --- a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.10.979 -### Updated: Thu Oct 26 14:47:07 SAST 2017 +### Version: V3.2017.11.980 +### Updated: Wed Nov 1 12:59:37 SAST 2017 ### Bad Referrer Count: 5904 -### Bad Bot Count: 513 +### Bad Bot Count: 496 ################################################### ### VERSION INFORMATION ## @@ -259,7 +259,6 @@ map $http_user_agent $bad_bot { "~*\bBolt\b" 3; "~*\bBotALot\b" 3; "~*\bBrandprotect\b" 3; - "~*\bBrandProtect\b" 3; "~*\bBrandwatch\b" 3; "~*\bBubing\b" 3; "~*\bBuddy\b" 3; @@ -289,7 +288,6 @@ map $http_user_agent $bad_bot { "~*\bCosmos\b" 3; "~*\bCraftbot\b" 3; "~*\bCrazyWebCrawler\b" 3; - "~*\bCRAZYWEBCRAWLER\b" 3; "~*\bCrescent\b" 3; "~*\bCSHttp\b" 3; "~*\bCurious\b" 3; @@ -313,8 +311,6 @@ map $http_user_agent $bad_bot { "~*\bDomainSigmaCrawler\b" 3; "~*\bDomainStatsBot\b" 3; "~*\bDotbot\b" 3; - "~*\bDownload\ Demon\b" 3; - "~*\bDownload\ Devil\b" 3; "~*\bDownload\ Wonder\b" 3; "~*\bDragonfly\b" 3; "~*\bDrip\b" 3; @@ -325,8 +321,6 @@ map $http_user_agent $bad_bot { "~*\bECCP/1.0\b" 3; "~*\bEcxi\b" 3; "~*\bEirGrabber\b" 3; - "~*\bEMail\ Collector\b" 3; - "~*\bEMail\ Extractor\b" 3; "~*\bEMail\ Siphon\b" 3; "~*\bEMail\ Wolf\b" 3; "~*\bEroCrawler\b" 3; @@ -386,7 +380,6 @@ map $http_user_agent $bad_bot { "~*\bId-search\b" 3; "~*\bIlseBot\b" 3; "~*\bImage\ Fetch\b" 3; - "~*\bImage\ Stripper\b" 3; "~*\bImage\ Sucker\b" 3; "~*\bIndy\ Library\b" 3; "~*\bInfoNaviRobot\b" 3; @@ -507,7 +500,6 @@ map $http_user_agent $bad_bot { "~*\bPageAnalyzer\b" 3; "~*\bPage\ Analyzer\b" 3; "~*\bPageGrabber\b" 3; - "~*\bPage\ Grabber\b" 3; "~*\bpage\ scorer\b" 3; "~*\bPageScorer\b" 3; "~*\bPanscient\b" 3; @@ -558,14 +550,12 @@ map $http_user_agent $bad_bot { "~*\bScanbot\b" 3; "~*\bScrapy\b" 3; "~*\bScreaming\b" 3; - "~*\bScreaming\ Frog\ SEO\ Spider\b" 3; "~*\bScreenerBot\b" 3; "~*\bSearchestate\b" 3; "~*\bSearchmetricsBot\b" 3; "~*\bSemrush\b" 3; "~*\bSemrushBot\b" 3; "~*\bSEOkicks\b" 3; - "~*\bSEOkicks-Robot\b" 3; "~*\bSEOlyticsCrawler\b" 3; "~*\bSeomoz\b" 3; "~*\bSEOprofiler\b" 3; @@ -573,7 +563,6 @@ map $http_user_agent $bad_bot { "~*\bSEOstats\b" 3; "~*\bSiphon\b" 3; "~*\bSISTRIX\b" 3; - "~*\bSISTRIX\ Crawler\b" 3; "~*\bSitebeam\b" 3; "~*\bSiteExplorer\b" 3; "~*\bSiteimprove\b" 3; @@ -628,12 +617,14 @@ map $http_user_agent $bad_bot { "~*\bTitan\b" 3; "~*\bToata\b" 3; "~*\bToweyabot\b" 3; + "~*\bTrendiction\b" 3; "~*\bTrendictionbot\b" 3; + "~*\btrendiction.com\b" 3; + "~*\btrendiction.de\b" 3; "~*\bTrue_Robot\b" 3; "~*\bTuringos\b" 3; + "~*\bTurnitin\b" 3; "~*\bTurnitinBot\b" 3; - "~*\bTurnitin\ Bot\b" 3; - "~*\bTurnitin\ Robot\b" 3; "~*\bTwengaBot\b" 3; "~*\bTwice\b" 3; "~*\bTyphoeus\b" 3; @@ -658,13 +649,10 @@ map $http_user_agent $bad_bot { "~*\bWebAuto\b" 3; "~*\bWeb\ Auto\b" 3; "~*\bWebBandit\b" 3; - "~*\bWeb\ Bandit\b" 3; "~*\bWebCollage\b" 3; "~*\bWeb\ Collage\b" 3; "~*\bWebCopier\b" 3; - "~*\bWeb\ Copier\b" 3; "~*\bWEBDAV\b" 3; - "~*\bWEBDAV\ Client\b" 3; "~*\bWebEnhancer\b" 3; "~*\bWeb\ Enhancer\b" 3; "~*\bWebFetch\b" 3; @@ -673,28 +661,23 @@ map $http_user_agent $bad_bot { "~*\bWeb\ Fuck\b" 3; "~*\bWebGo\ IS\b" 3; "~*\bWebImageCollector\b" 3; - "~*\bWeb\ Image\ Collector\b" 3; "~*\bWebLeacher\b" 3; "~*\bWebmasterWorldForumBot\b" 3; "~*\bwebmeup-crawler\b" 3; "~*\bWebPix\b" 3; "~*\bWeb\ Pix\b" 3; "~*\bWebReaper\b" 3; - "~*\bWeb\ Reaper\b" 3; "~*\bWebSauger\b" 3; "~*\bWeb\ Sauger\b" 3; "~*\bWebshag\b" 3; "~*\bWebsiteExtractor\b" 3; - "~*\bWebsite\ Extractor\b" 3; "~*\bWebsiteQuester\b" 3; "~*\bWebsite\ Quester\b" 3; "~*\bWebster\b" 3; "~*\bWebStripper\b" 3; - "~*\bWeb\ Stripper\b" 3; "~*\bWebSucker\b" 3; "~*\bWeb\ Sucker\b" 3; "~*\bWebWhacker\b" 3; - "~*\bWeb\ Whacker\b" 3; "~*\bWebZIP\b" 3; "~*\bWeSEE\b" 3; "~*\bWhack\b" 3; diff --git a/.dev-tools/_conf_files_test2/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test2/bots.d/whitelist-ips.conf index ac0f7c816..c9e3281f2 100644 --- a/.dev-tools/_conf_files_test2/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test2/bots.d/whitelist-ips.conf @@ -26,4 +26,4 @@ # Only add actual IP addresses and ranges here 111.111.111.111 0; -35.202.124.16 0; +35.202.33.153 0; diff --git a/.dev-tools/_conf_files_test2/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test2/conf.d/globalblacklist.conf index 3d5bd064a..cadce82d9 100644 --- a/.dev-tools/_conf_files_test2/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test2/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.10.979 -### Updated: Thu Oct 26 14:47:07 SAST 2017 +### Version: V3.2017.11.980 +### Updated: Wed Nov 1 12:59:37 SAST 2017 ### Bad Referrer Count: 5904 -### Bad Bot Count: 513 +### Bad Bot Count: 496 ################################################### ### VERSION INFORMATION ## @@ -259,7 +259,6 @@ map $http_user_agent $bad_bot { "~*\bBolt\b" 3; "~*\bBotALot\b" 3; "~*\bBrandprotect\b" 3; - "~*\bBrandProtect\b" 3; "~*\bBrandwatch\b" 3; "~*\bBubing\b" 3; "~*\bBuddy\b" 3; @@ -289,7 +288,6 @@ map $http_user_agent $bad_bot { "~*\bCosmos\b" 3; "~*\bCraftbot\b" 3; "~*\bCrazyWebCrawler\b" 3; - "~*\bCRAZYWEBCRAWLER\b" 3; "~*\bCrescent\b" 3; "~*\bCSHttp\b" 3; "~*\bCurious\b" 3; @@ -313,8 +311,6 @@ map $http_user_agent $bad_bot { "~*\bDomainSigmaCrawler\b" 3; "~*\bDomainStatsBot\b" 3; "~*\bDotbot\b" 3; - "~*\bDownload\ Demon\b" 3; - "~*\bDownload\ Devil\b" 3; "~*\bDownload\ Wonder\b" 3; "~*\bDragonfly\b" 3; "~*\bDrip\b" 3; @@ -325,8 +321,6 @@ map $http_user_agent $bad_bot { "~*\bECCP/1.0\b" 3; "~*\bEcxi\b" 3; "~*\bEirGrabber\b" 3; - "~*\bEMail\ Collector\b" 3; - "~*\bEMail\ Extractor\b" 3; "~*\bEMail\ Siphon\b" 3; "~*\bEMail\ Wolf\b" 3; "~*\bEroCrawler\b" 3; @@ -386,7 +380,6 @@ map $http_user_agent $bad_bot { "~*\bId-search\b" 3; "~*\bIlseBot\b" 3; "~*\bImage\ Fetch\b" 3; - "~*\bImage\ Stripper\b" 3; "~*\bImage\ Sucker\b" 3; "~*\bIndy\ Library\b" 3; "~*\bInfoNaviRobot\b" 3; @@ -507,7 +500,6 @@ map $http_user_agent $bad_bot { "~*\bPageAnalyzer\b" 3; "~*\bPage\ Analyzer\b" 3; "~*\bPageGrabber\b" 3; - "~*\bPage\ Grabber\b" 3; "~*\bpage\ scorer\b" 3; "~*\bPageScorer\b" 3; "~*\bPanscient\b" 3; @@ -558,14 +550,12 @@ map $http_user_agent $bad_bot { "~*\bScanbot\b" 3; "~*\bScrapy\b" 3; "~*\bScreaming\b" 3; - "~*\bScreaming\ Frog\ SEO\ Spider\b" 3; "~*\bScreenerBot\b" 3; "~*\bSearchestate\b" 3; "~*\bSearchmetricsBot\b" 3; "~*\bSemrush\b" 3; "~*\bSemrushBot\b" 3; "~*\bSEOkicks\b" 3; - "~*\bSEOkicks-Robot\b" 3; "~*\bSEOlyticsCrawler\b" 3; "~*\bSeomoz\b" 3; "~*\bSEOprofiler\b" 3; @@ -573,7 +563,6 @@ map $http_user_agent $bad_bot { "~*\bSEOstats\b" 3; "~*\bSiphon\b" 3; "~*\bSISTRIX\b" 3; - "~*\bSISTRIX\ Crawler\b" 3; "~*\bSitebeam\b" 3; "~*\bSiteExplorer\b" 3; "~*\bSiteimprove\b" 3; @@ -628,12 +617,14 @@ map $http_user_agent $bad_bot { "~*\bTitan\b" 3; "~*\bToata\b" 3; "~*\bToweyabot\b" 3; + "~*\bTrendiction\b" 3; "~*\bTrendictionbot\b" 3; + "~*\btrendiction.com\b" 3; + "~*\btrendiction.de\b" 3; "~*\bTrue_Robot\b" 3; "~*\bTuringos\b" 3; + "~*\bTurnitin\b" 3; "~*\bTurnitinBot\b" 3; - "~*\bTurnitin\ Bot\b" 3; - "~*\bTurnitin\ Robot\b" 3; "~*\bTwengaBot\b" 3; "~*\bTwice\b" 3; "~*\bTyphoeus\b" 3; @@ -658,13 +649,10 @@ map $http_user_agent $bad_bot { "~*\bWebAuto\b" 3; "~*\bWeb\ Auto\b" 3; "~*\bWebBandit\b" 3; - "~*\bWeb\ Bandit\b" 3; "~*\bWebCollage\b" 3; "~*\bWeb\ Collage\b" 3; "~*\bWebCopier\b" 3; - "~*\bWeb\ Copier\b" 3; "~*\bWEBDAV\b" 3; - "~*\bWEBDAV\ Client\b" 3; "~*\bWebEnhancer\b" 3; "~*\bWeb\ Enhancer\b" 3; "~*\bWebFetch\b" 3; @@ -673,28 +661,23 @@ map $http_user_agent $bad_bot { "~*\bWeb\ Fuck\b" 3; "~*\bWebGo\ IS\b" 3; "~*\bWebImageCollector\b" 3; - "~*\bWeb\ Image\ Collector\b" 3; "~*\bWebLeacher\b" 3; "~*\bWebmasterWorldForumBot\b" 3; "~*\bwebmeup-crawler\b" 3; "~*\bWebPix\b" 3; "~*\bWeb\ Pix\b" 3; "~*\bWebReaper\b" 3; - "~*\bWeb\ Reaper\b" 3; "~*\bWebSauger\b" 3; "~*\bWeb\ Sauger\b" 3; "~*\bWebshag\b" 3; "~*\bWebsiteExtractor\b" 3; - "~*\bWebsite\ Extractor\b" 3; "~*\bWebsiteQuester\b" 3; "~*\bWebsite\ Quester\b" 3; "~*\bWebster\b" 3; "~*\bWebStripper\b" 3; - "~*\bWeb\ Stripper\b" 3; "~*\bWebSucker\b" 3; "~*\bWeb\ Sucker\b" 3; "~*\bWebWhacker\b" 3; - "~*\bWeb\ Whacker\b" 3; "~*\bWebZIP\b" 3; "~*\bWeSEE\b" 3; "~*\bWhack\b" 3; diff --git a/.dev-tools/_conf_files_test3/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test3/bots.d/whitelist-ips.conf index ac0f7c816..c9e3281f2 100644 --- a/.dev-tools/_conf_files_test3/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test3/bots.d/whitelist-ips.conf @@ -26,4 +26,4 @@ # Only add actual IP addresses and ranges here 111.111.111.111 0; -35.202.124.16 0; +35.202.33.153 0; diff --git a/.dev-tools/_conf_files_test3/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test3/conf.d/globalblacklist.conf index 1494f7baf..f0b1b0b35 100644 --- a/.dev-tools/_conf_files_test3/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test3/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.10.979 -### Updated: Thu Oct 26 14:47:07 SAST 2017 +### Version: V3.2017.11.980 +### Updated: Wed Nov 1 12:59:37 SAST 2017 ### Bad Referrer Count: 5904 -### Bad Bot Count: 513 +### Bad Bot Count: 496 ################################################### ### VERSION INFORMATION ## @@ -259,7 +259,6 @@ map $http_user_agent $bad_bot { "~*\bBolt\b" 3; "~*\bBotALot\b" 3; "~*\bBrandprotect\b" 3; - "~*\bBrandProtect\b" 3; "~*\bBrandwatch\b" 3; "~*\bBubing\b" 3; "~*\bBuddy\b" 3; @@ -289,7 +288,6 @@ map $http_user_agent $bad_bot { "~*\bCosmos\b" 3; "~*\bCraftbot\b" 3; "~*\bCrazyWebCrawler\b" 3; - "~*\bCRAZYWEBCRAWLER\b" 3; "~*\bCrescent\b" 3; "~*\bCSHttp\b" 3; "~*\bCurious\b" 3; @@ -313,8 +311,6 @@ map $http_user_agent $bad_bot { "~*\bDomainSigmaCrawler\b" 3; "~*\bDomainStatsBot\b" 3; "~*\bDotbot\b" 3; - "~*\bDownload\ Demon\b" 3; - "~*\bDownload\ Devil\b" 3; "~*\bDownload\ Wonder\b" 3; "~*\bDragonfly\b" 3; "~*\bDrip\b" 3; @@ -325,8 +321,6 @@ map $http_user_agent $bad_bot { "~*\bECCP/1.0\b" 3; "~*\bEcxi\b" 3; "~*\bEirGrabber\b" 3; - "~*\bEMail\ Collector\b" 3; - "~*\bEMail\ Extractor\b" 3; "~*\bEMail\ Siphon\b" 3; "~*\bEMail\ Wolf\b" 3; "~*\bEroCrawler\b" 3; @@ -386,7 +380,6 @@ map $http_user_agent $bad_bot { "~*\bId-search\b" 3; "~*\bIlseBot\b" 3; "~*\bImage\ Fetch\b" 3; - "~*\bImage\ Stripper\b" 3; "~*\bImage\ Sucker\b" 3; "~*\bIndy\ Library\b" 3; "~*\bInfoNaviRobot\b" 3; @@ -507,7 +500,6 @@ map $http_user_agent $bad_bot { "~*\bPageAnalyzer\b" 3; "~*\bPage\ Analyzer\b" 3; "~*\bPageGrabber\b" 3; - "~*\bPage\ Grabber\b" 3; "~*\bpage\ scorer\b" 3; "~*\bPageScorer\b" 3; "~*\bPanscient\b" 3; @@ -558,14 +550,12 @@ map $http_user_agent $bad_bot { "~*\bScanbot\b" 3; "~*\bScrapy\b" 3; "~*\bScreaming\b" 3; - "~*\bScreaming\ Frog\ SEO\ Spider\b" 3; "~*\bScreenerBot\b" 3; "~*\bSearchestate\b" 3; "~*\bSearchmetricsBot\b" 3; "~*\bSemrush\b" 3; "~*\bSemrushBot\b" 3; "~*\bSEOkicks\b" 3; - "~*\bSEOkicks-Robot\b" 3; "~*\bSEOlyticsCrawler\b" 3; "~*\bSeomoz\b" 3; "~*\bSEOprofiler\b" 3; @@ -573,7 +563,6 @@ map $http_user_agent $bad_bot { "~*\bSEOstats\b" 3; "~*\bSiphon\b" 3; "~*\bSISTRIX\b" 3; - "~*\bSISTRIX\ Crawler\b" 3; "~*\bSitebeam\b" 3; "~*\bSiteExplorer\b" 3; "~*\bSiteimprove\b" 3; @@ -628,12 +617,14 @@ map $http_user_agent $bad_bot { "~*\bTitan\b" 3; "~*\bToata\b" 3; "~*\bToweyabot\b" 3; + "~*\bTrendiction\b" 3; "~*\bTrendictionbot\b" 3; + "~*\btrendiction.com\b" 3; + "~*\btrendiction.de\b" 3; "~*\bTrue_Robot\b" 3; "~*\bTuringos\b" 3; + "~*\bTurnitin\b" 3; "~*\bTurnitinBot\b" 3; - "~*\bTurnitin\ Bot\b" 3; - "~*\bTurnitin\ Robot\b" 3; "~*\bTwengaBot\b" 3; "~*\bTwice\b" 3; "~*\bTyphoeus\b" 3; @@ -658,13 +649,10 @@ map $http_user_agent $bad_bot { "~*\bWebAuto\b" 3; "~*\bWeb\ Auto\b" 3; "~*\bWebBandit\b" 3; - "~*\bWeb\ Bandit\b" 3; "~*\bWebCollage\b" 3; "~*\bWeb\ Collage\b" 3; "~*\bWebCopier\b" 3; - "~*\bWeb\ Copier\b" 3; "~*\bWEBDAV\b" 3; - "~*\bWEBDAV\ Client\b" 3; "~*\bWebEnhancer\b" 3; "~*\bWeb\ Enhancer\b" 3; "~*\bWebFetch\b" 3; @@ -673,28 +661,23 @@ map $http_user_agent $bad_bot { "~*\bWeb\ Fuck\b" 3; "~*\bWebGo\ IS\b" 3; "~*\bWebImageCollector\b" 3; - "~*\bWeb\ Image\ Collector\b" 3; "~*\bWebLeacher\b" 3; "~*\bWebmasterWorldForumBot\b" 3; "~*\bwebmeup-crawler\b" 3; "~*\bWebPix\b" 3; "~*\bWeb\ Pix\b" 3; "~*\bWebReaper\b" 3; - "~*\bWeb\ Reaper\b" 3; "~*\bWebSauger\b" 3; "~*\bWeb\ Sauger\b" 3; "~*\bWebshag\b" 3; "~*\bWebsiteExtractor\b" 3; - "~*\bWebsite\ Extractor\b" 3; "~*\bWebsiteQuester\b" 3; "~*\bWebsite\ Quester\b" 3; "~*\bWebster\b" 3; "~*\bWebStripper\b" 3; - "~*\bWeb\ Stripper\b" 3; "~*\bWebSucker\b" 3; "~*\bWeb\ Sucker\b" 3; "~*\bWebWhacker\b" 3; - "~*\bWeb\ Whacker\b" 3; "~*\bWebZIP\b" 3; "~*\bWeSEE\b" 3; "~*\bWhack\b" 3; diff --git a/.dev-tools/_conf_files_test4/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test4/bots.d/whitelist-ips.conf index ac0f7c816..c9e3281f2 100644 --- a/.dev-tools/_conf_files_test4/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test4/bots.d/whitelist-ips.conf @@ -26,4 +26,4 @@ # Only add actual IP addresses and ranges here 111.111.111.111 0; -35.202.124.16 0; +35.202.33.153 0; diff --git a/.dev-tools/_conf_files_test4/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test4/conf.d/globalblacklist.conf index de14c35d5..875cb9d35 100644 --- a/.dev-tools/_conf_files_test4/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test4/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.10.979 -### Updated: Thu Oct 26 14:47:07 SAST 2017 +### Version: V3.2017.11.980 +### Updated: Wed Nov 1 12:59:37 SAST 2017 ### Bad Referrer Count: 5904 -### Bad Bot Count: 513 +### Bad Bot Count: 496 ################################################### ### VERSION INFORMATION ## @@ -259,7 +259,6 @@ map $http_user_agent $bad_bot { "~*\bBolt\b" 3; "~*\bBotALot\b" 3; "~*\bBrandprotect\b" 3; - "~*\bBrandProtect\b" 3; "~*\bBrandwatch\b" 3; "~*\bBubing\b" 3; "~*\bBuddy\b" 3; @@ -289,7 +288,6 @@ map $http_user_agent $bad_bot { "~*\bCosmos\b" 3; "~*\bCraftbot\b" 3; "~*\bCrazyWebCrawler\b" 3; - "~*\bCRAZYWEBCRAWLER\b" 3; "~*\bCrescent\b" 3; "~*\bCSHttp\b" 3; "~*\bCurious\b" 3; @@ -313,8 +311,6 @@ map $http_user_agent $bad_bot { "~*\bDomainSigmaCrawler\b" 3; "~*\bDomainStatsBot\b" 3; "~*\bDotbot\b" 3; - "~*\bDownload\ Demon\b" 3; - "~*\bDownload\ Devil\b" 3; "~*\bDownload\ Wonder\b" 3; "~*\bDragonfly\b" 3; "~*\bDrip\b" 3; @@ -325,8 +321,6 @@ map $http_user_agent $bad_bot { "~*\bECCP/1.0\b" 3; "~*\bEcxi\b" 3; "~*\bEirGrabber\b" 3; - "~*\bEMail\ Collector\b" 3; - "~*\bEMail\ Extractor\b" 3; "~*\bEMail\ Siphon\b" 3; "~*\bEMail\ Wolf\b" 3; "~*\bEroCrawler\b" 3; @@ -386,7 +380,6 @@ map $http_user_agent $bad_bot { "~*\bId-search\b" 3; "~*\bIlseBot\b" 3; "~*\bImage\ Fetch\b" 3; - "~*\bImage\ Stripper\b" 3; "~*\bImage\ Sucker\b" 3; "~*\bIndy\ Library\b" 3; "~*\bInfoNaviRobot\b" 3; @@ -507,7 +500,6 @@ map $http_user_agent $bad_bot { "~*\bPageAnalyzer\b" 3; "~*\bPage\ Analyzer\b" 3; "~*\bPageGrabber\b" 3; - "~*\bPage\ Grabber\b" 3; "~*\bpage\ scorer\b" 3; "~*\bPageScorer\b" 3; "~*\bPanscient\b" 3; @@ -558,14 +550,12 @@ map $http_user_agent $bad_bot { "~*\bScanbot\b" 3; "~*\bScrapy\b" 3; "~*\bScreaming\b" 3; - "~*\bScreaming\ Frog\ SEO\ Spider\b" 3; "~*\bScreenerBot\b" 3; "~*\bSearchestate\b" 3; "~*\bSearchmetricsBot\b" 3; "~*\bSemrush\b" 3; "~*\bSemrushBot\b" 3; "~*\bSEOkicks\b" 3; - "~*\bSEOkicks-Robot\b" 3; "~*\bSEOlyticsCrawler\b" 3; "~*\bSeomoz\b" 3; "~*\bSEOprofiler\b" 3; @@ -573,7 +563,6 @@ map $http_user_agent $bad_bot { "~*\bSEOstats\b" 3; "~*\bSiphon\b" 3; "~*\bSISTRIX\b" 3; - "~*\bSISTRIX\ Crawler\b" 3; "~*\bSitebeam\b" 3; "~*\bSiteExplorer\b" 3; "~*\bSiteimprove\b" 3; @@ -628,12 +617,14 @@ map $http_user_agent $bad_bot { "~*\bTitan\b" 3; "~*\bToata\b" 3; "~*\bToweyabot\b" 3; + "~*\bTrendiction\b" 3; "~*\bTrendictionbot\b" 3; + "~*\btrendiction.com\b" 3; + "~*\btrendiction.de\b" 3; "~*\bTrue_Robot\b" 3; "~*\bTuringos\b" 3; + "~*\bTurnitin\b" 3; "~*\bTurnitinBot\b" 3; - "~*\bTurnitin\ Bot\b" 3; - "~*\bTurnitin\ Robot\b" 3; "~*\bTwengaBot\b" 3; "~*\bTwice\b" 3; "~*\bTyphoeus\b" 3; @@ -658,13 +649,10 @@ map $http_user_agent $bad_bot { "~*\bWebAuto\b" 3; "~*\bWeb\ Auto\b" 3; "~*\bWebBandit\b" 3; - "~*\bWeb\ Bandit\b" 3; "~*\bWebCollage\b" 3; "~*\bWeb\ Collage\b" 3; "~*\bWebCopier\b" 3; - "~*\bWeb\ Copier\b" 3; "~*\bWEBDAV\b" 3; - "~*\bWEBDAV\ Client\b" 3; "~*\bWebEnhancer\b" 3; "~*\bWeb\ Enhancer\b" 3; "~*\bWebFetch\b" 3; @@ -673,28 +661,23 @@ map $http_user_agent $bad_bot { "~*\bWeb\ Fuck\b" 3; "~*\bWebGo\ IS\b" 3; "~*\bWebImageCollector\b" 3; - "~*\bWeb\ Image\ Collector\b" 3; "~*\bWebLeacher\b" 3; "~*\bWebmasterWorldForumBot\b" 3; "~*\bwebmeup-crawler\b" 3; "~*\bWebPix\b" 3; "~*\bWeb\ Pix\b" 3; "~*\bWebReaper\b" 3; - "~*\bWeb\ Reaper\b" 3; "~*\bWebSauger\b" 3; "~*\bWeb\ Sauger\b" 3; "~*\bWebshag\b" 3; "~*\bWebsiteExtractor\b" 3; - "~*\bWebsite\ Extractor\b" 3; "~*\bWebsiteQuester\b" 3; "~*\bWebsite\ Quester\b" 3; "~*\bWebster\b" 3; "~*\bWebStripper\b" 3; - "~*\bWeb\ Stripper\b" 3; "~*\bWebSucker\b" 3; "~*\bWeb\ Sucker\b" 3; "~*\bWebWhacker\b" 3; - "~*\bWeb\ Whacker\b" 3; "~*\bWebZIP\b" 3; "~*\bWeSEE\b" 3; "~*\bWhack\b" 3; diff --git a/.dev-tools/_curl_tests/curltest1.txt b/.dev-tools/_curl_tests/curltest1.txt index 2abbb00ee..4dea49215 100755 --- a/.dev-tools/_curl_tests/curltest1.txt +++ b/.dev-tools/_curl_tests/curltest1.txt @@ -1,4 +1,4 @@ -Last Tested: Thu Oct 26 14:47:12 SAST 2017 +Last Tested: Wed Nov 1 12:59:43 SAST 2017 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest2.txt b/.dev-tools/_curl_tests/curltest2.txt index 2abbb00ee..4dea49215 100755 --- a/.dev-tools/_curl_tests/curltest2.txt +++ b/.dev-tools/_curl_tests/curltest2.txt @@ -1,4 +1,4 @@ -Last Tested: Thu Oct 26 14:47:12 SAST 2017 +Last Tested: Wed Nov 1 12:59:43 SAST 2017 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest3.txt b/.dev-tools/_curl_tests/curltest3.txt index 2abbb00ee..4dea49215 100755 --- a/.dev-tools/_curl_tests/curltest3.txt +++ b/.dev-tools/_curl_tests/curltest3.txt @@ -1,4 +1,4 @@ -Last Tested: Thu Oct 26 14:47:12 SAST 2017 +Last Tested: Wed Nov 1 12:59:43 SAST 2017 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest4.txt b/.dev-tools/_curl_tests/curltest4.txt index 2abbb00ee..4dea49215 100755 --- a/.dev-tools/_curl_tests/curltest4.txt +++ b/.dev-tools/_curl_tests/curltest4.txt @@ -1,4 +1,4 @@ -Last Tested: Thu Oct 26 14:47:12 SAST 2017 +Last Tested: Wed Nov 1 12:59:43 SAST 2017 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest5.txt b/.dev-tools/_curl_tests/curltest5.txt index 666df58f9..d2f7cb485 100755 --- a/.dev-tools/_curl_tests/curltest5.txt +++ b/.dev-tools/_curl_tests/curltest5.txt @@ -1,4 +1,4 @@ -Last Tested: Thu Oct 26 14:47:12 SAST 2017 +Last Tested: Wed Nov 1 12:59:43 SAST 2017 diff --git a/.dev-tools/_curl_tests_4/curltest6.txt b/.dev-tools/_curl_tests_4/curltest6.txt index 53685519d..603525264 100755 --- a/.dev-tools/_curl_tests_4/curltest6.txt +++ b/.dev-tools/_curl_tests_4/curltest6.txt @@ -1,4 +1,4 @@ -Last Tested: Thu Oct 26 14:47:51 SAST 2017 +Last Tested: Wed Nov 1 13:00:26 SAST 2017 diff --git a/.dev-tools/_curl_tests_4/curltest7.txt b/.dev-tools/_curl_tests_4/curltest7.txt index 53685519d..603525264 100644 --- a/.dev-tools/_curl_tests_4/curltest7.txt +++ b/.dev-tools/_curl_tests_4/curltest7.txt @@ -1,4 +1,4 @@ -Last Tested: Thu Oct 26 14:47:51 SAST 2017 +Last Tested: Wed Nov 1 13:00:26 SAST 2017 diff --git a/.dev-tools/_curl_tests_4/curltest8.txt b/.dev-tools/_curl_tests_4/curltest8.txt index 53685519d..603525264 100644 --- a/.dev-tools/_curl_tests_4/curltest8.txt +++ b/.dev-tools/_curl_tests_4/curltest8.txt @@ -1,4 +1,4 @@ -Last Tested: Thu Oct 26 14:47:51 SAST 2017 +Last Tested: Wed Nov 1 13:00:26 SAST 2017 diff --git a/.dev-tools/_robots_input/robots-input.txt b/.dev-tools/_robots_input/robots-input.txt index 2bfbe338c..fbbe6e03d 100644 --- a/.dev-tools/_robots_input/robots-input.txt +++ b/.dev-tools/_robots_input/robots-input.txt @@ -43,7 +43,6 @@ Boardreader Bolt BotALot Brandprotect -BrandProtect Brandwatch Bubing Buddy @@ -73,7 +72,6 @@ Copyscape Cosmos Craftbot CrazyWebCrawler -CRAZYWEBCRAWLER Crescent CSHttp Curious @@ -97,8 +95,6 @@ DomainCrawler DomainSigmaCrawler DomainStatsBot Dotbot -Download Demon -Download Devil Download Wonder Dragonfly Drip @@ -109,8 +105,6 @@ eCatch ECCP/1.0 Ecxi EirGrabber -EMail Collector -EMail Extractor EMail Siphon EMail Wolf EroCrawler @@ -170,7 +164,6 @@ IDBot Id-search IlseBot Image Fetch -Image Stripper Image Sucker Indy Library InfoNaviRobot @@ -291,7 +284,6 @@ OutfoxBot PageAnalyzer Page Analyzer PageGrabber -Page Grabber page scorer PageScorer Panscient @@ -342,14 +334,12 @@ ScanAlert Scanbot Scrapy Screaming -Screaming Frog SEO Spider ScreenerBot Searchestate SearchmetricsBot Semrush SemrushBot SEOkicks -SEOkicks-Robot SEOlyticsCrawler Seomoz SEOprofiler @@ -357,7 +347,6 @@ seoscanners SEOstats Siphon SISTRIX -SISTRIX Crawler Sitebeam SiteExplorer Siteimprove @@ -412,12 +401,14 @@ TightTwatBot Titan Toata Toweyabot +Trendiction Trendictionbot +trendiction.com +trendiction.de True_Robot Turingos +Turnitin TurnitinBot -Turnitin Bot -Turnitin Robot TwengaBot Twice Typhoeus @@ -442,13 +433,10 @@ Webalta WebAuto Web Auto WebBandit -Web Bandit WebCollage Web Collage WebCopier -Web Copier WEBDAV -WEBDAV Client WebEnhancer Web Enhancer WebFetch @@ -457,28 +445,23 @@ WebFuck Web Fuck WebGo IS WebImageCollector -Web Image Collector WebLeacher WebmasterWorldForumBot webmeup-crawler WebPix Web Pix WebReaper -Web Reaper WebSauger Web Sauger Webshag WebsiteExtractor -Website Extractor WebsiteQuester Website Quester Webster WebStripper -Web Stripper WebSucker Web Sucker WebWhacker -Web Whacker WebZIP WeSEE Whack diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index de14c35d5..875cb9d35 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.10.979 -### Updated: Thu Oct 26 14:47:07 SAST 2017 +### Version: V3.2017.11.980 +### Updated: Wed Nov 1 12:59:37 SAST 2017 ### Bad Referrer Count: 5904 -### Bad Bot Count: 513 +### Bad Bot Count: 496 ################################################### ### VERSION INFORMATION ## @@ -259,7 +259,6 @@ map $http_user_agent $bad_bot { "~*\bBolt\b" 3; "~*\bBotALot\b" 3; "~*\bBrandprotect\b" 3; - "~*\bBrandProtect\b" 3; "~*\bBrandwatch\b" 3; "~*\bBubing\b" 3; "~*\bBuddy\b" 3; @@ -289,7 +288,6 @@ map $http_user_agent $bad_bot { "~*\bCosmos\b" 3; "~*\bCraftbot\b" 3; "~*\bCrazyWebCrawler\b" 3; - "~*\bCRAZYWEBCRAWLER\b" 3; "~*\bCrescent\b" 3; "~*\bCSHttp\b" 3; "~*\bCurious\b" 3; @@ -313,8 +311,6 @@ map $http_user_agent $bad_bot { "~*\bDomainSigmaCrawler\b" 3; "~*\bDomainStatsBot\b" 3; "~*\bDotbot\b" 3; - "~*\bDownload\ Demon\b" 3; - "~*\bDownload\ Devil\b" 3; "~*\bDownload\ Wonder\b" 3; "~*\bDragonfly\b" 3; "~*\bDrip\b" 3; @@ -325,8 +321,6 @@ map $http_user_agent $bad_bot { "~*\bECCP/1.0\b" 3; "~*\bEcxi\b" 3; "~*\bEirGrabber\b" 3; - "~*\bEMail\ Collector\b" 3; - "~*\bEMail\ Extractor\b" 3; "~*\bEMail\ Siphon\b" 3; "~*\bEMail\ Wolf\b" 3; "~*\bEroCrawler\b" 3; @@ -386,7 +380,6 @@ map $http_user_agent $bad_bot { "~*\bId-search\b" 3; "~*\bIlseBot\b" 3; "~*\bImage\ Fetch\b" 3; - "~*\bImage\ Stripper\b" 3; "~*\bImage\ Sucker\b" 3; "~*\bIndy\ Library\b" 3; "~*\bInfoNaviRobot\b" 3; @@ -507,7 +500,6 @@ map $http_user_agent $bad_bot { "~*\bPageAnalyzer\b" 3; "~*\bPage\ Analyzer\b" 3; "~*\bPageGrabber\b" 3; - "~*\bPage\ Grabber\b" 3; "~*\bpage\ scorer\b" 3; "~*\bPageScorer\b" 3; "~*\bPanscient\b" 3; @@ -558,14 +550,12 @@ map $http_user_agent $bad_bot { "~*\bScanbot\b" 3; "~*\bScrapy\b" 3; "~*\bScreaming\b" 3; - "~*\bScreaming\ Frog\ SEO\ Spider\b" 3; "~*\bScreenerBot\b" 3; "~*\bSearchestate\b" 3; "~*\bSearchmetricsBot\b" 3; "~*\bSemrush\b" 3; "~*\bSemrushBot\b" 3; "~*\bSEOkicks\b" 3; - "~*\bSEOkicks-Robot\b" 3; "~*\bSEOlyticsCrawler\b" 3; "~*\bSeomoz\b" 3; "~*\bSEOprofiler\b" 3; @@ -573,7 +563,6 @@ map $http_user_agent $bad_bot { "~*\bSEOstats\b" 3; "~*\bSiphon\b" 3; "~*\bSISTRIX\b" 3; - "~*\bSISTRIX\ Crawler\b" 3; "~*\bSitebeam\b" 3; "~*\bSiteExplorer\b" 3; "~*\bSiteimprove\b" 3; @@ -628,12 +617,14 @@ map $http_user_agent $bad_bot { "~*\bTitan\b" 3; "~*\bToata\b" 3; "~*\bToweyabot\b" 3; + "~*\bTrendiction\b" 3; "~*\bTrendictionbot\b" 3; + "~*\btrendiction.com\b" 3; + "~*\btrendiction.de\b" 3; "~*\bTrue_Robot\b" 3; "~*\bTuringos\b" 3; + "~*\bTurnitin\b" 3; "~*\bTurnitinBot\b" 3; - "~*\bTurnitin\ Bot\b" 3; - "~*\bTurnitin\ Robot\b" 3; "~*\bTwengaBot\b" 3; "~*\bTwice\b" 3; "~*\bTyphoeus\b" 3; @@ -658,13 +649,10 @@ map $http_user_agent $bad_bot { "~*\bWebAuto\b" 3; "~*\bWeb\ Auto\b" 3; "~*\bWebBandit\b" 3; - "~*\bWeb\ Bandit\b" 3; "~*\bWebCollage\b" 3; "~*\bWeb\ Collage\b" 3; "~*\bWebCopier\b" 3; - "~*\bWeb\ Copier\b" 3; "~*\bWEBDAV\b" 3; - "~*\bWEBDAV\ Client\b" 3; "~*\bWebEnhancer\b" 3; "~*\bWeb\ Enhancer\b" 3; "~*\bWebFetch\b" 3; @@ -673,28 +661,23 @@ map $http_user_agent $bad_bot { "~*\bWeb\ Fuck\b" 3; "~*\bWebGo\ IS\b" 3; "~*\bWebImageCollector\b" 3; - "~*\bWeb\ Image\ Collector\b" 3; "~*\bWebLeacher\b" 3; "~*\bWebmasterWorldForumBot\b" 3; "~*\bwebmeup-crawler\b" 3; "~*\bWebPix\b" 3; "~*\bWeb\ Pix\b" 3; "~*\bWebReaper\b" 3; - "~*\bWeb\ Reaper\b" 3; "~*\bWebSauger\b" 3; "~*\bWeb\ Sauger\b" 3; "~*\bWebshag\b" 3; "~*\bWebsiteExtractor\b" 3; - "~*\bWebsite\ Extractor\b" 3; "~*\bWebsiteQuester\b" 3; "~*\bWebsite\ Quester\b" 3; "~*\bWebster\b" 3; "~*\bWebStripper\b" 3; - "~*\bWeb\ Stripper\b" 3; "~*\bWebSucker\b" 3; "~*\bWeb\ Sucker\b" 3; "~*\bWebWhacker\b" 3; - "~*\bWeb\ Whacker\b" 3; "~*\bWebZIP\b" 3; "~*\bWeSEE\b" 3; "~*\bWhack\b" 3; diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index 10b30f8b5..4a591b205 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index f80e8a069..fb61cce2c 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 91f81d579..500e7f8eb 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V3.2017.10.979 +#### Version: V3.2017.11.980 #### Bad Referrer Count: 5904 -#### Bad Bot Count: 513 +#### Bad Bot Count: 496 ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index cabf8f0d1..cb8515d42 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V3.2017.10.979 +#### Version: V3.2017.11.980 #### Bad Referrer Count: 5904 -#### Bad Bot Count: 513 +#### Bad Bot Count: 496 ____________________ ## Help Support This Project diff --git a/README.md b/README.md index 5dbb20a6d..f8f683ebb 100755 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic from ever reaching your web sites. _______________ -#### Version: V3.2017.10.979 +#### Version: V3.2017.11.980 #### Bad Referrer Count: 5904 -#### Bad Bot Count: 513 +#### Bad Bot Count: 496 ____________________ ## Help Support This Project diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 056b2c8b9..dab677a16 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V3.2017.10.979 +#### Version: V3.2017.11.980 #### Bad Referrer Count: 5904 -#### Bad Bot Count: 513 +#### Bad Bot Count: 496 ____________________ ## Help Support This Project diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index de14c35d5..875cb9d35 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.10.979 -### Updated: Thu Oct 26 14:47:07 SAST 2017 +### Version: V3.2017.11.980 +### Updated: Wed Nov 1 12:59:37 SAST 2017 ### Bad Referrer Count: 5904 -### Bad Bot Count: 513 +### Bad Bot Count: 496 ################################################### ### VERSION INFORMATION ## @@ -259,7 +259,6 @@ map $http_user_agent $bad_bot { "~*\bBolt\b" 3; "~*\bBotALot\b" 3; "~*\bBrandprotect\b" 3; - "~*\bBrandProtect\b" 3; "~*\bBrandwatch\b" 3; "~*\bBubing\b" 3; "~*\bBuddy\b" 3; @@ -289,7 +288,6 @@ map $http_user_agent $bad_bot { "~*\bCosmos\b" 3; "~*\bCraftbot\b" 3; "~*\bCrazyWebCrawler\b" 3; - "~*\bCRAZYWEBCRAWLER\b" 3; "~*\bCrescent\b" 3; "~*\bCSHttp\b" 3; "~*\bCurious\b" 3; @@ -313,8 +311,6 @@ map $http_user_agent $bad_bot { "~*\bDomainSigmaCrawler\b" 3; "~*\bDomainStatsBot\b" 3; "~*\bDotbot\b" 3; - "~*\bDownload\ Demon\b" 3; - "~*\bDownload\ Devil\b" 3; "~*\bDownload\ Wonder\b" 3; "~*\bDragonfly\b" 3; "~*\bDrip\b" 3; @@ -325,8 +321,6 @@ map $http_user_agent $bad_bot { "~*\bECCP/1.0\b" 3; "~*\bEcxi\b" 3; "~*\bEirGrabber\b" 3; - "~*\bEMail\ Collector\b" 3; - "~*\bEMail\ Extractor\b" 3; "~*\bEMail\ Siphon\b" 3; "~*\bEMail\ Wolf\b" 3; "~*\bEroCrawler\b" 3; @@ -386,7 +380,6 @@ map $http_user_agent $bad_bot { "~*\bId-search\b" 3; "~*\bIlseBot\b" 3; "~*\bImage\ Fetch\b" 3; - "~*\bImage\ Stripper\b" 3; "~*\bImage\ Sucker\b" 3; "~*\bIndy\ Library\b" 3; "~*\bInfoNaviRobot\b" 3; @@ -507,7 +500,6 @@ map $http_user_agent $bad_bot { "~*\bPageAnalyzer\b" 3; "~*\bPage\ Analyzer\b" 3; "~*\bPageGrabber\b" 3; - "~*\bPage\ Grabber\b" 3; "~*\bpage\ scorer\b" 3; "~*\bPageScorer\b" 3; "~*\bPanscient\b" 3; @@ -558,14 +550,12 @@ map $http_user_agent $bad_bot { "~*\bScanbot\b" 3; "~*\bScrapy\b" 3; "~*\bScreaming\b" 3; - "~*\bScreaming\ Frog\ SEO\ Spider\b" 3; "~*\bScreenerBot\b" 3; "~*\bSearchestate\b" 3; "~*\bSearchmetricsBot\b" 3; "~*\bSemrush\b" 3; "~*\bSemrushBot\b" 3; "~*\bSEOkicks\b" 3; - "~*\bSEOkicks-Robot\b" 3; "~*\bSEOlyticsCrawler\b" 3; "~*\bSeomoz\b" 3; "~*\bSEOprofiler\b" 3; @@ -573,7 +563,6 @@ map $http_user_agent $bad_bot { "~*\bSEOstats\b" 3; "~*\bSiphon\b" 3; "~*\bSISTRIX\b" 3; - "~*\bSISTRIX\ Crawler\b" 3; "~*\bSitebeam\b" 3; "~*\bSiteExplorer\b" 3; "~*\bSiteimprove\b" 3; @@ -628,12 +617,14 @@ map $http_user_agent $bad_bot { "~*\bTitan\b" 3; "~*\bToata\b" 3; "~*\bToweyabot\b" 3; + "~*\bTrendiction\b" 3; "~*\bTrendictionbot\b" 3; + "~*\btrendiction.com\b" 3; + "~*\btrendiction.de\b" 3; "~*\bTrue_Robot\b" 3; "~*\bTuringos\b" 3; + "~*\bTurnitin\b" 3; "~*\bTurnitinBot\b" 3; - "~*\bTurnitin\ Bot\b" 3; - "~*\bTurnitin\ Robot\b" 3; "~*\bTwengaBot\b" 3; "~*\bTwice\b" 3; "~*\bTyphoeus\b" 3; @@ -658,13 +649,10 @@ map $http_user_agent $bad_bot { "~*\bWebAuto\b" 3; "~*\bWeb\ Auto\b" 3; "~*\bWebBandit\b" 3; - "~*\bWeb\ Bandit\b" 3; "~*\bWebCollage\b" 3; "~*\bWeb\ Collage\b" 3; "~*\bWebCopier\b" 3; - "~*\bWeb\ Copier\b" 3; "~*\bWEBDAV\b" 3; - "~*\bWEBDAV\ Client\b" 3; "~*\bWebEnhancer\b" 3; "~*\bWeb\ Enhancer\b" 3; "~*\bWebFetch\b" 3; @@ -673,28 +661,23 @@ map $http_user_agent $bad_bot { "~*\bWeb\ Fuck\b" 3; "~*\bWebGo\ IS\b" 3; "~*\bWebImageCollector\b" 3; - "~*\bWeb\ Image\ Collector\b" 3; "~*\bWebLeacher\b" 3; "~*\bWebmasterWorldForumBot\b" 3; "~*\bwebmeup-crawler\b" 3; "~*\bWebPix\b" 3; "~*\bWeb\ Pix\b" 3; "~*\bWebReaper\b" 3; - "~*\bWeb\ Reaper\b" 3; "~*\bWebSauger\b" 3; "~*\bWeb\ Sauger\b" 3; "~*\bWebshag\b" 3; "~*\bWebsiteExtractor\b" 3; - "~*\bWebsite\ Extractor\b" 3; "~*\bWebsiteQuester\b" 3; "~*\bWebsite\ Quester\b" 3; "~*\bWebster\b" 3; "~*\bWebStripper\b" 3; - "~*\bWeb\ Stripper\b" 3; "~*\bWebSucker\b" 3; "~*\bWeb\ Sucker\b" 3; "~*\bWebWhacker\b" 3; - "~*\bWeb\ Whacker\b" 3; "~*\bWebZIP\b" 3; "~*\bWeSEE\b" 3; "~*\bWhack\b" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index de14c35d5..875cb9d35 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.10.979 -### Updated: Thu Oct 26 14:47:07 SAST 2017 +### Version: V3.2017.11.980 +### Updated: Wed Nov 1 12:59:37 SAST 2017 ### Bad Referrer Count: 5904 -### Bad Bot Count: 513 +### Bad Bot Count: 496 ################################################### ### VERSION INFORMATION ## @@ -259,7 +259,6 @@ map $http_user_agent $bad_bot { "~*\bBolt\b" 3; "~*\bBotALot\b" 3; "~*\bBrandprotect\b" 3; - "~*\bBrandProtect\b" 3; "~*\bBrandwatch\b" 3; "~*\bBubing\b" 3; "~*\bBuddy\b" 3; @@ -289,7 +288,6 @@ map $http_user_agent $bad_bot { "~*\bCosmos\b" 3; "~*\bCraftbot\b" 3; "~*\bCrazyWebCrawler\b" 3; - "~*\bCRAZYWEBCRAWLER\b" 3; "~*\bCrescent\b" 3; "~*\bCSHttp\b" 3; "~*\bCurious\b" 3; @@ -313,8 +311,6 @@ map $http_user_agent $bad_bot { "~*\bDomainSigmaCrawler\b" 3; "~*\bDomainStatsBot\b" 3; "~*\bDotbot\b" 3; - "~*\bDownload\ Demon\b" 3; - "~*\bDownload\ Devil\b" 3; "~*\bDownload\ Wonder\b" 3; "~*\bDragonfly\b" 3; "~*\bDrip\b" 3; @@ -325,8 +321,6 @@ map $http_user_agent $bad_bot { "~*\bECCP/1.0\b" 3; "~*\bEcxi\b" 3; "~*\bEirGrabber\b" 3; - "~*\bEMail\ Collector\b" 3; - "~*\bEMail\ Extractor\b" 3; "~*\bEMail\ Siphon\b" 3; "~*\bEMail\ Wolf\b" 3; "~*\bEroCrawler\b" 3; @@ -386,7 +380,6 @@ map $http_user_agent $bad_bot { "~*\bId-search\b" 3; "~*\bIlseBot\b" 3; "~*\bImage\ Fetch\b" 3; - "~*\bImage\ Stripper\b" 3; "~*\bImage\ Sucker\b" 3; "~*\bIndy\ Library\b" 3; "~*\bInfoNaviRobot\b" 3; @@ -507,7 +500,6 @@ map $http_user_agent $bad_bot { "~*\bPageAnalyzer\b" 3; "~*\bPage\ Analyzer\b" 3; "~*\bPageGrabber\b" 3; - "~*\bPage\ Grabber\b" 3; "~*\bpage\ scorer\b" 3; "~*\bPageScorer\b" 3; "~*\bPanscient\b" 3; @@ -558,14 +550,12 @@ map $http_user_agent $bad_bot { "~*\bScanbot\b" 3; "~*\bScrapy\b" 3; "~*\bScreaming\b" 3; - "~*\bScreaming\ Frog\ SEO\ Spider\b" 3; "~*\bScreenerBot\b" 3; "~*\bSearchestate\b" 3; "~*\bSearchmetricsBot\b" 3; "~*\bSemrush\b" 3; "~*\bSemrushBot\b" 3; "~*\bSEOkicks\b" 3; - "~*\bSEOkicks-Robot\b" 3; "~*\bSEOlyticsCrawler\b" 3; "~*\bSeomoz\b" 3; "~*\bSEOprofiler\b" 3; @@ -573,7 +563,6 @@ map $http_user_agent $bad_bot { "~*\bSEOstats\b" 3; "~*\bSiphon\b" 3; "~*\bSISTRIX\b" 3; - "~*\bSISTRIX\ Crawler\b" 3; "~*\bSitebeam\b" 3; "~*\bSiteExplorer\b" 3; "~*\bSiteimprove\b" 3; @@ -628,12 +617,14 @@ map $http_user_agent $bad_bot { "~*\bTitan\b" 3; "~*\bToata\b" 3; "~*\bToweyabot\b" 3; + "~*\bTrendiction\b" 3; "~*\bTrendictionbot\b" 3; + "~*\btrendiction.com\b" 3; + "~*\btrendiction.de\b" 3; "~*\bTrue_Robot\b" 3; "~*\bTuringos\b" 3; + "~*\bTurnitin\b" 3; "~*\bTurnitinBot\b" 3; - "~*\bTurnitin\ Bot\b" 3; - "~*\bTurnitin\ Robot\b" 3; "~*\bTwengaBot\b" 3; "~*\bTwice\b" 3; "~*\bTyphoeus\b" 3; @@ -658,13 +649,10 @@ map $http_user_agent $bad_bot { "~*\bWebAuto\b" 3; "~*\bWeb\ Auto\b" 3; "~*\bWebBandit\b" 3; - "~*\bWeb\ Bandit\b" 3; "~*\bWebCollage\b" 3; "~*\bWeb\ Collage\b" 3; "~*\bWebCopier\b" 3; - "~*\bWeb\ Copier\b" 3; "~*\bWEBDAV\b" 3; - "~*\bWEBDAV\ Client\b" 3; "~*\bWebEnhancer\b" 3; "~*\bWeb\ Enhancer\b" 3; "~*\bWebFetch\b" 3; @@ -673,28 +661,23 @@ map $http_user_agent $bad_bot { "~*\bWeb\ Fuck\b" 3; "~*\bWebGo\ IS\b" 3; "~*\bWebImageCollector\b" 3; - "~*\bWeb\ Image\ Collector\b" 3; "~*\bWebLeacher\b" 3; "~*\bWebmasterWorldForumBot\b" 3; "~*\bwebmeup-crawler\b" 3; "~*\bWebPix\b" 3; "~*\bWeb\ Pix\b" 3; "~*\bWebReaper\b" 3; - "~*\bWeb\ Reaper\b" 3; "~*\bWebSauger\b" 3; "~*\bWeb\ Sauger\b" 3; "~*\bWebshag\b" 3; "~*\bWebsiteExtractor\b" 3; - "~*\bWebsite\ Extractor\b" 3; "~*\bWebsiteQuester\b" 3; "~*\bWebsite\ Quester\b" 3; "~*\bWebster\b" 3; "~*\bWebStripper\b" 3; - "~*\bWeb\ Stripper\b" 3; "~*\bWebSucker\b" 3; "~*\bWeb\ Sucker\b" 3; "~*\bWebWhacker\b" 3; - "~*\bWeb\ Whacker\b" 3; "~*\bWebZIP\b" 3; "~*\bWeSEE\b" 3; "~*\bWhack\b" 3; diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 8e6e0c5af..ddd02098a 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -1,8 +1,8 @@ ### Version Information # ################################################### -### Version: V3.2017.10.979 -### Updated: Thu Oct 26 14:47:07 SAST 2017 -### Bad Bot Count: 513 +### Version: V3.2017.11.980 +### Updated: Wed Nov 1 12:59:38 SAST 2017 +### Bad Bot Count: 496 ################################################### ### Version Information ## @@ -99,8 +99,6 @@ User-agent: BotALot Disallow:/ User-agent: Brandprotect Disallow:/ -User-agent: BrandProtect -Disallow:/ User-agent: Brandwatch Disallow:/ User-agent: Bubing @@ -159,8 +157,6 @@ User-agent: Craftbot Disallow:/ User-agent: CrazyWebCrawler Disallow:/ -User-agent: CRAZYWEBCRAWLER -Disallow:/ User-agent: Crescent Disallow:/ User-agent: CSHttp @@ -207,10 +203,6 @@ User-agent: DomainStatsBot Disallow:/ User-agent: Dotbot Disallow:/ -User-agent: Download Demon -Disallow:/ -User-agent: Download Devil -Disallow:/ User-agent: Download Wonder Disallow:/ User-agent: Dragonfly @@ -231,10 +223,6 @@ User-agent: Ecxi Disallow:/ User-agent: EirGrabber Disallow:/ -User-agent: EMail Collector -Disallow:/ -User-agent: EMail Extractor -Disallow:/ User-agent: EMail Siphon Disallow:/ User-agent: EMail Wolf @@ -353,8 +341,6 @@ User-agent: IlseBot Disallow:/ User-agent: Image Fetch Disallow:/ -User-agent: Image Stripper -Disallow:/ User-agent: Image Sucker Disallow:/ User-agent: Indy Library @@ -595,8 +581,6 @@ User-agent: Page Analyzer Disallow:/ User-agent: PageGrabber Disallow:/ -User-agent: Page Grabber -Disallow:/ User-agent: page scorer Disallow:/ User-agent: PageScorer @@ -697,8 +681,6 @@ User-agent: Scrapy Disallow:/ User-agent: Screaming Disallow:/ -User-agent: Screaming Frog SEO Spider -Disallow:/ User-agent: ScreenerBot Disallow:/ User-agent: Searchestate @@ -711,8 +693,6 @@ User-agent: SemrushBot Disallow:/ User-agent: SEOkicks Disallow:/ -User-agent: SEOkicks-Robot -Disallow:/ User-agent: SEOlyticsCrawler Disallow:/ User-agent: Seomoz @@ -727,8 +707,6 @@ User-agent: Siphon Disallow:/ User-agent: SISTRIX Disallow:/ -User-agent: SISTRIX Crawler -Disallow:/ User-agent: Sitebeam Disallow:/ User-agent: SiteExplorer @@ -837,18 +815,22 @@ User-agent: Toata Disallow:/ User-agent: Toweyabot Disallow:/ +User-agent: Trendiction +Disallow:/ User-agent: Trendictionbot Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ User-agent: True_Robot Disallow:/ User-agent: Turingos Disallow:/ +User-agent: Turnitin +Disallow:/ User-agent: TurnitinBot Disallow:/ -User-agent: Turnitin Bot -Disallow:/ -User-agent: Turnitin Robot -Disallow:/ User-agent: TwengaBot Disallow:/ User-agent: Twice @@ -897,20 +879,14 @@ User-agent: Web Auto Disallow:/ User-agent: WebBandit Disallow:/ -User-agent: Web Bandit -Disallow:/ User-agent: WebCollage Disallow:/ User-agent: Web Collage Disallow:/ User-agent: WebCopier Disallow:/ -User-agent: Web Copier -Disallow:/ User-agent: WEBDAV Disallow:/ -User-agent: WEBDAV Client -Disallow:/ User-agent: WebEnhancer Disallow:/ User-agent: Web Enhancer @@ -927,8 +903,6 @@ User-agent: WebGo IS Disallow:/ User-agent: WebImageCollector Disallow:/ -User-agent: Web Image Collector -Disallow:/ User-agent: WebLeacher Disallow:/ User-agent: WebmasterWorldForumBot @@ -941,8 +915,6 @@ User-agent: Web Pix Disallow:/ User-agent: WebReaper Disallow:/ -User-agent: Web Reaper -Disallow:/ User-agent: WebSauger Disallow:/ User-agent: Web Sauger @@ -951,8 +923,6 @@ User-agent: Webshag Disallow:/ User-agent: WebsiteExtractor Disallow:/ -User-agent: Website Extractor -Disallow:/ User-agent: WebsiteQuester Disallow:/ User-agent: Website Quester @@ -961,16 +931,12 @@ User-agent: Webster Disallow:/ User-agent: WebStripper Disallow:/ -User-agent: Web Stripper -Disallow:/ User-agent: WebSucker Disallow:/ User-agent: Web Sucker Disallow:/ User-agent: WebWhacker Disallow:/ -User-agent: Web Whacker -Disallow:/ User-agent: WebZIP Disallow:/ User-agent: WeSEE