vikarti.anatra/nginx-ultimate-bad-bot-blocker
1
0
Fork 0
mirror of https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git synced 2025-09-03 19:20:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

V3.2017.11.980 [ci skip]

Browse source
This commit is contained in:
Travis 2017-11-01 13:00:26 +02:00
parent 47135f0e66
commit 8bb1d8e20b
51 changed files with 108 additions and 278 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf
View file

@ -26,4 +26,4 @@
# Only add actual IP addresses and ranges here # Only add actual IP addresses and ranges here
111.111.111.111 0; 111.111.111.111 0;
35.202.124.16 0; 35.202.33.153 0;

31
.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf
View file

@ -4,10 +4,10 @@
### VERSION INFORMATION # ### VERSION INFORMATION #
################################################### ###################################################
### Version: V3.2017.10.979 ### Version: V3.2017.11.980
### Updated: Thu Oct 26 14:47:07 SAST 2017 ### Updated: Wed Nov 1 12:59:37 SAST 2017
### Bad Referrer Count: 5904 ### Bad Referrer Count: 5904
### Bad Bot Count: 513 ### Bad Bot Count: 496
################################################### ###################################################
### VERSION INFORMATION ## ### VERSION INFORMATION ##
@ -259,7 +259,6 @@ map $http_user_agent $bad_bot {
"~*\bBolt\b" 3; "~*\bBolt\b" 3;
"~*\bBotALot\b" 3; "~*\bBotALot\b" 3;
"~*\bBrandprotect\b" 3; "~*\bBrandprotect\b" 3;
"~*\bBrandProtect\b" 3;
"~*\bBrandwatch\b" 3; "~*\bBrandwatch\b" 3;
"~*\bBubing\b" 3; "~*\bBubing\b" 3;
"~*\bBuddy\b" 3; "~*\bBuddy\b" 3;
@ -289,7 +288,6 @@ map $http_user_agent $bad_bot {
"~*\bCosmos\b" 3; "~*\bCosmos\b" 3;
"~*\bCraftbot\b" 3; "~*\bCraftbot\b" 3;
"~*\bCrazyWebCrawler\b" 3; "~*\bCrazyWebCrawler\b" 3;
"~*\bCRAZYWEBCRAWLER\b" 3;
"~*\bCrescent\b" 3; "~*\bCrescent\b" 3;
"~*\bCSHttp\b" 3; "~*\bCSHttp\b" 3;
"~*\bCurious\b" 3; "~*\bCurious\b" 3;
@ -313,8 +311,6 @@ map $http_user_agent $bad_bot {
"~*\bDomainSigmaCrawler\b" 3; "~*\bDomainSigmaCrawler\b" 3;
"~*\bDomainStatsBot\b" 3; "~*\bDomainStatsBot\b" 3;
"~*\bDotbot\b" 3; "~*\bDotbot\b" 3;
"~*\bDownload\ Demon\b" 3;
"~*\bDownload\ Devil\b" 3;
"~*\bDownload\ Wonder\b" 3; "~*\bDownload\ Wonder\b" 3;
"~*\bDragonfly\b" 3; "~*\bDragonfly\b" 3;
"~*\bDrip\b" 3; "~*\bDrip\b" 3;
@ -325,8 +321,6 @@ map $http_user_agent $bad_bot {
"~*\bECCP/1.0\b" 3; "~*\bECCP/1.0\b" 3;
"~*\bEcxi\b" 3; "~*\bEcxi\b" 3;
"~*\bEirGrabber\b" 3; "~*\bEirGrabber\b" 3;
"~*\bEMail\ Collector\b" 3;
"~*\bEMail\ Extractor\b" 3;
"~*\bEMail\ Siphon\b" 3; "~*\bEMail\ Siphon\b" 3;
"~*\bEMail\ Wolf\b" 3; "~*\bEMail\ Wolf\b" 3;
"~*\bEroCrawler\b" 3; "~*\bEroCrawler\b" 3;
@ -386,7 +380,6 @@ map $http_user_agent $bad_bot {
"~*\bId-search\b" 3; "~*\bId-search\b" 3;
"~*\bIlseBot\b" 3; "~*\bIlseBot\b" 3;
"~*\bImage\ Fetch\b" 3; "~*\bImage\ Fetch\b" 3;
"~*\bImage\ Stripper\b" 3;
"~*\bImage\ Sucker\b" 3; "~*\bImage\ Sucker\b" 3;
"~*\bIndy\ Library\b" 3; "~*\bIndy\ Library\b" 3;
"~*\bInfoNaviRobot\b" 3; "~*\bInfoNaviRobot\b" 3;
@ -507,7 +500,6 @@ map $http_user_agent $bad_bot {
"~*\bPageAnalyzer\b" 3; "~*\bPageAnalyzer\b" 3;
"~*\bPage\ Analyzer\b" 3; "~*\bPage\ Analyzer\b" 3;
"~*\bPageGrabber\b" 3; "~*\bPageGrabber\b" 3;
"~*\bPage\ Grabber\b" 3;
"~*\bpage\ scorer\b" 3; "~*\bpage\ scorer\b" 3;
"~*\bPageScorer\b" 3; "~*\bPageScorer\b" 3;
"~*\bPanscient\b" 3; "~*\bPanscient\b" 3;
@ -558,14 +550,12 @@ map $http_user_agent $bad_bot {
"~*\bScanbot\b" 3; "~*\bScanbot\b" 3;
"~*\bScrapy\b" 3; "~*\bScrapy\b" 3;
"~*\bScreaming\b" 3; "~*\bScreaming\b" 3;
"~*\bScreaming\ Frog\ SEO\ Spider\b" 3;
"~*\bScreenerBot\b" 3; "~*\bScreenerBot\b" 3;
"~*\bSearchestate\b" 3; "~*\bSearchestate\b" 3;
"~*\bSearchmetricsBot\b" 3; "~*\bSearchmetricsBot\b" 3;
"~*\bSemrush\b" 3; "~*\bSemrush\b" 3;
"~*\bSemrushBot\b" 3; "~*\bSemrushBot\b" 3;
"~*\bSEOkicks\b" 3; "~*\bSEOkicks\b" 3;
"~*\bSEOkicks-Robot\b" 3;
"~*\bSEOlyticsCrawler\b" 3; "~*\bSEOlyticsCrawler\b" 3;
"~*\bSeomoz\b" 3; "~*\bSeomoz\b" 3;
"~*\bSEOprofiler\b" 3; "~*\bSEOprofiler\b" 3;
@ -573,7 +563,6 @@ map $http_user_agent $bad_bot {
"~*\bSEOstats\b" 3; "~*\bSEOstats\b" 3;
"~*\bSiphon\b" 3; "~*\bSiphon\b" 3;
"~*\bSISTRIX\b" 3; "~*\bSISTRIX\b" 3;
"~*\bSISTRIX\ Crawler\b" 3;
"~*\bSitebeam\b" 3; "~*\bSitebeam\b" 3;
"~*\bSiteExplorer\b" 3; "~*\bSiteExplorer\b" 3;
"~*\bSiteimprove\b" 3; "~*\bSiteimprove\b" 3;
@ -628,12 +617,14 @@ map $http_user_agent $bad_bot {
"~*\bTitan\b" 3; "~*\bTitan\b" 3;
"~*\bToata\b" 3; "~*\bToata\b" 3;
"~*\bToweyabot\b" 3; "~*\bToweyabot\b" 3;
"~*\bTrendiction\b" 3;
"~*\bTrendictionbot\b" 3; "~*\bTrendictionbot\b" 3;
"~*\btrendiction.com\b" 3;
"~*\btrendiction.de\b" 3;
"~*\bTrue_Robot\b" 3; "~*\bTrue_Robot\b" 3;
"~*\bTuringos\b" 3; "~*\bTuringos\b" 3;
"~*\bTurnitin\b" 3;
"~*\bTurnitinBot\b" 3; "~*\bTurnitinBot\b" 3;
"~*\bTurnitin\ Bot\b" 3;
"~*\bTurnitin\ Robot\b" 3;
"~*\bTwengaBot\b" 3; "~*\bTwengaBot\b" 3;
"~*\bTwice\b" 3; "~*\bTwice\b" 3;
"~*\bTyphoeus\b" 3; "~*\bTyphoeus\b" 3;
@ -658,13 +649,10 @@ map $http_user_agent $bad_bot {
"~*\bWebAuto\b" 3; "~*\bWebAuto\b" 3;
"~*\bWeb\ Auto\b" 3; "~*\bWeb\ Auto\b" 3;
"~*\bWebBandit\b" 3; "~*\bWebBandit\b" 3;
"~*\bWeb\ Bandit\b" 3;
"~*\bWebCollage\b" 3; "~*\bWebCollage\b" 3;
"~*\bWeb\ Collage\b" 3; "~*\bWeb\ Collage\b" 3;
"~*\bWebCopier\b" 3; "~*\bWebCopier\b" 3;
"~*\bWeb\ Copier\b" 3;
"~*\bWEBDAV\b" 3; "~*\bWEBDAV\b" 3;
"~*\bWEBDAV\ Client\b" 3;
"~*\bWebEnhancer\b" 3; "~*\bWebEnhancer\b" 3;
"~*\bWeb\ Enhancer\b" 3; "~*\bWeb\ Enhancer\b" 3;
"~*\bWebFetch\b" 3; "~*\bWebFetch\b" 3;
@ -673,28 +661,23 @@ map $http_user_agent $bad_bot {
"~*\bWeb\ Fuck\b" 3; "~*\bWeb\ Fuck\b" 3;
"~*\bWebGo\ IS\b" 3; "~*\bWebGo\ IS\b" 3;
"~*\bWebImageCollector\b" 3; "~*\bWebImageCollector\b" 3;
"~*\bWeb\ Image\ Collector\b" 3;
"~*\bWebLeacher\b" 3; "~*\bWebLeacher\b" 3;
"~*\bWebmasterWorldForumBot\b" 3; "~*\bWebmasterWorldForumBot\b" 3;
"~*\bwebmeup-crawler\b" 3; "~*\bwebmeup-crawler\b" 3;
"~*\bWebPix\b" 3; "~*\bWebPix\b" 3;
"~*\bWeb\ Pix\b" 3; "~*\bWeb\ Pix\b" 3;
"~*\bWebReaper\b" 3; "~*\bWebReaper\b" 3;
"~*\bWeb\ Reaper\b" 3;
"~*\bWebSauger\b" 3; "~*\bWebSauger\b" 3;
"~*\bWeb\ Sauger\b" 3; "~*\bWeb\ Sauger\b" 3;
"~*\bWebshag\b" 3; "~*\bWebshag\b" 3;
"~*\bWebsiteExtractor\b" 3; "~*\bWebsiteExtractor\b" 3;
"~*\bWebsite\ Extractor\b" 3;
"~*\bWebsiteQuester\b" 3; "~*\bWebsiteQuester\b" 3;
"~*\bWebsite\ Quester\b" 3; "~*\bWebsite\ Quester\b" 3;
"~*\bWebster\b" 3; "~*\bWebster\b" 3;
"~*\bWebStripper\b" 3; "~*\bWebStripper\b" 3;
"~*\bWeb\ Stripper\b" 3;
"~*\bWebSucker\b" 3; "~*\bWebSucker\b" 3;
"~*\bWeb\ Sucker\b" 3; "~*\bWeb\ Sucker\b" 3;
"~*\bWebWhacker\b" 3; "~*\bWebWhacker\b" 3;
"~*\bWeb\ Whacker\b" 3;
"~*\bWebZIP\b" 3; "~*\bWebZIP\b" 3;
"~*\bWeSEE\b" 3; "~*\bWeSEE\b" 3;
"~*\bWhack\b" 3; "~*\bWhack\b" 3;

2
.dev-tools/_conf_files_test2/bots.d/whitelist-ips.conf
View file

@ -26,4 +26,4 @@
# Only add actual IP addresses and ranges here # Only add actual IP addresses and ranges here
111.111.111.111 0; 111.111.111.111 0;
35.202.124.16 0; 35.202.33.153 0;

31
.dev-tools/_conf_files_test2/conf.d/globalblacklist.conf
View file

@ -4,10 +4,10 @@
### VERSION INFORMATION # ### VERSION INFORMATION #
################################################### ###################################################
### Version: V3.2017.10.979 ### Version: V3.2017.11.980
### Updated: Thu Oct 26 14:47:07 SAST 2017 ### Updated: Wed Nov 1 12:59:37 SAST 2017
### Bad Referrer Count: 5904 ### Bad Referrer Count: 5904
### Bad Bot Count: 513 ### Bad Bot Count: 496
################################################### ###################################################
### VERSION INFORMATION ## ### VERSION INFORMATION ##
@ -259,7 +259,6 @@ map $http_user_agent $bad_bot {
"~*\bBolt\b" 3; "~*\bBolt\b" 3;
"~*\bBotALot\b" 3; "~*\bBotALot\b" 3;
"~*\bBrandprotect\b" 3; "~*\bBrandprotect\b" 3;
"~*\bBrandProtect\b" 3;
"~*\bBrandwatch\b" 3; "~*\bBrandwatch\b" 3;
"~*\bBubing\b" 3; "~*\bBubing\b" 3;
"~*\bBuddy\b" 3; "~*\bBuddy\b" 3;
@ -289,7 +288,6 @@ map $http_user_agent $bad_bot {
"~*\bCosmos\b" 3; "~*\bCosmos\b" 3;
"~*\bCraftbot\b" 3; "~*\bCraftbot\b" 3;
"~*\bCrazyWebCrawler\b" 3; "~*\bCrazyWebCrawler\b" 3;
"~*\bCRAZYWEBCRAWLER\b" 3;
"~*\bCrescent\b" 3; "~*\bCrescent\b" 3;
"~*\bCSHttp\b" 3; "~*\bCSHttp\b" 3;
"~*\bCurious\b" 3; "~*\bCurious\b" 3;
@ -313,8 +311,6 @@ map $http_user_agent $bad_bot {
"~*\bDomainSigmaCrawler\b" 3; "~*\bDomainSigmaCrawler\b" 3;
"~*\bDomainStatsBot\b" 3; "~*\bDomainStatsBot\b" 3;
"~*\bDotbot\b" 3; "~*\bDotbot\b" 3;
"~*\bDownload\ Demon\b" 3;
"~*\bDownload\ Devil\b" 3;
"~*\bDownload\ Wonder\b" 3; "~*\bDownload\ Wonder\b" 3;
"~*\bDragonfly\b" 3; "~*\bDragonfly\b" 3;
"~*\bDrip\b" 3; "~*\bDrip\b" 3;
@ -325,8 +321,6 @@ map $http_user_agent $bad_bot {
"~*\bECCP/1.0\b" 3; "~*\bECCP/1.0\b" 3;
"~*\bEcxi\b" 3; "~*\bEcxi\b" 3;
"~*\bEirGrabber\b" 3; "~*\bEirGrabber\b" 3;
"~*\bEMail\ Collector\b" 3;
"~*\bEMail\ Extractor\b" 3;
"~*\bEMail\ Siphon\b" 3; "~*\bEMail\ Siphon\b" 3;
"~*\bEMail\ Wolf\b" 3; "~*\bEMail\ Wolf\b" 3;
"~*\bEroCrawler\b" 3; "~*\bEroCrawler\b" 3;
@ -386,7 +380,6 @@ map $http_user_agent $bad_bot {
"~*\bId-search\b" 3; "~*\bId-search\b" 3;
"~*\bIlseBot\b" 3; "~*\bIlseBot\b" 3;
"~*\bImage\ Fetch\b" 3; "~*\bImage\ Fetch\b" 3;
"~*\bImage\ Stripper\b" 3;
"~*\bImage\ Sucker\b" 3; "~*\bImage\ Sucker\b" 3;
"~*\bIndy\ Library\b" 3; "~*\bIndy\ Library\b" 3;
"~*\bInfoNaviRobot\b" 3; "~*\bInfoNaviRobot\b" 3;
@ -507,7 +500,6 @@ map $http_user_agent $bad_bot {
"~*\bPageAnalyzer\b" 3; "~*\bPageAnalyzer\b" 3;
"~*\bPage\ Analyzer\b" 3; "~*\bPage\ Analyzer\b" 3;
"~*\bPageGrabber\b" 3; "~*\bPageGrabber\b" 3;
"~*\bPage\ Grabber\b" 3;
"~*\bpage\ scorer\b" 3; "~*\bpage\ scorer\b" 3;
"~*\bPageScorer\b" 3; "~*\bPageScorer\b" 3;
"~*\bPanscient\b" 3; "~*\bPanscient\b" 3;
@ -558,14 +550,12 @@ map $http_user_agent $bad_bot {
"~*\bScanbot\b" 3; "~*\bScanbot\b" 3;
"~*\bScrapy\b" 3; "~*\bScrapy\b" 3;
"~*\bScreaming\b" 3; "~*\bScreaming\b" 3;
"~*\bScreaming\ Frog\ SEO\ Spider\b" 3;
"~*\bScreenerBot\b" 3; "~*\bScreenerBot\b" 3;
"~*\bSearchestate\b" 3; "~*\bSearchestate\b" 3;
"~*\bSearchmetricsBot\b" 3; "~*\bSearchmetricsBot\b" 3;
"~*\bSemrush\b" 3; "~*\bSemrush\b" 3;
"~*\bSemrushBot\b" 3; "~*\bSemrushBot\b" 3;
"~*\bSEOkicks\b" 3; "~*\bSEOkicks\b" 3;
"~*\bSEOkicks-Robot\b" 3;
"~*\bSEOlyticsCrawler\b" 3; "~*\bSEOlyticsCrawler\b" 3;
"~*\bSeomoz\b" 3; "~*\bSeomoz\b" 3;
"~*\bSEOprofiler\b" 3; "~*\bSEOprofiler\b" 3;
@ -573,7 +563,6 @@ map $http_user_agent $bad_bot {
"~*\bSEOstats\b" 3; "~*\bSEOstats\b" 3;
"~*\bSiphon\b" 3; "~*\bSiphon\b" 3;
"~*\bSISTRIX\b" 3; "~*\bSISTRIX\b" 3;
"~*\bSISTRIX\ Crawler\b" 3;
"~*\bSitebeam\b" 3; "~*\bSitebeam\b" 3;
"~*\bSiteExplorer\b" 3; "~*\bSiteExplorer\b" 3;
"~*\bSiteimprove\b" 3; "~*\bSiteimprove\b" 3;
@ -628,12 +617,14 @@ map $http_user_agent $bad_bot {
"~*\bTitan\b" 3; "~*\bTitan\b" 3;
"~*\bToata\b" 3; "~*\bToata\b" 3;
"~*\bToweyabot\b" 3; "~*\bToweyabot\b" 3;
"~*\bTrendiction\b" 3;
"~*\bTrendictionbot\b" 3; "~*\bTrendictionbot\b" 3;
"~*\btrendiction.com\b" 3;
"~*\btrendiction.de\b" 3;
"~*\bTrue_Robot\b" 3; "~*\bTrue_Robot\b" 3;
"~*\bTuringos\b" 3; "~*\bTuringos\b" 3;
"~*\bTurnitin\b" 3;
"~*\bTurnitinBot\b" 3; "~*\bTurnitinBot\b" 3;
"~*\bTurnitin\ Bot\b" 3;
"~*\bTurnitin\ Robot\b" 3;
"~*\bTwengaBot\b" 3; "~*\bTwengaBot\b" 3;
"~*\bTwice\b" 3; "~*\bTwice\b" 3;
"~*\bTyphoeus\b" 3; "~*\bTyphoeus\b" 3;
@ -658,13 +649,10 @@ map $http_user_agent $bad_bot {
"~*\bWebAuto\b" 3; "~*\bWebAuto\b" 3;
"~*\bWeb\ Auto\b" 3; "~*\bWeb\ Auto\b" 3;
"~*\bWebBandit\b" 3; "~*\bWebBandit\b" 3;
"~*\bWeb\ Bandit\b" 3;
"~*\bWebCollage\b" 3; "~*\bWebCollage\b" 3;
"~*\bWeb\ Collage\b" 3; "~*\bWeb\ Collage\b" 3;
"~*\bWebCopier\b" 3; "~*\bWebCopier\b" 3;
"~*\bWeb\ Copier\b" 3;
"~*\bWEBDAV\b" 3; "~*\bWEBDAV\b" 3;
"~*\bWEBDAV\ Client\b" 3;
"~*\bWebEnhancer\b" 3; "~*\bWebEnhancer\b" 3;
"~*\bWeb\ Enhancer\b" 3; "~*\bWeb\ Enhancer\b" 3;
"~*\bWebFetch\b" 3; "~*\bWebFetch\b" 3;
@ -673,28 +661,23 @@ map $http_user_agent $bad_bot {
"~*\bWeb\ Fuck\b" 3; "~*\bWeb\ Fuck\b" 3;
"~*\bWebGo\ IS\b" 3; "~*\bWebGo\ IS\b" 3;
"~*\bWebImageCollector\b" 3; "~*\bWebImageCollector\b" 3;
"~*\bWeb\ Image\ Collector\b" 3;
"~*\bWebLeacher\b" 3; "~*\bWebLeacher\b" 3;
"~*\bWebmasterWorldForumBot\b" 3; "~*\bWebmasterWorldForumBot\b" 3;
"~*\bwebmeup-crawler\b" 3; "~*\bwebmeup-crawler\b" 3;
"~*\bWebPix\b" 3; "~*\bWebPix\b" 3;
"~*\bWeb\ Pix\b" 3; "~*\bWeb\ Pix\b" 3;
"~*\bWebReaper\b" 3; "~*\bWebReaper\b" 3;
"~*\bWeb\ Reaper\b" 3;
"~*\bWebSauger\b" 3; "~*\bWebSauger\b" 3;
"~*\bWeb\ Sauger\b" 3; "~*\bWeb\ Sauger\b" 3;
"~*\bWebshag\b" 3; "~*\bWebshag\b" 3;
"~*\bWebsiteExtractor\b" 3; "~*\bWebsiteExtractor\b" 3;
"~*\bWebsite\ Extractor\b" 3;
"~*\bWebsiteQuester\b" 3; "~*\bWebsiteQuester\b" 3;
"~*\bWebsite\ Quester\b" 3; "~*\bWebsite\ Quester\b" 3;
"~*\bWebster\b" 3; "~*\bWebster\b" 3;
"~*\bWebStripper\b" 3; "~*\bWebStripper\b" 3;
"~*\bWeb\ Stripper\b" 3;
"~*\bWebSucker\b" 3; "~*\bWebSucker\b" 3;
"~*\bWeb\ Sucker\b" 3; "~*\bWeb\ Sucker\b" 3;
"~*\bWebWhacker\b" 3; "~*\bWebWhacker\b" 3;
"~*\bWeb\ Whacker\b" 3;
"~*\bWebZIP\b" 3; "~*\bWebZIP\b" 3;
"~*\bWeSEE\b" 3; "~*\bWeSEE\b" 3;
"~*\bWhack\b" 3; "~*\bWhack\b" 3;

2
.dev-tools/_conf_files_test3/bots.d/whitelist-ips.conf
View file

@ -26,4 +26,4 @@
# Only add actual IP addresses and ranges here # Only add actual IP addresses and ranges here
111.111.111.111 0; 111.111.111.111 0;
35.202.124.16 0; 35.202.33.153 0;

31
.dev-tools/_conf_files_test3/conf.d/globalblacklist.conf
View file

@ -4,10 +4,10 @@
### VERSION INFORMATION # ### VERSION INFORMATION #
################################################### ###################################################
### Version: V3.2017.10.979 ### Version: V3.2017.11.980
### Updated: Thu Oct 26 14:47:07 SAST 2017 ### Updated: Wed Nov 1 12:59:37 SAST 2017
### Bad Referrer Count: 5904 ### Bad Referrer Count: 5904
### Bad Bot Count: 513 ### Bad Bot Count: 496
################################################### ###################################################
### VERSION INFORMATION ## ### VERSION INFORMATION ##
@ -259,7 +259,6 @@ map $http_user_agent $bad_bot {
"~*\bBolt\b" 3; "~*\bBolt\b" 3;
"~*\bBotALot\b" 3; "~*\bBotALot\b" 3;
"~*\bBrandprotect\b" 3; "~*\bBrandprotect\b" 3;
"~*\bBrandProtect\b" 3;
"~*\bBrandwatch\b" 3; "~*\bBrandwatch\b" 3;
"~*\bBubing\b" 3; "~*\bBubing\b" 3;
"~*\bBuddy\b" 3; "~*\bBuddy\b" 3;
@ -289,7 +288,6 @@ map $http_user_agent $bad_bot {
"~*\bCosmos\b" 3; "~*\bCosmos\b" 3;
"~*\bCraftbot\b" 3; "~*\bCraftbot\b" 3;
"~*\bCrazyWebCrawler\b" 3; "~*\bCrazyWebCrawler\b" 3;
"~*\bCRAZYWEBCRAWLER\b" 3;
"~*\bCrescent\b" 3; "~*\bCrescent\b" 3;
"~*\bCSHttp\b" 3; "~*\bCSHttp\b" 3;
"~*\bCurious\b" 3; "~*\bCurious\b" 3;
@ -313,8 +311,6 @@ map $http_user_agent $bad_bot {
"~*\bDomainSigmaCrawler\b" 3; "~*\bDomainSigmaCrawler\b" 3;
"~*\bDomainStatsBot\b" 3; "~*\bDomainStatsBot\b" 3;
"~*\bDotbot\b" 3; "~*\bDotbot\b" 3;
"~*\bDownload\ Demon\b" 3;
"~*\bDownload\ Devil\b" 3;
"~*\bDownload\ Wonder\b" 3; "~*\bDownload\ Wonder\b" 3;
"~*\bDragonfly\b" 3; "~*\bDragonfly\b" 3;
"~*\bDrip\b" 3; "~*\bDrip\b" 3;
@ -325,8 +321,6 @@ map $http_user_agent $bad_bot {
"~*\bECCP/1.0\b" 3; "~*\bECCP/1.0\b" 3;
"~*\bEcxi\b" 3; "~*\bEcxi\b" 3;
"~*\bEirGrabber\b" 3; "~*\bEirGrabber\b" 3;
"~*\bEMail\ Collector\b" 3;
"~*\bEMail\ Extractor\b" 3;
"~*\bEMail\ Siphon\b" 3; "~*\bEMail\ Siphon\b" 3;
"~*\bEMail\ Wolf\b" 3; "~*\bEMail\ Wolf\b" 3;
"~*\bEroCrawler\b" 3; "~*\bEroCrawler\b" 3;
@ -386,7 +380,6 @@ map $http_user_agent $bad_bot {
"~*\bId-search\b" 3; "~*\bId-search\b" 3;
"~*\bIlseBot\b" 3; "~*\bIlseBot\b" 3;
"~*\bImage\ Fetch\b" 3; "~*\bImage\ Fetch\b" 3;
"~*\bImage\ Stripper\b" 3;
"~*\bImage\ Sucker\b" 3; "~*\bImage\ Sucker\b" 3;
"~*\bIndy\ Library\b" 3; "~*\bIndy\ Library\b" 3;
"~*\bInfoNaviRobot\b" 3; "~*\bInfoNaviRobot\b" 3;
@ -507,7 +500,6 @@ map $http_user_agent $bad_bot {
"~*\bPageAnalyzer\b" 3; "~*\bPageAnalyzer\b" 3;
"~*\bPage\ Analyzer\b" 3; "~*\bPage\ Analyzer\b" 3;
"~*\bPageGrabber\b" 3; "~*\bPageGrabber\b" 3;
"~*\bPage\ Grabber\b" 3;
"~*\bpage\ scorer\b" 3; "~*\bpage\ scorer\b" 3;
"~*\bPageScorer\b" 3; "~*\bPageScorer\b" 3;
"~*\bPanscient\b" 3; "~*\bPanscient\b" 3;
@ -558,14 +550,12 @@ map $http_user_agent $bad_bot {
"~*\bScanbot\b" 3; "~*\bScanbot\b" 3;
"~*\bScrapy\b" 3; "~*\bScrapy\b" 3;
"~*\bScreaming\b" 3; "~*\bScreaming\b" 3;
"~*\bScreaming\ Frog\ SEO\ Spider\b" 3;
"~*\bScreenerBot\b" 3; "~*\bScreenerBot\b" 3;
"~*\bSearchestate\b" 3; "~*\bSearchestate\b" 3;
"~*\bSearchmetricsBot\b" 3; "~*\bSearchmetricsBot\b" 3;
"~*\bSemrush\b" 3; "~*\bSemrush\b" 3;
"~*\bSemrushBot\b" 3; "~*\bSemrushBot\b" 3;
"~*\bSEOkicks\b" 3; "~*\bSEOkicks\b" 3;
"~*\bSEOkicks-Robot\b" 3;
"~*\bSEOlyticsCrawler\b" 3; "~*\bSEOlyticsCrawler\b" 3;
"~*\bSeomoz\b" 3; "~*\bSeomoz\b" 3;
"~*\bSEOprofiler\b" 3; "~*\bSEOprofiler\b" 3;
@ -573,7 +563,6 @@ map $http_user_agent $bad_bot {
"~*\bSEOstats\b" 3; "~*\bSEOstats\b" 3;
"~*\bSiphon\b" 3; "~*\bSiphon\b" 3;
"~*\bSISTRIX\b" 3; "~*\bSISTRIX\b" 3;
"~*\bSISTRIX\ Crawler\b" 3;
"~*\bSitebeam\b" 3; "~*\bSitebeam\b" 3;
"~*\bSiteExplorer\b" 3; "~*\bSiteExplorer\b" 3;
"~*\bSiteimprove\b" 3; "~*\bSiteimprove\b" 3;
@ -628,12 +617,14 @@ map $http_user_agent $bad_bot {
"~*\bTitan\b" 3; "~*\bTitan\b" 3;
"~*\bToata\b" 3; "~*\bToata\b" 3;
"~*\bToweyabot\b" 3; "~*\bToweyabot\b" 3;
"~*\bTrendiction\b" 3;
"~*\bTrendictionbot\b" 3; "~*\bTrendictionbot\b" 3;
"~*\btrendiction.com\b" 3;
"~*\btrendiction.de\b" 3;
"~*\bTrue_Robot\b" 3; "~*\bTrue_Robot\b" 3;
"~*\bTuringos\b" 3; "~*\bTuringos\b" 3;
"~*\bTurnitin\b" 3;
"~*\bTurnitinBot\b" 3; "~*\bTurnitinBot\b" 3;
"~*\bTurnitin\ Bot\b" 3;
"~*\bTurnitin\ Robot\b" 3;
"~*\bTwengaBot\b" 3; "~*\bTwengaBot\b" 3;
"~*\bTwice\b" 3; "~*\bTwice\b" 3;
"~*\bTyphoeus\b" 3; "~*\bTyphoeus\b" 3;
@ -658,13 +649,10 @@ map $http_user_agent $bad_bot {
"~*\bWebAuto\b" 3; "~*\bWebAuto\b" 3;
"~*\bWeb\ Auto\b" 3; "~*\bWeb\ Auto\b" 3;
"~*\bWebBandit\b" 3; "~*\bWebBandit\b" 3;
"~*\bWeb\ Bandit\b" 3;
"~*\bWebCollage\b" 3; "~*\bWebCollage\b" 3;
"~*\bWeb\ Collage\b" 3; "~*\bWeb\ Collage\b" 3;
"~*\bWebCopier\b" 3; "~*\bWebCopier\b" 3;
"~*\bWeb\ Copier\b" 3;
"~*\bWEBDAV\b" 3; "~*\bWEBDAV\b" 3;
"~*\bWEBDAV\ Client\b" 3;
"~*\bWebEnhancer\b" 3; "~*\bWebEnhancer\b" 3;
"~*\bWeb\ Enhancer\b" 3; "~*\bWeb\ Enhancer\b" 3;
"~*\bWebFetch\b" 3; "~*\bWebFetch\b" 3;
@ -673,28 +661,23 @@ map $http_user_agent $bad_bot {
"~*\bWeb\ Fuck\b" 3; "~*\bWeb\ Fuck\b" 3;
"~*\bWebGo\ IS\b" 3; "~*\bWebGo\ IS\b" 3;
"~*\bWebImageCollector\b" 3; "~*\bWebImageCollector\b" 3;
"~*\bWeb\ Image\ Collector\b" 3;
"~*\bWebLeacher\b" 3; "~*\bWebLeacher\b" 3;
"~*\bWebmasterWorldForumBot\b" 3; "~*\bWebmasterWorldForumBot\b" 3;
"~*\bwebmeup-crawler\b" 3; "~*\bwebmeup-crawler\b" 3;
"~*\bWebPix\b" 3; "~*\bWebPix\b" 3;
"~*\bWeb\ Pix\b" 3; "~*\bWeb\ Pix\b" 3;
"~*\bWebReaper\b" 3; "~*\bWebReaper\b" 3;
"~*\bWeb\ Reaper\b" 3;
"~*\bWebSauger\b" 3; "~*\bWebSauger\b" 3;
"~*\bWeb\ Sauger\b" 3; "~*\bWeb\ Sauger\b" 3;
"~*\bWebshag\b" 3; "~*\bWebshag\b" 3;
"~*\bWebsiteExtractor\b" 3; "~*\bWebsiteExtractor\b" 3;
"~*\bWebsite\ Extractor\b" 3;
"~*\bWebsiteQuester\b" 3; "~*\bWebsiteQuester\b" 3;
"~*\bWebsite\ Quester\b" 3; "~*\bWebsite\ Quester\b" 3;
"~*\bWebster\b" 3; "~*\bWebster\b" 3;
"~*\bWebStripper\b" 3; "~*\bWebStripper\b" 3;
"~*\bWeb\ Stripper\b" 3;
"~*\bWebSucker\b" 3; "~*\bWebSucker\b" 3;
"~*\bWeb\ Sucker\b" 3; "~*\bWeb\ Sucker\b" 3;
"~*\bWebWhacker\b" 3; "~*\bWebWhacker\b" 3;
"~*\bWeb\ Whacker\b" 3;
"~*\bWebZIP\b" 3; "~*\bWebZIP\b" 3;
"~*\bWeSEE\b" 3; "~*\bWeSEE\b" 3;
"~*\bWhack\b" 3; "~*\bWhack\b" 3;

2
.dev-tools/_conf_files_test4/bots.d/whitelist-ips.conf
View file

@ -26,4 +26,4 @@
# Only add actual IP addresses and ranges here # Only add actual IP addresses and ranges here
111.111.111.111 0; 111.111.111.111 0;
35.202.124.16 0; 35.202.33.153 0;

31
.dev-tools/_conf_files_test4/conf.d/globalblacklist.conf
View file

@ -4,10 +4,10 @@
### VERSION INFORMATION # ### VERSION INFORMATION #
################################################### ###################################################
### Version: V3.2017.10.979 ### Version: V3.2017.11.980
### Updated: Thu Oct 26 14:47:07 SAST 2017 ### Updated: Wed Nov 1 12:59:37 SAST 2017
### Bad Referrer Count: 5904 ### Bad Referrer Count: 5904
### Bad Bot Count: 513 ### Bad Bot Count: 496
################################################### ###################################################
### VERSION INFORMATION ## ### VERSION INFORMATION ##
@ -259,7 +259,6 @@ map $http_user_agent $bad_bot {
"~*\bBolt\b" 3; "~*\bBolt\b" 3;
"~*\bBotALot\b" 3; "~*\bBotALot\b" 3;
"~*\bBrandprotect\b" 3; "~*\bBrandprotect\b" 3;
"~*\bBrandProtect\b" 3;
"~*\bBrandwatch\b" 3; "~*\bBrandwatch\b" 3;
"~*\bBubing\b" 3; "~*\bBubing\b" 3;
"~*\bBuddy\b" 3; "~*\bBuddy\b" 3;
@ -289,7 +288,6 @@ map $http_user_agent $bad_bot {
"~*\bCosmos\b" 3; "~*\bCosmos\b" 3;
"~*\bCraftbot\b" 3; "~*\bCraftbot\b" 3;
"~*\bCrazyWebCrawler\b" 3; "~*\bCrazyWebCrawler\b" 3;
"~*\bCRAZYWEBCRAWLER\b" 3;
"~*\bCrescent\b" 3; "~*\bCrescent\b" 3;
"~*\bCSHttp\b" 3; "~*\bCSHttp\b" 3;
"~*\bCurious\b" 3; "~*\bCurious\b" 3;
@ -313,8 +311,6 @@ map $http_user_agent $bad_bot {
"~*\bDomainSigmaCrawler\b" 3; "~*\bDomainSigmaCrawler\b" 3;
"~*\bDomainStatsBot\b" 3; "~*\bDomainStatsBot\b" 3;
"~*\bDotbot\b" 3; "~*\bDotbot\b" 3;
"~*\bDownload\ Demon\b" 3;
"~*\bDownload\ Devil\b" 3;
"~*\bDownload\ Wonder\b" 3; "~*\bDownload\ Wonder\b" 3;
"~*\bDragonfly\b" 3; "~*\bDragonfly\b" 3;
"~*\bDrip\b" 3; "~*\bDrip\b" 3;
@ -325,8 +321,6 @@ map $http_user_agent $bad_bot {
"~*\bECCP/1.0\b" 3; "~*\bECCP/1.0\b" 3;
"~*\bEcxi\b" 3; "~*\bEcxi\b" 3;
"~*\bEirGrabber\b" 3; "~*\bEirGrabber\b" 3;
"~*\bEMail\ Collector\b" 3;
"~*\bEMail\ Extractor\b" 3;
"~*\bEMail\ Siphon\b" 3; "~*\bEMail\ Siphon\b" 3;
"~*\bEMail\ Wolf\b" 3; "~*\bEMail\ Wolf\b" 3;
"~*\bEroCrawler\b" 3; "~*\bEroCrawler\b" 3;
@ -386,7 +380,6 @@ map $http_user_agent $bad_bot {
"~*\bId-search\b" 3; "~*\bId-search\b" 3;
"~*\bIlseBot\b" 3; "~*\bIlseBot\b" 3;
"~*\bImage\ Fetch\b" 3; "~*\bImage\ Fetch\b" 3;
"~*\bImage\ Stripper\b" 3;
"~*\bImage\ Sucker\b" 3; "~*\bImage\ Sucker\b" 3;
"~*\bIndy\ Library\b" 3; "~*\bIndy\ Library\b" 3;
"~*\bInfoNaviRobot\b" 3; "~*\bInfoNaviRobot\b" 3;
@ -507,7 +500,6 @@ map $http_user_agent $bad_bot {
"~*\bPageAnalyzer\b" 3; "~*\bPageAnalyzer\b" 3;
"~*\bPage\ Analyzer\b" 3; "~*\bPage\ Analyzer\b" 3;
"~*\bPageGrabber\b" 3; "~*\bPageGrabber\b" 3;
"~*\bPage\ Grabber\b" 3;
"~*\bpage\ scorer\b" 3; "~*\bpage\ scorer\b" 3;
"~*\bPageScorer\b" 3; "~*\bPageScorer\b" 3;
"~*\bPanscient\b" 3; "~*\bPanscient\b" 3;
@ -558,14 +550,12 @@ map $http_user_agent $bad_bot {
"~*\bScanbot\b" 3; "~*\bScanbot\b" 3;
"~*\bScrapy\b" 3; "~*\bScrapy\b" 3;
"~*\bScreaming\b" 3; "~*\bScreaming\b" 3;
"~*\bScreaming\ Frog\ SEO\ Spider\b" 3;
"~*\bScreenerBot\b" 3; "~*\bScreenerBot\b" 3;
"~*\bSearchestate\b" 3; "~*\bSearchestate\b" 3;
"~*\bSearchmetricsBot\b" 3; "~*\bSearchmetricsBot\b" 3;
"~*\bSemrush\b" 3; "~*\bSemrush\b" 3;
"~*\bSemrushBot\b" 3; "~*\bSemrushBot\b" 3;
"~*\bSEOkicks\b" 3; "~*\bSEOkicks\b" 3;
"~*\bSEOkicks-Robot\b" 3;
"~*\bSEOlyticsCrawler\b" 3; "~*\bSEOlyticsCrawler\b" 3;
"~*\bSeomoz\b" 3; "~*\bSeomoz\b" 3;
"~*\bSEOprofiler\b" 3; "~*\bSEOprofiler\b" 3;
@ -573,7 +563,6 @@ map $http_user_agent $bad_bot {
"~*\bSEOstats\b" 3; "~*\bSEOstats\b" 3;
"~*\bSiphon\b" 3; "~*\bSiphon\b" 3;
"~*\bSISTRIX\b" 3; "~*\bSISTRIX\b" 3;
"~*\bSISTRIX\ Crawler\b" 3;
"~*\bSitebeam\b" 3; "~*\bSitebeam\b" 3;
"~*\bSiteExplorer\b" 3; "~*\bSiteExplorer\b" 3;
"~*\bSiteimprove\b" 3; "~*\bSiteimprove\b" 3;
@ -628,12 +617,14 @@ map $http_user_agent $bad_bot {
"~*\bTitan\b" 3; "~*\bTitan\b" 3;
"~*\bToata\b" 3; "~*\bToata\b" 3;
"~*\bToweyabot\b" 3; "~*\bToweyabot\b" 3;
"~*\bTrendiction\b" 3;
"~*\bTrendictionbot\b" 3; "~*\bTrendictionbot\b" 3;
"~*\btrendiction.com\b" 3;
"~*\btrendiction.de\b" 3;
"~*\bTrue_Robot\b" 3; "~*\bTrue_Robot\b" 3;
"~*\bTuringos\b" 3; "~*\bTuringos\b" 3;
"~*\bTurnitin\b" 3;
"~*\bTurnitinBot\b" 3; "~*\bTurnitinBot\b" 3;
"~*\bTurnitin\ Bot\b" 3;
"~*\bTurnitin\ Robot\b" 3;
"~*\bTwengaBot\b" 3; "~*\bTwengaBot\b" 3;
"~*\bTwice\b" 3; "~*\bTwice\b" 3;
"~*\bTyphoeus\b" 3; "~*\bTyphoeus\b" 3;
@ -658,13 +649,10 @@ map $http_user_agent $bad_bot {
"~*\bWebAuto\b" 3; "~*\bWebAuto\b" 3;
"~*\bWeb\ Auto\b" 3; "~*\bWeb\ Auto\b" 3;
"~*\bWebBandit\b" 3; "~*\bWebBandit\b" 3;
"~*\bWeb\ Bandit\b" 3;
"~*\bWebCollage\b" 3; "~*\bWebCollage\b" 3;
"~*\bWeb\ Collage\b" 3; "~*\bWeb\ Collage\b" 3;
"~*\bWebCopier\b" 3; "~*\bWebCopier\b" 3;
"~*\bWeb\ Copier\b" 3;
"~*\bWEBDAV\b" 3; "~*\bWEBDAV\b" 3;
"~*\bWEBDAV\ Client\b" 3;
"~*\bWebEnhancer\b" 3; "~*\bWebEnhancer\b" 3;
"~*\bWeb\ Enhancer\b" 3; "~*\bWeb\ Enhancer\b" 3;
"~*\bWebFetch\b" 3; "~*\bWebFetch\b" 3;
@ -673,28 +661,23 @@ map $http_user_agent $bad_bot {
"~*\bWeb\ Fuck\b" 3; "~*\bWeb\ Fuck\b" 3;
"~*\bWebGo\ IS\b" 3; "~*\bWebGo\ IS\b" 3;
"~*\bWebImageCollector\b" 3; "~*\bWebImageCollector\b" 3;
"~*\bWeb\ Image\ Collector\b" 3;
"~*\bWebLeacher\b" 3; "~*\bWebLeacher\b" 3;
"~*\bWebmasterWorldForumBot\b" 3; "~*\bWebmasterWorldForumBot\b" 3;
"~*\bwebmeup-crawler\b" 3; "~*\bwebmeup-crawler\b" 3;
"~*\bWebPix\b" 3; "~*\bWebPix\b" 3;
"~*\bWeb\ Pix\b" 3; "~*\bWeb\ Pix\b" 3;
"~*\bWebReaper\b" 3; "~*\bWebReaper\b" 3;
"~*\bWeb\ Reaper\b" 3;
"~*\bWebSauger\b" 3; "~*\bWebSauger\b" 3;
"~*\bWeb\ Sauger\b" 3; "~*\bWeb\ Sauger\b" 3;
"~*\bWebshag\b" 3; "~*\bWebshag\b" 3;
"~*\bWebsiteExtractor\b" 3; "~*\bWebsiteExtractor\b" 3;
"~*\bWebsite\ Extractor\b" 3;
"~*\bWebsiteQuester\b" 3; "~*\bWebsiteQuester\b" 3;
"~*\bWebsite\ Quester\b" 3; "~*\bWebsite\ Quester\b" 3;
"~*\bWebster\b" 3; "~*\bWebster\b" 3;
"~*\bWebStripper\b" 3; "~*\bWebStripper\b" 3;
"~*\bWeb\ Stripper\b" 3;
"~*\bWebSucker\b" 3; "~*\bWebSucker\b" 3;
"~*\bWeb\ Sucker\b" 3; "~*\bWeb\ Sucker\b" 3;
"~*\bWebWhacker\b" 3; "~*\bWebWhacker\b" 3;
"~*\bWeb\ Whacker\b" 3;
"~*\bWebZIP\b" 3; "~*\bWebZIP\b" 3;
"~*\bWeSEE\b" 3; "~*\bWeSEE\b" 3;
"~*\bWhack\b" 3; "~*\bWhack\b" 3;

2
.dev-tools/_curl_tests/curltest1.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:12 SAST 2017 Last Tested: Wed Nov 1 12:59:43 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests/curltest2.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:12 SAST 2017 Last Tested: Wed Nov 1 12:59:43 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests/curltest3.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:12 SAST 2017 Last Tested: Wed Nov 1 12:59:43 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests/curltest4.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:12 SAST 2017 Last Tested: Wed Nov 1 12:59:43 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests/curltest5.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:12 SAST 2017 Last Tested: Wed Nov 1 12:59:43 SAST 2017
<?php <?php

2
.dev-tools/_curl_tests/curltest6.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:12 SAST 2017 Last Tested: Wed Nov 1 12:59:43 SAST 2017
<?php <?php

2
.dev-tools/_curl_tests/curltest7.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:12 SAST 2017 Last Tested: Wed Nov 1 12:59:43 SAST 2017
<?php <?php

2
.dev-tools/_curl_tests/curltest8.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:12 SAST 2017 Last Tested: Wed Nov 1 12:59:43 SAST 2017
<?php <?php

2
.dev-tools/_curl_tests_2/curltest1.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:17 SAST 2017 Last Tested: Wed Nov 1 12:59:47 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests_2/curltest2.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:17 SAST 2017 Last Tested: Wed Nov 1 12:59:47 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests_2/curltest3.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:17 SAST 2017 Last Tested: Wed Nov 1 12:59:47 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests_2/curltest4.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:17 SAST 2017 Last Tested: Wed Nov 1 12:59:47 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests_2/curltest5.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:17 SAST 2017 Last Tested: Wed Nov 1 12:59:47 SAST 2017
<?php <?php

2
.dev-tools/_curl_tests_2/curltest6.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:17 SAST 2017 Last Tested: Wed Nov 1 12:59:47 SAST 2017
<?php <?php

2
.dev-tools/_curl_tests_2/curltest7.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:17 SAST 2017 Last Tested: Wed Nov 1 12:59:47 SAST 2017
<?php <?php

2
.dev-tools/_curl_tests_2/curltest8.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:17 SAST 2017 Last Tested: Wed Nov 1 12:59:47 SAST 2017
<?php <?php

2
.dev-tools/_curl_tests_3/curltest1.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:21 SAST 2017 Last Tested: Wed Nov 1 12:59:52 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests_3/curltest2.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:21 SAST 2017 Last Tested: Wed Nov 1 12:59:52 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests_3/curltest3.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:21 SAST 2017 Last Tested: Wed Nov 1 12:59:52 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests_3/curltest4.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:21 SAST 2017 Last Tested: Wed Nov 1 12:59:52 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests_3/curltest5.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:21 SAST 2017 Last Tested: Wed Nov 1 12:59:52 SAST 2017
<?php <?php

2
.dev-tools/_curl_tests_3/curltest6.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:21 SAST 2017 Last Tested: Wed Nov 1 12:59:52 SAST 2017
<?php <?php

2
.dev-tools/_curl_tests_3/curltest7.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:21 SAST 2017 Last Tested: Wed Nov 1 12:59:52 SAST 2017
<?php <?php

2
.dev-tools/_curl_tests_3/curltest8.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:21 SAST 2017 Last Tested: Wed Nov 1 12:59:52 SAST 2017
<?php <?php

2
.dev-tools/_curl_tests_4/curltest1.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:51 SAST 2017 Last Tested: Wed Nov 1 13:00:26 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests_4/curltest2.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:51 SAST 2017 Last Tested: Wed Nov 1 13:00:26 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests_4/curltest3.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:51 SAST 2017 Last Tested: Wed Nov 1 13:00:26 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests_4/curltest4.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:51 SAST 2017 Last Tested: Wed Nov 1 13:00:26 SAST 2017
% Total % Received % Xferd Average Speed Time Time Time Current % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed Dload Upload Total Spent Left Speed

2
.dev-tools/_curl_tests_4/curltest5.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:51 SAST 2017 Last Tested: Wed Nov 1 13:00:26 SAST 2017
<!DOCTYPE html> <!DOCTYPE html>
<html> <html>

2
.dev-tools/_curl_tests_4/curltest6.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:51 SAST 2017 Last Tested: Wed Nov 1 13:00:26 SAST 2017
<!DOCTYPE html> <!DOCTYPE html>
<html> <html>

2
.dev-tools/_curl_tests_4/curltest7.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:51 SAST 2017 Last Tested: Wed Nov 1 13:00:26 SAST 2017
<!DOCTYPE html> <!DOCTYPE html>
<html> <html>

2
.dev-tools/_curl_tests_4/curltest8.txt
View file

@ -1,4 +1,4 @@
Last Tested: Thu Oct 26 14:47:51 SAST 2017 Last Tested: Wed Nov 1 13:00:26 SAST 2017
<!DOCTYPE html> <!DOCTYPE html>
<html> <html>

25
.dev-tools/_robots_input/robots-input.txt
View file

@ -43,7 +43,6 @@ Boardreader
Bolt Bolt
BotALot BotALot
Brandprotect Brandprotect
BrandProtect
Brandwatch Brandwatch
Bubing Bubing
Buddy Buddy
@ -73,7 +72,6 @@ Copyscape
Cosmos Cosmos
Craftbot Craftbot
CrazyWebCrawler CrazyWebCrawler
CRAZYWEBCRAWLER
Crescent Crescent
CSHttp CSHttp
Curious Curious
@ -97,8 +95,6 @@ DomainCrawler
DomainSigmaCrawler DomainSigmaCrawler
DomainStatsBot DomainStatsBot
Dotbot Dotbot
Download Demon
Download Devil
Download Wonder Download Wonder
Dragonfly Dragonfly
Drip Drip
@ -109,8 +105,6 @@ eCatch
ECCP/1.0 ECCP/1.0
Ecxi Ecxi
EirGrabber EirGrabber
EMail Collector
EMail Extractor
EMail Siphon EMail Siphon
EMail Wolf EMail Wolf
EroCrawler EroCrawler
@ -170,7 +164,6 @@ IDBot
Id-search Id-search
IlseBot IlseBot
Image Fetch Image Fetch
Image Stripper
Image Sucker Image Sucker
Indy Library Indy Library
InfoNaviRobot InfoNaviRobot
@ -291,7 +284,6 @@ OutfoxBot
PageAnalyzer PageAnalyzer
Page Analyzer Page Analyzer
PageGrabber PageGrabber
Page Grabber
page scorer page scorer
PageScorer PageScorer
Panscient Panscient
@ -342,14 +334,12 @@ ScanAlert
Scanbot Scanbot
Scrapy Scrapy
Screaming Screaming
Screaming Frog SEO Spider
ScreenerBot ScreenerBot
Searchestate Searchestate
SearchmetricsBot SearchmetricsBot
Semrush Semrush
SemrushBot SemrushBot
SEOkicks SEOkicks
SEOkicks-Robot
SEOlyticsCrawler SEOlyticsCrawler
Seomoz Seomoz
SEOprofiler SEOprofiler
@ -357,7 +347,6 @@ seoscanners
SEOstats SEOstats
Siphon Siphon
SISTRIX SISTRIX
SISTRIX Crawler
Sitebeam Sitebeam
SiteExplorer SiteExplorer
Siteimprove Siteimprove
@ -412,12 +401,14 @@ TightTwatBot
Titan Titan
Toata Toata
Toweyabot Toweyabot
Trendiction
Trendictionbot Trendictionbot
trendiction.com
trendiction.de
True_Robot True_Robot
Turingos Turingos
Turnitin
TurnitinBot TurnitinBot
Turnitin Bot
Turnitin Robot
TwengaBot TwengaBot
Twice Twice
Typhoeus Typhoeus
@ -442,13 +433,10 @@ Webalta
WebAuto WebAuto
Web Auto Web Auto
WebBandit WebBandit
Web Bandit
WebCollage WebCollage
Web Collage Web Collage
WebCopier WebCopier
Web Copier
WEBDAV WEBDAV
WEBDAV Client
WebEnhancer WebEnhancer
Web Enhancer Web Enhancer
WebFetch WebFetch
@ -457,28 +445,23 @@ WebFuck
Web Fuck Web Fuck
WebGo IS WebGo IS
WebImageCollector WebImageCollector
Web Image Collector
WebLeacher WebLeacher
WebmasterWorldForumBot WebmasterWorldForumBot
webmeup-crawler webmeup-crawler
WebPix WebPix
Web Pix Web Pix
WebReaper WebReaper
Web Reaper
WebSauger WebSauger
Web Sauger Web Sauger
Webshag Webshag
WebsiteExtractor WebsiteExtractor
Website Extractor
WebsiteQuester WebsiteQuester
Website Quester Website Quester
Webster Webster
WebStripper WebStripper
Web Stripper
WebSucker WebSucker
Web Sucker Web Sucker
WebWhacker WebWhacker
Web Whacker
WebZIP WebZIP
WeSEE WeSEE
Whack Whack

31
.dev-tools/globalblacklist.template
View file

@ -4,10 +4,10 @@
### VERSION INFORMATION # ### VERSION INFORMATION #
################################################### ###################################################
### Version: V3.2017.10.979 ### Version: V3.2017.11.980
### Updated: Thu Oct 26 14:47:07 SAST 2017 ### Updated: Wed Nov 1 12:59:37 SAST 2017
### Bad Referrer Count: 5904 ### Bad Referrer Count: 5904
### Bad Bot Count: 513 ### Bad Bot Count: 496
################################################### ###################################################
### VERSION INFORMATION ## ### VERSION INFORMATION ##
@ -259,7 +259,6 @@ map $http_user_agent $bad_bot {
"~*\bBolt\b" 3; "~*\bBolt\b" 3;
"~*\bBotALot\b" 3; "~*\bBotALot\b" 3;
"~*\bBrandprotect\b" 3; "~*\bBrandprotect\b" 3;
"~*\bBrandProtect\b" 3;
"~*\bBrandwatch\b" 3; "~*\bBrandwatch\b" 3;
"~*\bBubing\b" 3; "~*\bBubing\b" 3;
"~*\bBuddy\b" 3; "~*\bBuddy\b" 3;
@ -289,7 +288,6 @@ map $http_user_agent $bad_bot {
"~*\bCosmos\b" 3; "~*\bCosmos\b" 3;
"~*\bCraftbot\b" 3; "~*\bCraftbot\b" 3;
"~*\bCrazyWebCrawler\b" 3; "~*\bCrazyWebCrawler\b" 3;
"~*\bCRAZYWEBCRAWLER\b" 3;
"~*\bCrescent\b" 3; "~*\bCrescent\b" 3;
"~*\bCSHttp\b" 3; "~*\bCSHttp\b" 3;
"~*\bCurious\b" 3; "~*\bCurious\b" 3;
@ -313,8 +311,6 @@ map $http_user_agent $bad_bot {
"~*\bDomainSigmaCrawler\b" 3; "~*\bDomainSigmaCrawler\b" 3;
"~*\bDomainStatsBot\b" 3; "~*\bDomainStatsBot\b" 3;
"~*\bDotbot\b" 3; "~*\bDotbot\b" 3;
"~*\bDownload\ Demon\b" 3;
"~*\bDownload\ Devil\b" 3;
"~*\bDownload\ Wonder\b" 3; "~*\bDownload\ Wonder\b" 3;
"~*\bDragonfly\b" 3; "~*\bDragonfly\b" 3;
"~*\bDrip\b" 3; "~*\bDrip\b" 3;
@ -325,8 +321,6 @@ map $http_user_agent $bad_bot {
"~*\bECCP/1.0\b" 3; "~*\bECCP/1.0\b" 3;
"~*\bEcxi\b" 3; "~*\bEcxi\b" 3;
"~*\bEirGrabber\b" 3; "~*\bEirGrabber\b" 3;
"~*\bEMail\ Collector\b" 3;
"~*\bEMail\ Extractor\b" 3;
"~*\bEMail\ Siphon\b" 3; "~*\bEMail\ Siphon\b" 3;
"~*\bEMail\ Wolf\b" 3; "~*\bEMail\ Wolf\b" 3;
"~*\bEroCrawler\b" 3; "~*\bEroCrawler\b" 3;
@ -386,7 +380,6 @@ map $http_user_agent $bad_bot {
"~*\bId-search\b" 3; "~*\bId-search\b" 3;
"~*\bIlseBot\b" 3; "~*\bIlseBot\b" 3;
"~*\bImage\ Fetch\b" 3; "~*\bImage\ Fetch\b" 3;
"~*\bImage\ Stripper\b" 3;
"~*\bImage\ Sucker\b" 3; "~*\bImage\ Sucker\b" 3;
"~*\bIndy\ Library\b" 3; "~*\bIndy\ Library\b" 3;
"~*\bInfoNaviRobot\b" 3; "~*\bInfoNaviRobot\b" 3;
@ -507,7 +500,6 @@ map $http_user_agent $bad_bot {
"~*\bPageAnalyzer\b" 3; "~*\bPageAnalyzer\b" 3;
"~*\bPage\ Analyzer\b" 3; "~*\bPage\ Analyzer\b" 3;
"~*\bPageGrabber\b" 3; "~*\bPageGrabber\b" 3;
"~*\bPage\ Grabber\b" 3;
"~*\bpage\ scorer\b" 3; "~*\bpage\ scorer\b" 3;
"~*\bPageScorer\b" 3; "~*\bPageScorer\b" 3;
"~*\bPanscient\b" 3; "~*\bPanscient\b" 3;
@ -558,14 +550,12 @@ map $http_user_agent $bad_bot {
"~*\bScanbot\b" 3; "~*\bScanbot\b" 3;
"~*\bScrapy\b" 3; "~*\bScrapy\b" 3;
"~*\bScreaming\b" 3; "~*\bScreaming\b" 3;
"~*\bScreaming\ Frog\ SEO\ Spider\b" 3;
"~*\bScreenerBot\b" 3; "~*\bScreenerBot\b" 3;
"~*\bSearchestate\b" 3; "~*\bSearchestate\b" 3;
"~*\bSearchmetricsBot\b" 3; "~*\bSearchmetricsBot\b" 3;
"~*\bSemrush\b" 3; "~*\bSemrush\b" 3;
"~*\bSemrushBot\b" 3; "~*\bSemrushBot\b" 3;
"~*\bSEOkicks\b" 3; "~*\bSEOkicks\b" 3;
"~*\bSEOkicks-Robot\b" 3;
"~*\bSEOlyticsCrawler\b" 3; "~*\bSEOlyticsCrawler\b" 3;
"~*\bSeomoz\b" 3; "~*\bSeomoz\b" 3;
"~*\bSEOprofiler\b" 3; "~*\bSEOprofiler\b" 3;
@ -573,7 +563,6 @@ map $http_user_agent $bad_bot {
"~*\bSEOstats\b" 3; "~*\bSEOstats\b" 3;
"~*\bSiphon\b" 3; "~*\bSiphon\b" 3;
"~*\bSISTRIX\b" 3; "~*\bSISTRIX\b" 3;
"~*\bSISTRIX\ Crawler\b" 3;
"~*\bSitebeam\b" 3; "~*\bSitebeam\b" 3;
"~*\bSiteExplorer\b" 3; "~*\bSiteExplorer\b" 3;
"~*\bSiteimprove\b" 3; "~*\bSiteimprove\b" 3;
@ -628,12 +617,14 @@ map $http_user_agent $bad_bot {
"~*\bTitan\b" 3; "~*\bTitan\b" 3;
"~*\bToata\b" 3; "~*\bToata\b" 3;
"~*\bToweyabot\b" 3; "~*\bToweyabot\b" 3;
"~*\bTrendiction\b" 3;
"~*\bTrendictionbot\b" 3; "~*\bTrendictionbot\b" 3;
"~*\btrendiction.com\b" 3;
"~*\btrendiction.de\b" 3;
"~*\bTrue_Robot\b" 3; "~*\bTrue_Robot\b" 3;
"~*\bTuringos\b" 3; "~*\bTuringos\b" 3;
"~*\bTurnitin\b" 3;
"~*\bTurnitinBot\b" 3; "~*\bTurnitinBot\b" 3;
"~*\bTurnitin\ Bot\b" 3;
"~*\bTurnitin\ Robot\b" 3;
"~*\bTwengaBot\b" 3; "~*\bTwengaBot\b" 3;
"~*\bTwice\b" 3; "~*\bTwice\b" 3;
"~*\bTyphoeus\b" 3; "~*\bTyphoeus\b" 3;
@ -658,13 +649,10 @@ map $http_user_agent $bad_bot {
"~*\bWebAuto\b" 3; "~*\bWebAuto\b" 3;
"~*\bWeb\ Auto\b" 3; "~*\bWeb\ Auto\b" 3;
"~*\bWebBandit\b" 3; "~*\bWebBandit\b" 3;
"~*\bWeb\ Bandit\b" 3;
"~*\bWebCollage\b" 3; "~*\bWebCollage\b" 3;
"~*\bWeb\ Collage\b" 3; "~*\bWeb\ Collage\b" 3;
"~*\bWebCopier\b" 3; "~*\bWebCopier\b" 3;
"~*\bWeb\ Copier\b" 3;
"~*\bWEBDAV\b" 3; "~*\bWEBDAV\b" 3;
"~*\bWEBDAV\ Client\b" 3;
"~*\bWebEnhancer\b" 3; "~*\bWebEnhancer\b" 3;
"~*\bWeb\ Enhancer\b" 3; "~*\bWeb\ Enhancer\b" 3;
"~*\bWebFetch\b" 3; "~*\bWebFetch\b" 3;
@ -673,28 +661,23 @@ map $http_user_agent $bad_bot {
"~*\bWeb\ Fuck\b" 3; "~*\bWeb\ Fuck\b" 3;
"~*\bWebGo\ IS\b" 3; "~*\bWebGo\ IS\b" 3;
"~*\bWebImageCollector\b" 3; "~*\bWebImageCollector\b" 3;
"~*\bWeb\ Image\ Collector\b" 3;
"~*\bWebLeacher\b" 3; "~*\bWebLeacher\b" 3;
"~*\bWebmasterWorldForumBot\b" 3; "~*\bWebmasterWorldForumBot\b" 3;
"~*\bwebmeup-crawler\b" 3; "~*\bwebmeup-crawler\b" 3;
"~*\bWebPix\b" 3; "~*\bWebPix\b" 3;
"~*\bWeb\ Pix\b" 3; "~*\bWeb\ Pix\b" 3;
"~*\bWebReaper\b" 3; "~*\bWebReaper\b" 3;
"~*\bWeb\ Reaper\b" 3;
"~*\bWebSauger\b" 3; "~*\bWebSauger\b" 3;
"~*\bWeb\ Sauger\b" 3; "~*\bWeb\ Sauger\b" 3;
"~*\bWebshag\b" 3; "~*\bWebshag\b" 3;
"~*\bWebsiteExtractor\b" 3; "~*\bWebsiteExtractor\b" 3;
"~*\bWebsite\ Extractor\b" 3;
"~*\bWebsiteQuester\b" 3; "~*\bWebsiteQuester\b" 3;
"~*\bWebsite\ Quester\b" 3; "~*\bWebsite\ Quester\b" 3;
"~*\bWebster\b" 3; "~*\bWebster\b" 3;
"~*\bWebStripper\b" 3; "~*\bWebStripper\b" 3;
"~*\bWeb\ Stripper\b" 3;
"~*\bWebSucker\b" 3; "~*\bWebSucker\b" 3;
"~*\bWeb\ Sucker\b" 3; "~*\bWeb\ Sucker\b" 3;
"~*\bWebWhacker\b" 3; "~*\bWebWhacker\b" 3;
"~*\bWeb\ Whacker\b" 3;
"~*\bWebZIP\b" 3; "~*\bWebZIP\b" 3;
"~*\bWeSEE\b" 3; "~*\bWeSEE\b" 3;
"~*\bWhack\b" 3; "~*\bWhack\b" 3;

BIN
.latest_release/bots.d.tar.gz
View file

Binary file not shown.

BIN
.latest_release/conf.d.tar.gz
View file

Binary file not shown.

4
AUTO-CONFIGURATION.md
View file

@ -4,9 +4,9 @@
### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation:
_______________ _______________
#### Version: V3.2017.10.979 #### Version: V3.2017.11.980
#### Bad Referrer Count: 5904 #### Bad Referrer Count: 5904
#### Bad Bot Count: 513 #### Bad Bot Count: 496
____________________ ____________________

4
MANUAL-CONFIGURATION.md
View file

@ -4,9 +4,9 @@
### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation:
_______________ _______________
#### Version: V3.2017.10.979 #### Version: V3.2017.11.980
#### Bad Referrer Count: 5904 #### Bad Referrer Count: 5904
#### Bad Bot Count: 513 #### Bad Bot Count: 496
____________________ ____________________
## Help Support This Project ## Help Support This Project

4
README.md
View file

@ -4,9 +4,9 @@
##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic from ever reaching your web sites. ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic from ever reaching your web sites.
_______________ _______________
#### Version: V3.2017.10.979 #### Version: V3.2017.11.980
#### Bad Referrer Count: 5904 #### Bad Referrer Count: 5904
#### Bad Bot Count: 513 #### Bad Bot Count: 496
____________________ ____________________
## Help Support This Project ## Help Support This Project

4
_google_analytics_ghost_spam/README.md
View file

@ -3,9 +3,9 @@
# EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM
_______________ _______________
#### Version: V3.2017.10.979 #### Version: V3.2017.11.980
#### Bad Referrer Count: 5904 #### Bad Referrer Count: 5904
#### Bad Bot Count: 513 #### Bad Bot Count: 496
____________________ ____________________
## Help Support This Project ## Help Support This Project

31
_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf
View file

@ -4,10 +4,10 @@
### VERSION INFORMATION # ### VERSION INFORMATION #
################################################### ###################################################
### Version: V3.2017.10.979 ### Version: V3.2017.11.980
### Updated: Thu Oct 26 14:47:07 SAST 2017 ### Updated: Wed Nov 1 12:59:37 SAST 2017
### Bad Referrer Count: 5904 ### Bad Referrer Count: 5904
### Bad Bot Count: 513 ### Bad Bot Count: 496
################################################### ###################################################
### VERSION INFORMATION ## ### VERSION INFORMATION ##
@ -259,7 +259,6 @@ map $http_user_agent $bad_bot {
"~*\bBolt\b" 3; "~*\bBolt\b" 3;
"~*\bBotALot\b" 3; "~*\bBotALot\b" 3;
"~*\bBrandprotect\b" 3; "~*\bBrandprotect\b" 3;
"~*\bBrandProtect\b" 3;
"~*\bBrandwatch\b" 3; "~*\bBrandwatch\b" 3;
"~*\bBubing\b" 3; "~*\bBubing\b" 3;
"~*\bBuddy\b" 3; "~*\bBuddy\b" 3;
@ -289,7 +288,6 @@ map $http_user_agent $bad_bot {
"~*\bCosmos\b" 3; "~*\bCosmos\b" 3;
"~*\bCraftbot\b" 3; "~*\bCraftbot\b" 3;
"~*\bCrazyWebCrawler\b" 3; "~*\bCrazyWebCrawler\b" 3;
"~*\bCRAZYWEBCRAWLER\b" 3;
"~*\bCrescent\b" 3; "~*\bCrescent\b" 3;
"~*\bCSHttp\b" 3; "~*\bCSHttp\b" 3;
"~*\bCurious\b" 3; "~*\bCurious\b" 3;
@ -313,8 +311,6 @@ map $http_user_agent $bad_bot {
"~*\bDomainSigmaCrawler\b" 3; "~*\bDomainSigmaCrawler\b" 3;
"~*\bDomainStatsBot\b" 3; "~*\bDomainStatsBot\b" 3;
"~*\bDotbot\b" 3; "~*\bDotbot\b" 3;
"~*\bDownload\ Demon\b" 3;
"~*\bDownload\ Devil\b" 3;
"~*\bDownload\ Wonder\b" 3; "~*\bDownload\ Wonder\b" 3;
"~*\bDragonfly\b" 3; "~*\bDragonfly\b" 3;
"~*\bDrip\b" 3; "~*\bDrip\b" 3;
@ -325,8 +321,6 @@ map $http_user_agent $bad_bot {
"~*\bECCP/1.0\b" 3; "~*\bECCP/1.0\b" 3;
"~*\bEcxi\b" 3; "~*\bEcxi\b" 3;
"~*\bEirGrabber\b" 3; "~*\bEirGrabber\b" 3;
"~*\bEMail\ Collector\b" 3;
"~*\bEMail\ Extractor\b" 3;
"~*\bEMail\ Siphon\b" 3; "~*\bEMail\ Siphon\b" 3;
"~*\bEMail\ Wolf\b" 3; "~*\bEMail\ Wolf\b" 3;
"~*\bEroCrawler\b" 3; "~*\bEroCrawler\b" 3;
@ -386,7 +380,6 @@ map $http_user_agent $bad_bot {
"~*\bId-search\b" 3; "~*\bId-search\b" 3;
"~*\bIlseBot\b" 3; "~*\bIlseBot\b" 3;
"~*\bImage\ Fetch\b" 3; "~*\bImage\ Fetch\b" 3;
"~*\bImage\ Stripper\b" 3;
"~*\bImage\ Sucker\b" 3; "~*\bImage\ Sucker\b" 3;
"~*\bIndy\ Library\b" 3; "~*\bIndy\ Library\b" 3;
"~*\bInfoNaviRobot\b" 3; "~*\bInfoNaviRobot\b" 3;
@ -507,7 +500,6 @@ map $http_user_agent $bad_bot {
"~*\bPageAnalyzer\b" 3; "~*\bPageAnalyzer\b" 3;
"~*\bPage\ Analyzer\b" 3; "~*\bPage\ Analyzer\b" 3;
"~*\bPageGrabber\b" 3; "~*\bPageGrabber\b" 3;
"~*\bPage\ Grabber\b" 3;
"~*\bpage\ scorer\b" 3; "~*\bpage\ scorer\b" 3;
"~*\bPageScorer\b" 3; "~*\bPageScorer\b" 3;
"~*\bPanscient\b" 3; "~*\bPanscient\b" 3;
@ -558,14 +550,12 @@ map $http_user_agent $bad_bot {
"~*\bScanbot\b" 3; "~*\bScanbot\b" 3;
"~*\bScrapy\b" 3; "~*\bScrapy\b" 3;
"~*\bScreaming\b" 3; "~*\bScreaming\b" 3;
"~*\bScreaming\ Frog\ SEO\ Spider\b" 3;
"~*\bScreenerBot\b" 3; "~*\bScreenerBot\b" 3;
"~*\bSearchestate\b" 3; "~*\bSearchestate\b" 3;
"~*\bSearchmetricsBot\b" 3; "~*\bSearchmetricsBot\b" 3;
"~*\bSemrush\b" 3; "~*\bSemrush\b" 3;
"~*\bSemrushBot\b" 3; "~*\bSemrushBot\b" 3;
"~*\bSEOkicks\b" 3; "~*\bSEOkicks\b" 3;
"~*\bSEOkicks-Robot\b" 3;
"~*\bSEOlyticsCrawler\b" 3; "~*\bSEOlyticsCrawler\b" 3;
"~*\bSeomoz\b" 3; "~*\bSeomoz\b" 3;
"~*\bSEOprofiler\b" 3; "~*\bSEOprofiler\b" 3;
@ -573,7 +563,6 @@ map $http_user_agent $bad_bot {
"~*\bSEOstats\b" 3; "~*\bSEOstats\b" 3;
"~*\bSiphon\b" 3; "~*\bSiphon\b" 3;
"~*\bSISTRIX\b" 3; "~*\bSISTRIX\b" 3;
"~*\bSISTRIX\ Crawler\b" 3;
"~*\bSitebeam\b" 3; "~*\bSitebeam\b" 3;
"~*\bSiteExplorer\b" 3; "~*\bSiteExplorer\b" 3;
"~*\bSiteimprove\b" 3; "~*\bSiteimprove\b" 3;
@ -628,12 +617,14 @@ map $http_user_agent $bad_bot {
"~*\bTitan\b" 3; "~*\bTitan\b" 3;
"~*\bToata\b" 3; "~*\bToata\b" 3;
"~*\bToweyabot\b" 3; "~*\bToweyabot\b" 3;
"~*\bTrendiction\b" 3;
"~*\bTrendictionbot\b" 3; "~*\bTrendictionbot\b" 3;
"~*\btrendiction.com\b" 3;
"~*\btrendiction.de\b" 3;
"~*\bTrue_Robot\b" 3; "~*\bTrue_Robot\b" 3;
"~*\bTuringos\b" 3; "~*\bTuringos\b" 3;
"~*\bTurnitin\b" 3;
"~*\bTurnitinBot\b" 3; "~*\bTurnitinBot\b" 3;
"~*\bTurnitin\ Bot\b" 3;
"~*\bTurnitin\ Robot\b" 3;
"~*\bTwengaBot\b" 3; "~*\bTwengaBot\b" 3;
"~*\bTwice\b" 3; "~*\bTwice\b" 3;
"~*\bTyphoeus\b" 3; "~*\bTyphoeus\b" 3;
@ -658,13 +649,10 @@ map $http_user_agent $bad_bot {
"~*\bWebAuto\b" 3; "~*\bWebAuto\b" 3;
"~*\bWeb\ Auto\b" 3; "~*\bWeb\ Auto\b" 3;
"~*\bWebBandit\b" 3; "~*\bWebBandit\b" 3;
"~*\bWeb\ Bandit\b" 3;
"~*\bWebCollage\b" 3; "~*\bWebCollage\b" 3;
"~*\bWeb\ Collage\b" 3; "~*\bWeb\ Collage\b" 3;
"~*\bWebCopier\b" 3; "~*\bWebCopier\b" 3;
"~*\bWeb\ Copier\b" 3;
"~*\bWEBDAV\b" 3; "~*\bWEBDAV\b" 3;
"~*\bWEBDAV\ Client\b" 3;
"~*\bWebEnhancer\b" 3; "~*\bWebEnhancer\b" 3;
"~*\bWeb\ Enhancer\b" 3; "~*\bWeb\ Enhancer\b" 3;
"~*\bWebFetch\b" 3; "~*\bWebFetch\b" 3;
@ -673,28 +661,23 @@ map $http_user_agent $bad_bot {
"~*\bWeb\ Fuck\b" 3; "~*\bWeb\ Fuck\b" 3;
"~*\bWebGo\ IS\b" 3; "~*\bWebGo\ IS\b" 3;
"~*\bWebImageCollector\b" 3; "~*\bWebImageCollector\b" 3;
"~*\bWeb\ Image\ Collector\b" 3;
"~*\bWebLeacher\b" 3; "~*\bWebLeacher\b" 3;
"~*\bWebmasterWorldForumBot\b" 3; "~*\bWebmasterWorldForumBot\b" 3;
"~*\bwebmeup-crawler\b" 3; "~*\bwebmeup-crawler\b" 3;
"~*\bWebPix\b" 3; "~*\bWebPix\b" 3;
"~*\bWeb\ Pix\b" 3; "~*\bWeb\ Pix\b" 3;
"~*\bWebReaper\b" 3; "~*\bWebReaper\b" 3;
"~*\bWeb\ Reaper\b" 3;
"~*\bWebSauger\b" 3; "~*\bWebSauger\b" 3;
"~*\bWeb\ Sauger\b" 3; "~*\bWeb\ Sauger\b" 3;
"~*\bWebshag\b" 3; "~*\bWebshag\b" 3;
"~*\bWebsiteExtractor\b" 3; "~*\bWebsiteExtractor\b" 3;
"~*\bWebsite\ Extractor\b" 3;
"~*\bWebsiteQuester\b" 3; "~*\bWebsiteQuester\b" 3;
"~*\bWebsite\ Quester\b" 3; "~*\bWebsite\ Quester\b" 3;
"~*\bWebster\b" 3; "~*\bWebster\b" 3;
"~*\bWebStripper\b" 3; "~*\bWebStripper\b" 3;
"~*\bWeb\ Stripper\b" 3;
"~*\bWebSucker\b" 3; "~*\bWebSucker\b" 3;
"~*\bWeb\ Sucker\b" 3; "~*\bWeb\ Sucker\b" 3;
"~*\bWebWhacker\b" 3; "~*\bWebWhacker\b" 3;
"~*\bWeb\ Whacker\b" 3;
"~*\bWebZIP\b" 3; "~*\bWebZIP\b" 3;
"~*\bWeSEE\b" 3; "~*\bWeSEE\b" 3;
"~*\bWhack\b" 3; "~*\bWhack\b" 3;

31
conf.d/globalblacklist.conf
View file

@ -4,10 +4,10 @@
### VERSION INFORMATION # ### VERSION INFORMATION #
################################################### ###################################################
### Version: V3.2017.10.979 ### Version: V3.2017.11.980
### Updated: Thu Oct 26 14:47:07 SAST 2017 ### Updated: Wed Nov 1 12:59:37 SAST 2017
### Bad Referrer Count: 5904 ### Bad Referrer Count: 5904
### Bad Bot Count: 513 ### Bad Bot Count: 496
################################################### ###################################################
### VERSION INFORMATION ## ### VERSION INFORMATION ##
@ -259,7 +259,6 @@ map $http_user_agent $bad_bot {
"~*\bBolt\b" 3; "~*\bBolt\b" 3;
"~*\bBotALot\b" 3; "~*\bBotALot\b" 3;
"~*\bBrandprotect\b" 3; "~*\bBrandprotect\b" 3;
"~*\bBrandProtect\b" 3;
"~*\bBrandwatch\b" 3; "~*\bBrandwatch\b" 3;
"~*\bBubing\b" 3; "~*\bBubing\b" 3;
"~*\bBuddy\b" 3; "~*\bBuddy\b" 3;
@ -289,7 +288,6 @@ map $http_user_agent $bad_bot {
"~*\bCosmos\b" 3; "~*\bCosmos\b" 3;
"~*\bCraftbot\b" 3; "~*\bCraftbot\b" 3;
"~*\bCrazyWebCrawler\b" 3; "~*\bCrazyWebCrawler\b" 3;
"~*\bCRAZYWEBCRAWLER\b" 3;
"~*\bCrescent\b" 3; "~*\bCrescent\b" 3;
"~*\bCSHttp\b" 3; "~*\bCSHttp\b" 3;
"~*\bCurious\b" 3; "~*\bCurious\b" 3;
@ -313,8 +311,6 @@ map $http_user_agent $bad_bot {
"~*\bDomainSigmaCrawler\b" 3; "~*\bDomainSigmaCrawler\b" 3;
"~*\bDomainStatsBot\b" 3; "~*\bDomainStatsBot\b" 3;
"~*\bDotbot\b" 3; "~*\bDotbot\b" 3;
"~*\bDownload\ Demon\b" 3;
"~*\bDownload\ Devil\b" 3;
"~*\bDownload\ Wonder\b" 3; "~*\bDownload\ Wonder\b" 3;
"~*\bDragonfly\b" 3; "~*\bDragonfly\b" 3;
"~*\bDrip\b" 3; "~*\bDrip\b" 3;
@ -325,8 +321,6 @@ map $http_user_agent $bad_bot {
"~*\bECCP/1.0\b" 3; "~*\bECCP/1.0\b" 3;
"~*\bEcxi\b" 3; "~*\bEcxi\b" 3;
"~*\bEirGrabber\b" 3; "~*\bEirGrabber\b" 3;
"~*\bEMail\ Collector\b" 3;
"~*\bEMail\ Extractor\b" 3;
"~*\bEMail\ Siphon\b" 3; "~*\bEMail\ Siphon\b" 3;
"~*\bEMail\ Wolf\b" 3; "~*\bEMail\ Wolf\b" 3;
"~*\bEroCrawler\b" 3; "~*\bEroCrawler\b" 3;
@ -386,7 +380,6 @@ map $http_user_agent $bad_bot {
"~*\bId-search\b" 3; "~*\bId-search\b" 3;
"~*\bIlseBot\b" 3; "~*\bIlseBot\b" 3;
"~*\bImage\ Fetch\b" 3; "~*\bImage\ Fetch\b" 3;
"~*\bImage\ Stripper\b" 3;
"~*\bImage\ Sucker\b" 3; "~*\bImage\ Sucker\b" 3;
"~*\bIndy\ Library\b" 3; "~*\bIndy\ Library\b" 3;
"~*\bInfoNaviRobot\b" 3; "~*\bInfoNaviRobot\b" 3;
@ -507,7 +500,6 @@ map $http_user_agent $bad_bot {
"~*\bPageAnalyzer\b" 3; "~*\bPageAnalyzer\b" 3;
"~*\bPage\ Analyzer\b" 3; "~*\bPage\ Analyzer\b" 3;
"~*\bPageGrabber\b" 3; "~*\bPageGrabber\b" 3;
"~*\bPage\ Grabber\b" 3;
"~*\bpage\ scorer\b" 3; "~*\bpage\ scorer\b" 3;
"~*\bPageScorer\b" 3; "~*\bPageScorer\b" 3;
"~*\bPanscient\b" 3; "~*\bPanscient\b" 3;
@ -558,14 +550,12 @@ map $http_user_agent $bad_bot {
"~*\bScanbot\b" 3; "~*\bScanbot\b" 3;
"~*\bScrapy\b" 3; "~*\bScrapy\b" 3;
"~*\bScreaming\b" 3; "~*\bScreaming\b" 3;
"~*\bScreaming\ Frog\ SEO\ Spider\b" 3;
"~*\bScreenerBot\b" 3; "~*\bScreenerBot\b" 3;
"~*\bSearchestate\b" 3; "~*\bSearchestate\b" 3;
"~*\bSearchmetricsBot\b" 3; "~*\bSearchmetricsBot\b" 3;
"~*\bSemrush\b" 3; "~*\bSemrush\b" 3;
"~*\bSemrushBot\b" 3; "~*\bSemrushBot\b" 3;
"~*\bSEOkicks\b" 3; "~*\bSEOkicks\b" 3;
"~*\bSEOkicks-Robot\b" 3;
"~*\bSEOlyticsCrawler\b" 3; "~*\bSEOlyticsCrawler\b" 3;
"~*\bSeomoz\b" 3; "~*\bSeomoz\b" 3;
"~*\bSEOprofiler\b" 3; "~*\bSEOprofiler\b" 3;
@ -573,7 +563,6 @@ map $http_user_agent $bad_bot {
"~*\bSEOstats\b" 3; "~*\bSEOstats\b" 3;
"~*\bSiphon\b" 3; "~*\bSiphon\b" 3;
"~*\bSISTRIX\b" 3; "~*\bSISTRIX\b" 3;
"~*\bSISTRIX\ Crawler\b" 3;
"~*\bSitebeam\b" 3; "~*\bSitebeam\b" 3;
"~*\bSiteExplorer\b" 3; "~*\bSiteExplorer\b" 3;
"~*\bSiteimprove\b" 3; "~*\bSiteimprove\b" 3;
@ -628,12 +617,14 @@ map $http_user_agent $bad_bot {
"~*\bTitan\b" 3; "~*\bTitan\b" 3;
"~*\bToata\b" 3; "~*\bToata\b" 3;
"~*\bToweyabot\b" 3; "~*\bToweyabot\b" 3;
"~*\bTrendiction\b" 3;
"~*\bTrendictionbot\b" 3; "~*\bTrendictionbot\b" 3;
"~*\btrendiction.com\b" 3;
"~*\btrendiction.de\b" 3;
"~*\bTrue_Robot\b" 3; "~*\bTrue_Robot\b" 3;
"~*\bTuringos\b" 3; "~*\bTuringos\b" 3;
"~*\bTurnitin\b" 3;
"~*\bTurnitinBot\b" 3; "~*\bTurnitinBot\b" 3;
"~*\bTurnitin\ Bot\b" 3;
"~*\bTurnitin\ Robot\b" 3;
"~*\bTwengaBot\b" 3; "~*\bTwengaBot\b" 3;
"~*\bTwice\b" 3; "~*\bTwice\b" 3;
"~*\bTyphoeus\b" 3; "~*\bTyphoeus\b" 3;
@ -658,13 +649,10 @@ map $http_user_agent $bad_bot {
"~*\bWebAuto\b" 3; "~*\bWebAuto\b" 3;
"~*\bWeb\ Auto\b" 3; "~*\bWeb\ Auto\b" 3;
"~*\bWebBandit\b" 3; "~*\bWebBandit\b" 3;
"~*\bWeb\ Bandit\b" 3;
"~*\bWebCollage\b" 3; "~*\bWebCollage\b" 3;
"~*\bWeb\ Collage\b" 3; "~*\bWeb\ Collage\b" 3;
"~*\bWebCopier\b" 3; "~*\bWebCopier\b" 3;
"~*\bWeb\ Copier\b" 3;
"~*\bWEBDAV\b" 3; "~*\bWEBDAV\b" 3;
"~*\bWEBDAV\ Client\b" 3;
"~*\bWebEnhancer\b" 3; "~*\bWebEnhancer\b" 3;
"~*\bWeb\ Enhancer\b" 3; "~*\bWeb\ Enhancer\b" 3;
"~*\bWebFetch\b" 3; "~*\bWebFetch\b" 3;
@ -673,28 +661,23 @@ map $http_user_agent $bad_bot {
"~*\bWeb\ Fuck\b" 3; "~*\bWeb\ Fuck\b" 3;
"~*\bWebGo\ IS\b" 3; "~*\bWebGo\ IS\b" 3;
"~*\bWebImageCollector\b" 3; "~*\bWebImageCollector\b" 3;
"~*\bWeb\ Image\ Collector\b" 3;
"~*\bWebLeacher\b" 3; "~*\bWebLeacher\b" 3;
"~*\bWebmasterWorldForumBot\b" 3; "~*\bWebmasterWorldForumBot\b" 3;
"~*\bwebmeup-crawler\b" 3; "~*\bwebmeup-crawler\b" 3;
"~*\bWebPix\b" 3; "~*\bWebPix\b" 3;
"~*\bWeb\ Pix\b" 3; "~*\bWeb\ Pix\b" 3;
"~*\bWebReaper\b" 3; "~*\bWebReaper\b" 3;
"~*\bWeb\ Reaper\b" 3;
"~*\bWebSauger\b" 3; "~*\bWebSauger\b" 3;
"~*\bWeb\ Sauger\b" 3; "~*\bWeb\ Sauger\b" 3;
"~*\bWebshag\b" 3; "~*\bWebshag\b" 3;
"~*\bWebsiteExtractor\b" 3; "~*\bWebsiteExtractor\b" 3;
"~*\bWebsite\ Extractor\b" 3;
"~*\bWebsiteQuester\b" 3; "~*\bWebsiteQuester\b" 3;
"~*\bWebsite\ Quester\b" 3; "~*\bWebsite\ Quester\b" 3;
"~*\bWebster\b" 3; "~*\bWebster\b" 3;
"~*\bWebStripper\b" 3; "~*\bWebStripper\b" 3;
"~*\bWeb\ Stripper\b" 3;
"~*\bWebSucker\b" 3; "~*\bWebSucker\b" 3;
"~*\bWeb\ Sucker\b" 3; "~*\bWeb\ Sucker\b" 3;
"~*\bWebWhacker\b" 3; "~*\bWebWhacker\b" 3;
"~*\bWeb\ Whacker\b" 3;
"~*\bWebZIP\b" 3; "~*\bWebZIP\b" 3;
"~*\bWeSEE\b" 3; "~*\bWeSEE\b" 3;
"~*\bWhack\b" 3; "~*\bWhack\b" 3;

56
robots.txt/robots.txt
View file

@ -1,8 +1,8 @@
### Version Information # ### Version Information #
################################################### ###################################################
### Version: V3.2017.10.979 ### Version: V3.2017.11.980
### Updated: Thu Oct 26 14:47:07 SAST 2017 ### Updated: Wed Nov 1 12:59:38 SAST 2017
### Bad Bot Count: 513 ### Bad Bot Count: 496
################################################### ###################################################
### Version Information ## ### Version Information ##
@ -99,8 +99,6 @@ User-agent: BotALot
Disallow:/ Disallow:/
User-agent: Brandprotect User-agent: Brandprotect
Disallow:/ Disallow:/
User-agent: BrandProtect
Disallow:/
User-agent: Brandwatch User-agent: Brandwatch
Disallow:/ Disallow:/
User-agent: Bubing User-agent: Bubing
@ -159,8 +157,6 @@ User-agent: Craftbot
Disallow:/ Disallow:/
User-agent: CrazyWebCrawler User-agent: CrazyWebCrawler
Disallow:/ Disallow:/
User-agent: CRAZYWEBCRAWLER
Disallow:/
User-agent: Crescent User-agent: Crescent
Disallow:/ Disallow:/
User-agent: CSHttp User-agent: CSHttp
@ -207,10 +203,6 @@ User-agent: DomainStatsBot
Disallow:/ Disallow:/
User-agent: Dotbot User-agent: Dotbot
Disallow:/ Disallow:/
User-agent: Download Demon
Disallow:/
User-agent: Download Devil
Disallow:/
User-agent: Download Wonder User-agent: Download Wonder
Disallow:/ Disallow:/
User-agent: Dragonfly User-agent: Dragonfly
@ -231,10 +223,6 @@ User-agent: Ecxi
Disallow:/ Disallow:/
User-agent: EirGrabber User-agent: EirGrabber
Disallow:/ Disallow:/
User-agent: EMail Collector
Disallow:/
User-agent: EMail Extractor
Disallow:/
User-agent: EMail Siphon User-agent: EMail Siphon
Disallow:/ Disallow:/
User-agent: EMail Wolf User-agent: EMail Wolf
@ -353,8 +341,6 @@ User-agent: IlseBot
Disallow:/ Disallow:/
User-agent: Image Fetch User-agent: Image Fetch
Disallow:/ Disallow:/
User-agent: Image Stripper
Disallow:/
User-agent: Image Sucker User-agent: Image Sucker
Disallow:/ Disallow:/
User-agent: Indy Library User-agent: Indy Library
@ -595,8 +581,6 @@ User-agent: Page Analyzer
Disallow:/ Disallow:/
User-agent: PageGrabber User-agent: PageGrabber
Disallow:/ Disallow:/
User-agent: Page Grabber
Disallow:/
User-agent: page scorer User-agent: page scorer
Disallow:/ Disallow:/
User-agent: PageScorer User-agent: PageScorer
@ -697,8 +681,6 @@ User-agent: Scrapy
Disallow:/ Disallow:/
User-agent: Screaming User-agent: Screaming
Disallow:/ Disallow:/
User-agent: Screaming Frog SEO Spider
Disallow:/
User-agent: ScreenerBot User-agent: ScreenerBot
Disallow:/ Disallow:/
User-agent: Searchestate User-agent: Searchestate
@ -711,8 +693,6 @@ User-agent: SemrushBot
Disallow:/ Disallow:/
User-agent: SEOkicks User-agent: SEOkicks
Disallow:/ Disallow:/
User-agent: SEOkicks-Robot
Disallow:/
User-agent: SEOlyticsCrawler User-agent: SEOlyticsCrawler
Disallow:/ Disallow:/
User-agent: Seomoz User-agent: Seomoz
@ -727,8 +707,6 @@ User-agent: Siphon
Disallow:/ Disallow:/
User-agent: SISTRIX User-agent: SISTRIX
Disallow:/ Disallow:/
User-agent: SISTRIX Crawler
Disallow:/
User-agent: Sitebeam User-agent: Sitebeam
Disallow:/ Disallow:/
User-agent: SiteExplorer User-agent: SiteExplorer
@ -837,18 +815,22 @@ User-agent: Toata
Disallow:/ Disallow:/
User-agent: Toweyabot User-agent: Toweyabot
Disallow:/ Disallow:/
User-agent: Trendiction
Disallow:/
User-agent: Trendictionbot User-agent: Trendictionbot
Disallow:/ Disallow:/
User-agent: trendiction.com
Disallow:/
User-agent: trendiction.de
Disallow:/
User-agent: True_Robot User-agent: True_Robot
Disallow:/ Disallow:/
User-agent: Turingos User-agent: Turingos
Disallow:/ Disallow:/
User-agent: Turnitin
Disallow:/
User-agent: TurnitinBot User-agent: TurnitinBot
Disallow:/ Disallow:/
User-agent: Turnitin Bot
Disallow:/
User-agent: Turnitin Robot
Disallow:/
User-agent: TwengaBot User-agent: TwengaBot
Disallow:/ Disallow:/
User-agent: Twice User-agent: Twice
@ -897,20 +879,14 @@ User-agent: Web Auto
Disallow:/ Disallow:/
User-agent: WebBandit User-agent: WebBandit
Disallow:/ Disallow:/
User-agent: Web Bandit
Disallow:/
User-agent: WebCollage User-agent: WebCollage
Disallow:/ Disallow:/
User-agent: Web Collage User-agent: Web Collage
Disallow:/ Disallow:/
User-agent: WebCopier User-agent: WebCopier
Disallow:/ Disallow:/
User-agent: Web Copier
Disallow:/
User-agent: WEBDAV User-agent: WEBDAV
Disallow:/ Disallow:/
User-agent: WEBDAV Client
Disallow:/
User-agent: WebEnhancer User-agent: WebEnhancer
Disallow:/ Disallow:/
User-agent: Web Enhancer User-agent: Web Enhancer
@ -927,8 +903,6 @@ User-agent: WebGo IS
Disallow:/ Disallow:/
User-agent: WebImageCollector User-agent: WebImageCollector
Disallow:/ Disallow:/
User-agent: Web Image Collector
Disallow:/
User-agent: WebLeacher User-agent: WebLeacher
Disallow:/ Disallow:/
User-agent: WebmasterWorldForumBot User-agent: WebmasterWorldForumBot
@ -941,8 +915,6 @@ User-agent: Web Pix
Disallow:/ Disallow:/
User-agent: WebReaper User-agent: WebReaper
Disallow:/ Disallow:/
User-agent: Web Reaper
Disallow:/
User-agent: WebSauger User-agent: WebSauger
Disallow:/ Disallow:/
User-agent: Web Sauger User-agent: Web Sauger
@ -951,8 +923,6 @@ User-agent: Webshag
Disallow:/ Disallow:/
User-agent: WebsiteExtractor User-agent: WebsiteExtractor
Disallow:/ Disallow:/
User-agent: Website Extractor
Disallow:/
User-agent: WebsiteQuester User-agent: WebsiteQuester
Disallow:/ Disallow:/
User-agent: Website Quester User-agent: Website Quester
@ -961,16 +931,12 @@ User-agent: Webster
Disallow:/ Disallow:/
User-agent: WebStripper User-agent: WebStripper
Disallow:/ Disallow:/
User-agent: Web Stripper
Disallow:/
User-agent: WebSucker User-agent: WebSucker
Disallow:/ Disallow:/
User-agent: Web Sucker User-agent: Web Sucker
Disallow:/ Disallow:/
User-agent: WebWhacker User-agent: WebWhacker
Disallow:/ Disallow:/
User-agent: Web Whacker
Disallow:/
User-agent: WebZIP User-agent: WebZIP
Disallow:/ Disallow:/
User-agent: WeSEE User-agent: WeSEE