diff --git a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf index 345f0dd1f..a164e8bb0 100644 --- a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf @@ -26,4 +26,4 @@ # Only add actual IP addresses and ranges here 111.111.111.111 0; -35.184.63.133 0; +35.202.249.48 0; diff --git a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf index a27a8da44..5123b18a3 100644 --- a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.12.1012 -### Updated: Sun Dec 3 10:16:47 SAST 2017 +### Version: V3.2017.12.1013 +### Updated: Wed Dec 20 11:29:46 SAST 2017 ### Bad Referrer Count: 6069 -### Bad Bot Count: 503 +### Bad Bot Count: 510 ################################################### ### VERSION INFORMATION ## @@ -212,6 +212,7 @@ map $http_user_agent $bad_bot { # Known Vulnerability Scanners (now merged into one section) # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### + "~*\b\b" 3; "~*\b360Spider\b" 3; "~*\b80legs\b" 3; "~*\bAbonti\b" 3; @@ -278,6 +279,7 @@ map $http_user_agent $bad_bot { "~*\bClaritybot\b" 3; "~*\bCliqzbot\b" 3; "~*\bCloud\ mapping\b" 3; + "~*\bcoccocbot-web\b" 3; "~*\bCogentbot\b" 3; "~*\bcognitiveseo\b" 3; "~*\bCollector\b" 3; @@ -300,6 +302,7 @@ map $http_user_agent $bad_bot { "~*\bDeusu\b" 3; "~*\bDevil\b" 3; "~*\bDigincore\b" 3; + "~*\bDigitalPebble\b" 3; "~*\bDIIbot\b" 3; "~*\bDirbuster\b" 3; "~*\bDisco\b" 3; @@ -335,6 +338,7 @@ map $http_user_agent $bad_bot { "~*\bEyeNetIE\b" 3; "~*\bEzooms\b" 3; "~*\bFDM\b" 3; + "~*\bFemtosearchBot\b" 3; "~*\bFHscan\b" 3; "~*\bFimap\b" 3; "~*\bFirefox/7.0\b" 3; @@ -392,6 +396,7 @@ map $http_user_agent $bad_bot { "~*\bInternet\ Ninja\b" 3; "~*\bInternetSeer\b" 3; "~*\binternetVista\ monitor\b" 3; + "~*\bips-agent\b" 3; "~*\bIria\b" 3; "~*\bIRLbot\b" 3; "~*\bIskanie\b" 3; @@ -489,6 +494,7 @@ map $http_user_agent $bad_bot { "~*\bNmap\b" 3; "~*\bNPbot\b" 3; "~*\bNutch\b" 3; + "~*\boBot\b" 3; "~*\bOctopus\b" 3; "~*\bOffline\ Explorer\b" 3; "~*\bOffline\ Navigator\b" 3; @@ -578,6 +584,7 @@ map $http_user_agent $bad_bot { "~*\bSlackbot-LinkExpanding\b" 3; "~*\bSlySearch\b" 3; "~*\bSmartDownload\b" 3; + "~*\bSMTBot\b" 3; "~*\bSnake\b" 3; "~*\bSnapbot\b" 3; "~*\bSnoopy\b" 3; diff --git a/.dev-tools/_conf_files_test2/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test2/bots.d/whitelist-ips.conf index 345f0dd1f..a164e8bb0 100644 --- a/.dev-tools/_conf_files_test2/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test2/bots.d/whitelist-ips.conf @@ -26,4 +26,4 @@ # Only add actual IP addresses and ranges here 111.111.111.111 0; -35.184.63.133 0; +35.202.249.48 0; diff --git a/.dev-tools/_conf_files_test2/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test2/conf.d/globalblacklist.conf index a27a8da44..8f17e85fd 100644 --- a/.dev-tools/_conf_files_test2/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test2/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.12.1012 -### Updated: Sun Dec 3 10:16:47 SAST 2017 +### Version: V3.2017.12.1013 +### Updated: Wed Dec 20 11:29:46 SAST 2017 ### Bad Referrer Count: 6069 -### Bad Bot Count: 503 +### Bad Bot Count: 510 ################################################### ### VERSION INFORMATION ## @@ -118,7 +118,7 @@ map $http_user_agent $bad_bot { # START CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### # *********************************************************************************** - include /etc/nginx/bots.d/blacklist-user-agents.conf; + include /usr/local/nginx/bots.d/blacklist-user-agents.conf; # ********************************************************************************* # END CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### @@ -212,6 +212,7 @@ map $http_user_agent $bad_bot { # Known Vulnerability Scanners (now merged into one section) # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### + "~*\b\b" 3; "~*\b360Spider\b" 3; "~*\b80legs\b" 3; "~*\bAbonti\b" 3; @@ -278,6 +279,7 @@ map $http_user_agent $bad_bot { "~*\bClaritybot\b" 3; "~*\bCliqzbot\b" 3; "~*\bCloud\ mapping\b" 3; + "~*\bcoccocbot-web\b" 3; "~*\bCogentbot\b" 3; "~*\bcognitiveseo\b" 3; "~*\bCollector\b" 3; @@ -300,6 +302,7 @@ map $http_user_agent $bad_bot { "~*\bDeusu\b" 3; "~*\bDevil\b" 3; "~*\bDigincore\b" 3; + "~*\bDigitalPebble\b" 3; "~*\bDIIbot\b" 3; "~*\bDirbuster\b" 3; "~*\bDisco\b" 3; @@ -335,6 +338,7 @@ map $http_user_agent $bad_bot { "~*\bEyeNetIE\b" 3; "~*\bEzooms\b" 3; "~*\bFDM\b" 3; + "~*\bFemtosearchBot\b" 3; "~*\bFHscan\b" 3; "~*\bFimap\b" 3; "~*\bFirefox/7.0\b" 3; @@ -392,6 +396,7 @@ map $http_user_agent $bad_bot { "~*\bInternet\ Ninja\b" 3; "~*\bInternetSeer\b" 3; "~*\binternetVista\ monitor\b" 3; + "~*\bips-agent\b" 3; "~*\bIria\b" 3; "~*\bIRLbot\b" 3; "~*\bIskanie\b" 3; @@ -489,6 +494,7 @@ map $http_user_agent $bad_bot { "~*\bNmap\b" 3; "~*\bNPbot\b" 3; "~*\bNutch\b" 3; + "~*\boBot\b" 3; "~*\bOctopus\b" 3; "~*\bOffline\ Explorer\b" 3; "~*\bOffline\ Navigator\b" 3; @@ -578,6 +584,7 @@ map $http_user_agent $bad_bot { "~*\bSlackbot-LinkExpanding\b" 3; "~*\bSlySearch\b" 3; "~*\bSmartDownload\b" 3; + "~*\bSMTBot\b" 3; "~*\bSnake\b" 3; "~*\bSnapbot\b" 3; "~*\bSnoopy\b" 3; @@ -766,7 +773,7 @@ map $http_referer $bad_words { # START WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### # ************************************************************************ - include /etc/nginx/bots.d/whitelist-domains.conf; + include /usr/local/nginx/bots.d/whitelist-domains.conf; # ********************************************************************** # END WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### @@ -777,7 +784,7 @@ map $http_referer $bad_words { # START CUSTOM BAD REFERRER WORDS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### # ****************************************************************************** - include /etc/nginx/bots.d/bad-referrer-words.conf; + include /usr/local/nginx/bots.d/bad-referrer-words.conf; # **************************************************************************** # END CUSTOM BAD REFERRER WORDS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### @@ -823,7 +830,7 @@ map $http_referer $bad_words { # kill all your SEO in a week. # To add your own custom bad referrers use the custom include file -# /etc/nginx/bots.d/custom-bad-referrers.conf +# /usr/local/nginx/bots.d/custom-bad-referrers.conf # Or send a Pull Request to add it to the global blacklist for other users. # In the bad referrers section I also include sites that hotlink images without permission. @@ -852,7 +859,7 @@ map $http_referer $bad_referer { # START WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### # ************************************************************************ - include /etc/nginx/bots.d/whitelist-domains.conf; + include /usr/local/nginx/bots.d/whitelist-domains.conf; # ********************************************************************** # END WHITELISTED DOMAINS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### @@ -869,7 +876,7 @@ map $http_referer $bad_referer { # START CUSTOM BAD REFERRERS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### # ************************************************************************* - include /etc/nginx/bots.d/custom-bad-referrers.conf; + include /usr/local/nginx/bots.d/custom-bad-referrers.conf; # *********************************************************************** # END CUSTOM BAD REFERRERS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### @@ -6980,7 +6987,7 @@ geo $validate_client { # START WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### # ************************************************************************** - include /etc/nginx/bots.d/whitelist-ips.conf; + include /usr/local/nginx/bots.d/whitelist-ips.conf; # ************************************************************************ # END WHITELISTED IP RANGES ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### @@ -7152,7 +7159,7 @@ geo $validate_client { # START BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### # ******************************************************************** - include /etc/nginx/bots.d/blacklist-ips.conf; + include /usr/local/nginx/bots.d/blacklist-ips.conf; # ****************************************************************** # END BLACKLISTED IPS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### @@ -7173,7 +7180,7 @@ geo $ratelimited { # START WHITELISTED IP RANGES2 ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### # *************************************************************************** - include /etc/nginx/bots.d/whitelist-ips.conf; + include /usr/local/nginx/bots.d/whitelist-ips.conf; # ************************************************************************* # END WHITELISTED IP RANGES2 ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### diff --git a/.dev-tools/_conf_files_test3/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test3/bots.d/whitelist-ips.conf index 345f0dd1f..a164e8bb0 100644 --- a/.dev-tools/_conf_files_test3/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test3/bots.d/whitelist-ips.conf @@ -26,4 +26,4 @@ # Only add actual IP addresses and ranges here 111.111.111.111 0; -35.184.63.133 0; +35.202.249.48 0; diff --git a/.dev-tools/_conf_files_test3/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test3/conf.d/globalblacklist.conf index 4b0236e0a..a1d32b1fd 100644 --- a/.dev-tools/_conf_files_test3/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test3/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.12.1012 -### Updated: Sun Dec 3 10:16:47 SAST 2017 +### Version: V3.2017.12.1013 +### Updated: Wed Dec 20 11:29:46 SAST 2017 ### Bad Referrer Count: 6069 -### Bad Bot Count: 503 +### Bad Bot Count: 510 ################################################### ### VERSION INFORMATION ## @@ -212,6 +212,7 @@ map $http_user_agent $bad_bot { # Known Vulnerability Scanners (now merged into one section) # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### + "~*\b\b" 3; "~*\b360Spider\b" 3; "~*\b80legs\b" 3; "~*\bAbonti\b" 3; @@ -278,6 +279,7 @@ map $http_user_agent $bad_bot { "~*\bClaritybot\b" 3; "~*\bCliqzbot\b" 3; "~*\bCloud\ mapping\b" 3; + "~*\bcoccocbot-web\b" 3; "~*\bCogentbot\b" 3; "~*\bcognitiveseo\b" 3; "~*\bCollector\b" 3; @@ -300,6 +302,7 @@ map $http_user_agent $bad_bot { "~*\bDeusu\b" 3; "~*\bDevil\b" 3; "~*\bDigincore\b" 3; + "~*\bDigitalPebble\b" 3; "~*\bDIIbot\b" 3; "~*\bDirbuster\b" 3; "~*\bDisco\b" 3; @@ -335,6 +338,7 @@ map $http_user_agent $bad_bot { "~*\bEyeNetIE\b" 3; "~*\bEzooms\b" 3; "~*\bFDM\b" 3; + "~*\bFemtosearchBot\b" 3; "~*\bFHscan\b" 3; "~*\bFimap\b" 3; "~*\bFirefox/7.0\b" 3; @@ -392,6 +396,7 @@ map $http_user_agent $bad_bot { "~*\bInternet\ Ninja\b" 3; "~*\bInternetSeer\b" 3; "~*\binternetVista\ monitor\b" 3; + "~*\bips-agent\b" 3; "~*\bIria\b" 3; "~*\bIRLbot\b" 3; "~*\bIskanie\b" 3; @@ -489,6 +494,7 @@ map $http_user_agent $bad_bot { "~*\bNmap\b" 3; "~*\bNPbot\b" 3; "~*\bNutch\b" 3; + "~*\boBot\b" 3; "~*\bOctopus\b" 3; "~*\bOffline\ Explorer\b" 3; "~*\bOffline\ Navigator\b" 3; @@ -578,6 +584,7 @@ map $http_user_agent $bad_bot { "~*\bSlackbot-LinkExpanding\b" 3; "~*\bSlySearch\b" 3; "~*\bSmartDownload\b" 3; + "~*\bSMTBot\b" 3; "~*\bSnake\b" 3; "~*\bSnapbot\b" 3; "~*\bSnoopy\b" 3; diff --git a/.dev-tools/_conf_files_test4/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test4/bots.d/whitelist-ips.conf index 345f0dd1f..a164e8bb0 100644 --- a/.dev-tools/_conf_files_test4/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test4/bots.d/whitelist-ips.conf @@ -26,4 +26,4 @@ # Only add actual IP addresses and ranges here 111.111.111.111 0; -35.184.63.133 0; +35.202.249.48 0; diff --git a/.dev-tools/_conf_files_test4/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test4/conf.d/globalblacklist.conf index a27a8da44..5123b18a3 100644 --- a/.dev-tools/_conf_files_test4/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test4/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.12.1012 -### Updated: Sun Dec 3 10:16:47 SAST 2017 +### Version: V3.2017.12.1013 +### Updated: Wed Dec 20 11:29:46 SAST 2017 ### Bad Referrer Count: 6069 -### Bad Bot Count: 503 +### Bad Bot Count: 510 ################################################### ### VERSION INFORMATION ## @@ -212,6 +212,7 @@ map $http_user_agent $bad_bot { # Known Vulnerability Scanners (now merged into one section) # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### + "~*\b\b" 3; "~*\b360Spider\b" 3; "~*\b80legs\b" 3; "~*\bAbonti\b" 3; @@ -278,6 +279,7 @@ map $http_user_agent $bad_bot { "~*\bClaritybot\b" 3; "~*\bCliqzbot\b" 3; "~*\bCloud\ mapping\b" 3; + "~*\bcoccocbot-web\b" 3; "~*\bCogentbot\b" 3; "~*\bcognitiveseo\b" 3; "~*\bCollector\b" 3; @@ -300,6 +302,7 @@ map $http_user_agent $bad_bot { "~*\bDeusu\b" 3; "~*\bDevil\b" 3; "~*\bDigincore\b" 3; + "~*\bDigitalPebble\b" 3; "~*\bDIIbot\b" 3; "~*\bDirbuster\b" 3; "~*\bDisco\b" 3; @@ -335,6 +338,7 @@ map $http_user_agent $bad_bot { "~*\bEyeNetIE\b" 3; "~*\bEzooms\b" 3; "~*\bFDM\b" 3; + "~*\bFemtosearchBot\b" 3; "~*\bFHscan\b" 3; "~*\bFimap\b" 3; "~*\bFirefox/7.0\b" 3; @@ -392,6 +396,7 @@ map $http_user_agent $bad_bot { "~*\bInternet\ Ninja\b" 3; "~*\bInternetSeer\b" 3; "~*\binternetVista\ monitor\b" 3; + "~*\bips-agent\b" 3; "~*\bIria\b" 3; "~*\bIRLbot\b" 3; "~*\bIskanie\b" 3; @@ -489,6 +494,7 @@ map $http_user_agent $bad_bot { "~*\bNmap\b" 3; "~*\bNPbot\b" 3; "~*\bNutch\b" 3; + "~*\boBot\b" 3; "~*\bOctopus\b" 3; "~*\bOffline\ Explorer\b" 3; "~*\bOffline\ Navigator\b" 3; @@ -578,6 +584,7 @@ map $http_user_agent $bad_bot { "~*\bSlackbot-LinkExpanding\b" 3; "~*\bSlySearch\b" 3; "~*\bSmartDownload\b" 3; + "~*\bSMTBot\b" 3; "~*\bSnake\b" 3; "~*\bSnapbot\b" 3; "~*\bSnoopy\b" 3; diff --git a/.dev-tools/_curl_tests/curltest1.txt b/.dev-tools/_curl_tests/curltest1.txt index da06e0258..f286cd25c 100755 --- a/.dev-tools/_curl_tests/curltest1.txt +++ b/.dev-tools/_curl_tests/curltest1.txt @@ -1,4 +1,4 @@ -Last Tested: Sun Dec 3 10:16:53 SAST 2017 +Last Tested: Wed Dec 20 11:29:51 SAST 2017 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest2.txt b/.dev-tools/_curl_tests/curltest2.txt index da06e0258..f286cd25c 100755 --- a/.dev-tools/_curl_tests/curltest2.txt +++ b/.dev-tools/_curl_tests/curltest2.txt @@ -1,4 +1,4 @@ -Last Tested: Sun Dec 3 10:16:53 SAST 2017 +Last Tested: Wed Dec 20 11:29:51 SAST 2017 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest3.txt b/.dev-tools/_curl_tests/curltest3.txt index da06e0258..f286cd25c 100755 --- a/.dev-tools/_curl_tests/curltest3.txt +++ b/.dev-tools/_curl_tests/curltest3.txt @@ -1,4 +1,4 @@ -Last Tested: Sun Dec 3 10:16:53 SAST 2017 +Last Tested: Wed Dec 20 11:29:51 SAST 2017 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest4.txt b/.dev-tools/_curl_tests/curltest4.txt index da06e0258..f286cd25c 100755 --- a/.dev-tools/_curl_tests/curltest4.txt +++ b/.dev-tools/_curl_tests/curltest4.txt @@ -1,4 +1,4 @@ -Last Tested: Sun Dec 3 10:16:53 SAST 2017 +Last Tested: Wed Dec 20 11:29:51 SAST 2017 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest5.txt b/.dev-tools/_curl_tests/curltest5.txt index 65d6b7e86..f20e0c022 100755 --- a/.dev-tools/_curl_tests/curltest5.txt +++ b/.dev-tools/_curl_tests/curltest5.txt @@ -1,4 +1,4 @@ -Last Tested: Sun Dec 3 10:16:53 SAST 2017 +Last Tested: Wed Dec 20 11:29:51 SAST 2017 diff --git a/.dev-tools/_curl_tests_4/curltest6.txt b/.dev-tools/_curl_tests_4/curltest6.txt index 0acc93db2..f5832f70e 100755 --- a/.dev-tools/_curl_tests_4/curltest6.txt +++ b/.dev-tools/_curl_tests_4/curltest6.txt @@ -1,4 +1,4 @@ -Last Tested: Sun Dec 3 10:17:33 SAST 2017 +Last Tested: Wed Dec 20 11:30:31 SAST 2017 diff --git a/.dev-tools/_curl_tests_4/curltest7.txt b/.dev-tools/_curl_tests_4/curltest7.txt index 0acc93db2..f5832f70e 100644 --- a/.dev-tools/_curl_tests_4/curltest7.txt +++ b/.dev-tools/_curl_tests_4/curltest7.txt @@ -1,4 +1,4 @@ -Last Tested: Sun Dec 3 10:17:33 SAST 2017 +Last Tested: Wed Dec 20 11:30:31 SAST 2017 diff --git a/.dev-tools/_curl_tests_4/curltest8.txt b/.dev-tools/_curl_tests_4/curltest8.txt index 0acc93db2..f5832f70e 100644 --- a/.dev-tools/_curl_tests_4/curltest8.txt +++ b/.dev-tools/_curl_tests_4/curltest8.txt @@ -1,4 +1,4 @@ -Last Tested: Sun Dec 3 10:17:33 SAST 2017 +Last Tested: Wed Dec 20 11:30:31 SAST 2017 diff --git a/.dev-tools/_robots_input/robots-input.txt b/.dev-tools/_robots_input/robots-input.txt index 39956557e..5f4f4408b 100644 --- a/.dev-tools/_robots_input/robots-input.txt +++ b/.dev-tools/_robots_input/robots-input.txt @@ -1,3 +1,4 @@ + 360Spider 80legs Abonti @@ -64,6 +65,7 @@ Chlooe Claritybot Cliqzbot Cloud mapping +coccocbot-web Cogentbot cognitiveseo Collector @@ -86,6 +88,7 @@ Demon Deusu Devil Digincore +DigitalPebble DIIbot Dirbuster Disco @@ -121,6 +124,7 @@ Extreme Picture Finder EyeNetIE Ezooms FDM +FemtosearchBot FHscan Fimap Firefox/7.0 @@ -178,6 +182,7 @@ InterGET Internet Ninja InternetSeer internetVista monitor +ips-agent Iria IRLbot Iskanie @@ -275,6 +280,7 @@ Ninja Nmap NPbot Nutch +oBot Octopus Offline Explorer Offline Navigator @@ -364,6 +370,7 @@ Sitevigil Slackbot-LinkExpanding SlySearch SmartDownload +SMTBot Snake Snapbot Snoopy diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index a27a8da44..5123b18a3 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.12.1012 -### Updated: Sun Dec 3 10:16:47 SAST 2017 +### Version: V3.2017.12.1013 +### Updated: Wed Dec 20 11:29:46 SAST 2017 ### Bad Referrer Count: 6069 -### Bad Bot Count: 503 +### Bad Bot Count: 510 ################################################### ### VERSION INFORMATION ## @@ -212,6 +212,7 @@ map $http_user_agent $bad_bot { # Known Vulnerability Scanners (now merged into one section) # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### + "~*\b\b" 3; "~*\b360Spider\b" 3; "~*\b80legs\b" 3; "~*\bAbonti\b" 3; @@ -278,6 +279,7 @@ map $http_user_agent $bad_bot { "~*\bClaritybot\b" 3; "~*\bCliqzbot\b" 3; "~*\bCloud\ mapping\b" 3; + "~*\bcoccocbot-web\b" 3; "~*\bCogentbot\b" 3; "~*\bcognitiveseo\b" 3; "~*\bCollector\b" 3; @@ -300,6 +302,7 @@ map $http_user_agent $bad_bot { "~*\bDeusu\b" 3; "~*\bDevil\b" 3; "~*\bDigincore\b" 3; + "~*\bDigitalPebble\b" 3; "~*\bDIIbot\b" 3; "~*\bDirbuster\b" 3; "~*\bDisco\b" 3; @@ -335,6 +338,7 @@ map $http_user_agent $bad_bot { "~*\bEyeNetIE\b" 3; "~*\bEzooms\b" 3; "~*\bFDM\b" 3; + "~*\bFemtosearchBot\b" 3; "~*\bFHscan\b" 3; "~*\bFimap\b" 3; "~*\bFirefox/7.0\b" 3; @@ -392,6 +396,7 @@ map $http_user_agent $bad_bot { "~*\bInternet\ Ninja\b" 3; "~*\bInternetSeer\b" 3; "~*\binternetVista\ monitor\b" 3; + "~*\bips-agent\b" 3; "~*\bIria\b" 3; "~*\bIRLbot\b" 3; "~*\bIskanie\b" 3; @@ -489,6 +494,7 @@ map $http_user_agent $bad_bot { "~*\bNmap\b" 3; "~*\bNPbot\b" 3; "~*\bNutch\b" 3; + "~*\boBot\b" 3; "~*\bOctopus\b" 3; "~*\bOffline\ Explorer\b" 3; "~*\bOffline\ Navigator\b" 3; @@ -578,6 +584,7 @@ map $http_user_agent $bad_bot { "~*\bSlackbot-LinkExpanding\b" 3; "~*\bSlySearch\b" 3; "~*\bSmartDownload\b" 3; + "~*\bSMTBot\b" 3; "~*\bSnake\b" 3; "~*\bSnapbot\b" 3; "~*\bSnoopy\b" 3; diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index 37debf2cc..42e3446de 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index a6775bbba..f686e59e1 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 80191aa0b..db4c43295 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V3.2017.12.1012 +#### Version: V3.2017.12.1013 #### Bad Referrer Count: 6069 -#### Bad Bot Count: 503 +#### Bad Bot Count: 510 ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 5fe36fe95..34e20e08f 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V3.2017.12.1012 +#### Version: V3.2017.12.1013 #### Bad Referrer Count: 6069 -#### Bad Bot Count: 503 +#### Bad Bot Count: 510 ____________________ ## Help Support This Project diff --git a/README.md b/README.md index 0346e6479..aabdae8a9 100755 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic from ever reaching your web sites. _______________ -#### Version: V3.2017.12.1012 +#### Version: V3.2017.12.1013 #### Bad Referrer Count: 6069 -#### Bad Bot Count: 503 +#### Bad Bot Count: 510 ____________________ ## Help Support This Project diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index 58e20365d..ca5514cda 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,3 +1,4 @@ + 360Spider 80legs Abonti @@ -122,8 +123,8 @@ ExtractorPro Extreme\ Picture\ Finder EyeNetIE Ezooms -FemtosearchBot FDM +FemtosearchBot FHscan Fimap Firefox/7.0 @@ -507,4 +508,3 @@ Zitebot ZmEu ZumBot ZyBorg - diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 242a05c35..72c67ec20 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V3.2017.12.1012 +#### Version: V3.2017.12.1013 #### Bad Referrer Count: 6069 -#### Bad Bot Count: 503 +#### Bad Bot Count: 510 ____________________ ## Help Support This Project diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index a27a8da44..5123b18a3 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.12.1012 -### Updated: Sun Dec 3 10:16:47 SAST 2017 +### Version: V3.2017.12.1013 +### Updated: Wed Dec 20 11:29:46 SAST 2017 ### Bad Referrer Count: 6069 -### Bad Bot Count: 503 +### Bad Bot Count: 510 ################################################### ### VERSION INFORMATION ## @@ -212,6 +212,7 @@ map $http_user_agent $bad_bot { # Known Vulnerability Scanners (now merged into one section) # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### + "~*\b\b" 3; "~*\b360Spider\b" 3; "~*\b80legs\b" 3; "~*\bAbonti\b" 3; @@ -278,6 +279,7 @@ map $http_user_agent $bad_bot { "~*\bClaritybot\b" 3; "~*\bCliqzbot\b" 3; "~*\bCloud\ mapping\b" 3; + "~*\bcoccocbot-web\b" 3; "~*\bCogentbot\b" 3; "~*\bcognitiveseo\b" 3; "~*\bCollector\b" 3; @@ -300,6 +302,7 @@ map $http_user_agent $bad_bot { "~*\bDeusu\b" 3; "~*\bDevil\b" 3; "~*\bDigincore\b" 3; + "~*\bDigitalPebble\b" 3; "~*\bDIIbot\b" 3; "~*\bDirbuster\b" 3; "~*\bDisco\b" 3; @@ -335,6 +338,7 @@ map $http_user_agent $bad_bot { "~*\bEyeNetIE\b" 3; "~*\bEzooms\b" 3; "~*\bFDM\b" 3; + "~*\bFemtosearchBot\b" 3; "~*\bFHscan\b" 3; "~*\bFimap\b" 3; "~*\bFirefox/7.0\b" 3; @@ -392,6 +396,7 @@ map $http_user_agent $bad_bot { "~*\bInternet\ Ninja\b" 3; "~*\bInternetSeer\b" 3; "~*\binternetVista\ monitor\b" 3; + "~*\bips-agent\b" 3; "~*\bIria\b" 3; "~*\bIRLbot\b" 3; "~*\bIskanie\b" 3; @@ -489,6 +494,7 @@ map $http_user_agent $bad_bot { "~*\bNmap\b" 3; "~*\bNPbot\b" 3; "~*\bNutch\b" 3; + "~*\boBot\b" 3; "~*\bOctopus\b" 3; "~*\bOffline\ Explorer\b" 3; "~*\bOffline\ Navigator\b" 3; @@ -578,6 +584,7 @@ map $http_user_agent $bad_bot { "~*\bSlackbot-LinkExpanding\b" 3; "~*\bSlySearch\b" 3; "~*\bSmartDownload\b" 3; + "~*\bSMTBot\b" 3; "~*\bSnake\b" 3; "~*\bSnapbot\b" 3; "~*\bSnoopy\b" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index a27a8da44..5123b18a3 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.12.1012 -### Updated: Sun Dec 3 10:16:47 SAST 2017 +### Version: V3.2017.12.1013 +### Updated: Wed Dec 20 11:29:46 SAST 2017 ### Bad Referrer Count: 6069 -### Bad Bot Count: 503 +### Bad Bot Count: 510 ################################################### ### VERSION INFORMATION ## @@ -212,6 +212,7 @@ map $http_user_agent $bad_bot { # Known Vulnerability Scanners (now merged into one section) # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### + "~*\b\b" 3; "~*\b360Spider\b" 3; "~*\b80legs\b" 3; "~*\bAbonti\b" 3; @@ -278,6 +279,7 @@ map $http_user_agent $bad_bot { "~*\bClaritybot\b" 3; "~*\bCliqzbot\b" 3; "~*\bCloud\ mapping\b" 3; + "~*\bcoccocbot-web\b" 3; "~*\bCogentbot\b" 3; "~*\bcognitiveseo\b" 3; "~*\bCollector\b" 3; @@ -300,6 +302,7 @@ map $http_user_agent $bad_bot { "~*\bDeusu\b" 3; "~*\bDevil\b" 3; "~*\bDigincore\b" 3; + "~*\bDigitalPebble\b" 3; "~*\bDIIbot\b" 3; "~*\bDirbuster\b" 3; "~*\bDisco\b" 3; @@ -335,6 +338,7 @@ map $http_user_agent $bad_bot { "~*\bEyeNetIE\b" 3; "~*\bEzooms\b" 3; "~*\bFDM\b" 3; + "~*\bFemtosearchBot\b" 3; "~*\bFHscan\b" 3; "~*\bFimap\b" 3; "~*\bFirefox/7.0\b" 3; @@ -392,6 +396,7 @@ map $http_user_agent $bad_bot { "~*\bInternet\ Ninja\b" 3; "~*\bInternetSeer\b" 3; "~*\binternetVista\ monitor\b" 3; + "~*\bips-agent\b" 3; "~*\bIria\b" 3; "~*\bIRLbot\b" 3; "~*\bIskanie\b" 3; @@ -489,6 +494,7 @@ map $http_user_agent $bad_bot { "~*\bNmap\b" 3; "~*\bNPbot\b" 3; "~*\bNutch\b" 3; + "~*\boBot\b" 3; "~*\bOctopus\b" 3; "~*\bOffline\ Explorer\b" 3; "~*\bOffline\ Navigator\b" 3; @@ -578,6 +584,7 @@ map $http_user_agent $bad_bot { "~*\bSlackbot-LinkExpanding\b" 3; "~*\bSlySearch\b" 3; "~*\bSmartDownload\b" 3; + "~*\bSMTBot\b" 3; "~*\bSnake\b" 3; "~*\bSnapbot\b" 3; "~*\bSnoopy\b" 3; diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 8c9a69f6e..6d8269cfc 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -1,14 +1,16 @@ ### Version Information # ################################################### -### Version: V3.2017.12.1012 -### Updated: Sun Dec 3 10:16:48 SAST 2017 -### Bad Bot Count: 503 +### Version: V3.2017.12.1013 +### Updated: Wed Dec 20 11:29:46 SAST 2017 +### Bad Bot Count: 510 ################################################### ### Version Information ## User-agent: * Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php +User-agent: +Disallow:/ User-agent: 360Spider Disallow:/ User-agent: 80legs @@ -141,6 +143,8 @@ User-agent: Cliqzbot Disallow:/ User-agent: Cloud mapping Disallow:/ +User-agent: coccocbot-web +Disallow:/ User-agent: Cogentbot Disallow:/ User-agent: cognitiveseo @@ -185,6 +189,8 @@ User-agent: Devil Disallow:/ User-agent: Digincore Disallow:/ +User-agent: DigitalPebble +Disallow:/ User-agent: DIIbot Disallow:/ User-agent: Dirbuster @@ -255,6 +261,8 @@ User-agent: Ezooms Disallow:/ User-agent: FDM Disallow:/ +User-agent: FemtosearchBot +Disallow:/ User-agent: FHscan Disallow:/ User-agent: Fimap @@ -369,6 +377,8 @@ User-agent: InternetSeer Disallow:/ User-agent: internetVista monitor Disallow:/ +User-agent: ips-agent +Disallow:/ User-agent: Iria Disallow:/ User-agent: IRLbot @@ -563,6 +573,8 @@ User-agent: NPbot Disallow:/ User-agent: Nutch Disallow:/ +User-agent: oBot +Disallow:/ User-agent: Octopus Disallow:/ User-agent: Offline Explorer @@ -741,6 +753,8 @@ User-agent: SlySearch Disallow:/ User-agent: SmartDownload Disallow:/ +User-agent: SMTBot +Disallow:/ User-agent: Snake Disallow:/ User-agent: Snapbot