diff --git a/.dev-tools/_robots_input/robots-input.txt b/.dev-tools/_robots_input/robots-input.txt index ee0b8277e..04cc54a72 100644 --- a/.dev-tools/_robots_input/robots-input.txt +++ b/.dev-tools/_robots_input/robots-input.txt @@ -58,6 +58,7 @@ Bolt BotALot Brandprotect Brandwatch +Buck Buddy BuiltBotTough BuiltWith @@ -354,6 +355,7 @@ Pavuk pcBrowser PECL::HTTP PeoplePal +Petalbot PHPCrawl Picscout Picsearch @@ -408,6 +410,7 @@ Searchestate SearchmetricsBot Semrush SemrushBot +SentiBot SEOkicks SEOkicks-Robot SEOlyticsCrawler @@ -579,5 +582,6 @@ Zeus zgrab Zitebot ZmEu +ZoominfoBot ZumBot ZyBorg diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index f2cc431bf..ee9f42d80 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.01.2229 -### Updated: Sat Jan 2 14:16:55 SAST 2021 +### Version: V3.2021.01.2230 +### Updated: Sat Jan 2 15:38:25 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 3bd4c7e2e..18b642a16 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 1; +104.154.182.187 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index cc2baf875..dd5291a4a 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 0; +104.154.182.187 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index f2cc431bf..ee9f42d80 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.01.2229 -### Updated: Sat Jan 2 14:16:55 SAST 2021 +### Version: V3.2021.01.2230 +### Updated: Sat Jan 2 15:38:25 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 3bd4c7e2e..18b642a16 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 1; +104.154.182.187 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index cc2baf875..dd5291a4a 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 0; +104.154.182.187 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index f2cc431bf..ee9f42d80 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.01.2229 -### Updated: Sat Jan 2 14:16:55 SAST 2021 +### Version: V3.2021.01.2230 +### Updated: Sat Jan 2 15:38:25 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 3bd4c7e2e..18b642a16 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 1; +104.154.182.187 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 4e7a334b2..6f08a6b3d 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -119,6 +119,7 @@ "~*(?:\b)BotALot(?:\b|)" 0; "~*(?:\b)Brandprotect(?:\b|)" 0; "~*(?:\b)Brandwatch(?:\b|)" 0; + "~*(?:\b)Buck(?:\b|)" 0; "~*(?:\b)Buddy(?:\b|)" 0; "~*(?:\b)BuiltBotTough(?:\b|)" 0; "~*(?:\b)BuiltWith(?:\b|)" 0; @@ -415,6 +416,7 @@ "~*(?:\b)pcBrowser(?:\b|)" 0; "~*(?:\b)PECL::HTTP(?:\b|)" 0; "~*(?:\b)PeoplePal(?:\b|)" 0; + "~*(?:\b)Petalbot(?:\b|)" 0; "~*(?:\b)PHPCrawl(?:\b|)" 0; "~*(?:\b)Picscout(?:\b|)" 0; "~*(?:\b)Picsearch(?:\b|)" 0; @@ -469,6 +471,7 @@ "~*(?:\b)SearchmetricsBot(?:\b|)" 0; "~*(?:\b)Semrush(?:\b|)" 0; "~*(?:\b)SemrushBot(?:\b|)" 0; + "~*(?:\b)SentiBot(?:\b|)" 0; "~*(?:\b)SEOkicks(?:\b|)" 0; "~*(?:\b)SEOkicks-Robot(?:\b|)" 0; "~*(?:\b)SEOlyticsCrawler(?:\b|)" 0; @@ -640,6 +643,7 @@ "~*(?:\b)zgrab(?:\b|)" 0; "~*(?:\b)Zitebot(?:\b|)" 0; "~*(?:\b)ZmEu(?:\b|)" 0; + "~*(?:\b)ZoominfoBot(?:\b|)" 0; "~*(?:\b)ZumBot(?:\b|)" 0; "~*(?:\b)ZyBorg(?:\b|)" 0; # END MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index cc2baf875..dd5291a4a 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 0; +104.154.182.187 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index f2cc431bf..ee9f42d80 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.01.2229 -### Updated: Sat Jan 2 14:16:55 SAST 2021 +### Version: V3.2021.01.2230 +### Updated: Sat Jan 2 15:38:25 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index f2cc431bf..ee9f42d80 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.01.2229 -### Updated: Sat Jan 2 14:16:55 SAST 2021 +### Version: V3.2021.01.2230 +### Updated: Sat Jan 2 15:38:25 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 4efc70554..a4ec81574 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2229 -### Updated: Sat Jan 2 14:16:54 SAST 2021 +### Version: V4.2021.01.2230 +### Updated: Sat Jan 2 15:38:24 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index c4667b0ac..22793dd10 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.122.208.80 0; +35.193.184.18 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 4efc70554..a4ec81574 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2229 -### Updated: Sat Jan 2 14:16:54 SAST 2021 +### Version: V4.2021.01.2230 +### Updated: Sat Jan 2 15:38:24 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 3bd4c7e2e..18b642a16 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 1; +104.154.182.187 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index cc2baf875..dd5291a4a 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 0; +104.154.182.187 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 4efc70554..a4ec81574 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2229 -### Updated: Sat Jan 2 14:16:54 SAST 2021 +### Version: V4.2021.01.2230 +### Updated: Sat Jan 2 15:38:24 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 3bd4c7e2e..18b642a16 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 1; +104.154.182.187 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index cc2baf875..dd5291a4a 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 0; +104.154.182.187 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 4efc70554..a4ec81574 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2229 -### Updated: Sat Jan 2 14:16:54 SAST 2021 +### Version: V4.2021.01.2230 +### Updated: Sat Jan 2 15:38:24 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 3bd4c7e2e..18b642a16 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 1; +104.154.182.187 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 4e7a334b2..6f08a6b3d 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -119,6 +119,7 @@ "~*(?:\b)BotALot(?:\b|)" 0; "~*(?:\b)Brandprotect(?:\b|)" 0; "~*(?:\b)Brandwatch(?:\b|)" 0; + "~*(?:\b)Buck(?:\b|)" 0; "~*(?:\b)Buddy(?:\b|)" 0; "~*(?:\b)BuiltBotTough(?:\b|)" 0; "~*(?:\b)BuiltWith(?:\b|)" 0; @@ -415,6 +416,7 @@ "~*(?:\b)pcBrowser(?:\b|)" 0; "~*(?:\b)PECL::HTTP(?:\b|)" 0; "~*(?:\b)PeoplePal(?:\b|)" 0; + "~*(?:\b)Petalbot(?:\b|)" 0; "~*(?:\b)PHPCrawl(?:\b|)" 0; "~*(?:\b)Picscout(?:\b|)" 0; "~*(?:\b)Picsearch(?:\b|)" 0; @@ -469,6 +471,7 @@ "~*(?:\b)SearchmetricsBot(?:\b|)" 0; "~*(?:\b)Semrush(?:\b|)" 0; "~*(?:\b)SemrushBot(?:\b|)" 0; + "~*(?:\b)SentiBot(?:\b|)" 0; "~*(?:\b)SEOkicks(?:\b|)" 0; "~*(?:\b)SEOkicks-Robot(?:\b|)" 0; "~*(?:\b)SEOlyticsCrawler(?:\b|)" 0; @@ -640,6 +643,7 @@ "~*(?:\b)zgrab(?:\b|)" 0; "~*(?:\b)Zitebot(?:\b|)" 0; "~*(?:\b)ZmEu(?:\b|)" 0; + "~*(?:\b)ZoominfoBot(?:\b|)" 0; "~*(?:\b)ZumBot(?:\b|)" 0; "~*(?:\b)ZyBorg(?:\b|)" 0; # END MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index cc2baf875..dd5291a4a 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 0; +104.154.182.187 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 4efc70554..a4ec81574 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2229 -### Updated: Sat Jan 2 14:16:54 SAST 2021 +### Version: V4.2021.01.2230 +### Updated: Sat Jan 2 15:38:24 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 5f9b58ab0..a40be186b 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.122.208.80 0; -104.154.182.187 0; +34.122.173.211 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 401ff8977..edfbaa5c8 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2229 -### Updated: Sat Jan 2 14:16:54 SAST 2021 +### Version: V4.2021.01.2230 +### Updated: Sat Jan 2 15:38:24 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index a40be186b..32af08fbb 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -46,3 +46,4 @@ # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST 34.122.173.211 0; +35.193.184.18 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 893b2f953..1f718a662 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2229 -### Updated: Sat Jan 2 14:16:54 SAST 2021 +### Version: V4.2021.01.2230 +### Updated: Sat Jan 2 15:38:24 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index 69abfe4c7..32af08fbb 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.193.184.18 0; -104.154.182.187 0; 34.122.173.211 0; +35.193.184.18 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 4efc70554..a4ec81574 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2229 -### Updated: Sat Jan 2 14:16:54 SAST 2021 +### Version: V4.2021.01.2230 +### Updated: Sat Jan 2 15:38:24 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 5e80f4199..3d512ce96 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.193.184.18 0; -34.122.208.80 0; +34.122.173.211 0; +104.154.182.187 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 4efc70554..a4ec81574 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2229 -### Updated: Sat Jan 2 14:16:54 SAST 2021 +### Version: V4.2021.01.2230 +### Updated: Sat Jan 2 15:38:24 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 3bd4c7e2e..18b642a16 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 1; +104.154.182.187 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/blacklist-user-agents.conf b/.dev-tools/test_units/blacklist-user-agents.conf index 4e7a334b2..6f08a6b3d 100644 --- a/.dev-tools/test_units/blacklist-user-agents.conf +++ b/.dev-tools/test_units/blacklist-user-agents.conf @@ -119,6 +119,7 @@ "~*(?:\b)BotALot(?:\b|)" 0; "~*(?:\b)Brandprotect(?:\b|)" 0; "~*(?:\b)Brandwatch(?:\b|)" 0; + "~*(?:\b)Buck(?:\b|)" 0; "~*(?:\b)Buddy(?:\b|)" 0; "~*(?:\b)BuiltBotTough(?:\b|)" 0; "~*(?:\b)BuiltWith(?:\b|)" 0; @@ -415,6 +416,7 @@ "~*(?:\b)pcBrowser(?:\b|)" 0; "~*(?:\b)PECL::HTTP(?:\b|)" 0; "~*(?:\b)PeoplePal(?:\b|)" 0; + "~*(?:\b)Petalbot(?:\b|)" 0; "~*(?:\b)PHPCrawl(?:\b|)" 0; "~*(?:\b)Picscout(?:\b|)" 0; "~*(?:\b)Picsearch(?:\b|)" 0; @@ -469,6 +471,7 @@ "~*(?:\b)SearchmetricsBot(?:\b|)" 0; "~*(?:\b)Semrush(?:\b|)" 0; "~*(?:\b)SemrushBot(?:\b|)" 0; + "~*(?:\b)SentiBot(?:\b|)" 0; "~*(?:\b)SEOkicks(?:\b|)" 0; "~*(?:\b)SEOkicks-Robot(?:\b|)" 0; "~*(?:\b)SEOlyticsCrawler(?:\b|)" 0; @@ -640,6 +643,7 @@ "~*(?:\b)zgrab(?:\b|)" 0; "~*(?:\b)Zitebot(?:\b|)" 0; "~*(?:\b)ZmEu(?:\b|)" 0; + "~*(?:\b)ZoominfoBot(?:\b|)" 0; "~*(?:\b)ZumBot(?:\b|)" 0; "~*(?:\b)ZyBorg(?:\b|)" 0; # END MAKE BAD BOTS GOOD ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index fa7c9cbc6..635a81af4 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Attach -Brandwatch -crawler.feedback -Discoverybot -Mojolicious -NextGenSearchBot -Nutch -OPPO A33 -PeoplePal -Siphon +BBBike +Collector +crawl.sogou.com +Flunky +Nessus +NetAnts +NetLyzer +Sosospider +Surfbot +Wotbox diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index a02f6e870..181b8dc76 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -Abonti Acunetix -AiHitBot -Aipbot -AlphaBot +Alexibot Anarchie -Ankit -Anthill -ASPSeek -Asterias -autoemailspider -Backlink-Ceck +Apexoo +arquivo.pt +arquivo-web-crawler +AwarioRssBot +AwarioSmartBot +BackDoorBot +backlink-check BacklinkCrawler BackStreet -Battleztar Bazinga -BBBike -BDCbot +BackWeb +Bandit +BatchFTP BetaBot +Bigfoot +Bitacle Black Hole -BLEXBot -Blow -BlowFish -Boardreader -BotALot -Brandprotect -Brandwatch +Buck +Buddy +BuiltBotTough +BunnySlippers +BuzzSumo CATExplorador CazoodleBot CCBot -Cliqzbot -Cloud mapping -coccocbot-web -Cogentbot -cognitiveseo +CheTeam +ChinaClaw com.plumanalytics Copier -CopyRightCheck Cosmos Craftbot -crawler.feedback -CrunchBot -DataCha0s -demandbase-bot +Curious +Custo +DatabaseDriverMysqli +DBLBot Demon -Deusu -DigitalPebble -Disco -Discobot -DittoSpyder -DomainCrawler -Domains Project +Digincore +Dirbuster +Dispatch DomainStatsBot Download Wonder Dragonfly -Drip +DSearch +DTS Agent +ECCP/1.0 +Ecxi EMail Siphon EMail Wolf -evc-batch +EroCrawler +Exabot +Express WebPictures ExtLinksBot +Extractor EyeNetIE +Ezooms facebookscraper -FDM -Fimap -Firefox/7.0 -Freeuploader +FemtosearchBot +FHscan +FlashGet +Flunky +FrontPage FyberSpider Fyrebot -Genieo +GermCrawler Gigablast Gigabot -Go-Ahead-Got-It +G-i-g-a-b-o-t Gotit GoZilla -Go!Zilla -GrabNet -GrapeFX GrapeshotCrawler -Haansoft -HEADMasterSEO -Hloader -HTMLparser +GridBot +GT::WWW +HaosouSpider +Havij +HTTP::Lite +HTTrack Humanlinks Iblog IDBTE4M Id-search IlseBot -Image Sucker +Image Fetch IndeedBot +Indy Library instabid -InterGET Internet Ninja -InternetSeer internetVista monitor -ips-agent -IRLbot -isitwp.com -JamesBOT -Jbrofuzz -JetCar -Joomla -Jorgee +Iria +Iskanie +JOC Web Spider JustView Jyxobot Keyword Density Kinza +Kozmosbot Larbin +LeechGet +Lftp LibWeb Libwhisker LieBaoFast -LinkextractorPro +Lightspeedsystems +Likse +Linkdexbot LinkpadBot -LinkScan +LinksManager LinqiaRSSBot LinqiaScrapeBot -Lipperhey -LNSpiderguy +Litemage_walker +Lmspider +lwp-request Magnet magpie-crawler -Mail.RU_Bot -Majestic SEO -Majestic-SEO -MarkMonitor -Masscan -Mata Hari +Majestic12 +Mass Downloader meanpathbot +Meanpathbot MeanPath Bot -Mediatoolkitbot -Microsoft URL Control -MIIxpc -Mister PiX +MegaIndex.ru +MFC_Tear_Sample +MicroMessenger MJ12bot Mojeek Mojolicious +Morfeus Fucking Scanner Mozlila -Mr.4x3 -MSIECrawler -Msrabot -NearSite +MQQBrowser +MSFrontPage +Musobot +Navroad +Needle NetAnts -netEstate NE Crawler -NetMechanic NetSpider Nettrack -Netvibes NextGenSearchBot -NICErsPRO -Niki-bot -Nikto -NPbot +Nibbler +Nimbostratus +Ninja +Nmap Nutch -oBot -Offline Explorer +Offline Navigator +OnCrawl Openfind -OPPO A33 -OrangeBot -OrangeSpider -OutfoxBot +Openvas +OutclicksBot +PageAnalyzer +PageGrabber page scorer +Pavuk PECL::HTTP PeoplePal -Picsearch +PHPCrawl PictureFinder -PleaseCrawl -plumanalytics -ProPowerBot +Pimonster +Pi-Monster +Probethenet +ProWebWalker +Psbot Pump PxBroker PyCurl -RankFlex -RankingBot -RankurBot +Quick-Crawler +RankActive Reaper -RebelMouse -Recorder -RedesScrapy -RepoMonkey +ReGet +Ripper +Rogerbot +RSSingBot +s1z.ru SalesIntelligent satoristudio.net SBIder -ScanAlert -Scanbot +ScoutJet +Scrapy +Semrush SEOkicks -Seomoz -SeoSiteCheckup +serpstatbot sexsearcher -Shodan -Siphon SISTRIX +Sitebeam SiteCheckerBotCrawler -sitechecker.pro -SiteSucker -Site Sucker -SmartDownload +SiteLockSpider +SiteSnagger +SMTBot Snapbot Sogou web spider -Sottopop +Sosospider +SpankBot Spanner sp_auditbot -SputnikBot -spyfu +Spbot +Sqlmap +Sqlworm +Sqworm +Steeler Sucker -Surfbot -Telesoft -Telesphoreo -Telesphorep -TightTwatBot +Sucuri +SuperBot +SuperHTTP +SurveyBot +Suzuran +sysscan +Szukacz +T8Abot +tAkeOut +Teleport +TheNomad Titan Toata -Tracemyfile +Trendiction +Trendictionbot trendiction.com trendiction.de -True_Robot -TurnitinBot -UnisterBot -Upflow +Turingos +Turnitin +Twice +URLy Warning Vacuum -Vagabondo -Voil +VidibleScraper +Virusdie +VoidEYE WallpapersHD -WBSearchBot -Webalta -WebBandit +Web Auto WebCollage +WebCopier +WebEnhancer Web Fetch -WebImageCollector -Web Pix -Webshag +WebFuck +WebmasterWorldForumBot WebsiteExtractor -Webster WebSucker -WebWhacker +Web Sucker WebZIP -WeSEE -Whatweb Who.is Bot +Widow WinHTTrack -Wonderbot Wotbox +WWW-Collector-E WWW::Mechanize -x09Mozilla -Xaldon WebSpider -Xenu +WWWOFFLE +x22Mozilla +Xaldon_WebSpider xpymep1.exe +YoudaoBot Zade -Zauba -zauba.io -Zermelo Zeus -zgrab -ZmEu +Zitebot +ZumBot diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index b1d3410bf..0fd00b1fc 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ 360Spider 404checker -Aboundex -AIBOT -Alligator +AfD-Verbotsverfahren +Aipbot AllSubmitter -AlphaBot -Apexoo +Anthill archive.org_bot -Aspiegel -autoemailspider -AwarioRssBot +arquivo.pt +arquivo-web-crawler +ASPSeek +Asterias BackDoorBot +Backlink-Ceck BacklinkCrawler -BackStreet +BackWeb Bandit Barkrowler +BatchFTP Battleztar Bazinga +BBBike +BDCbot +BDFetch +BetaBot Blackboard -Black Hole -BlowFish -Boardreader -Brandprotect -Brandwatch -Bullseye -BunnySlippers -BuzzSumo -CCBot -CheeseBot +BlackWidow +BLEXBot +Blow +Buddy +BuiltWith +Calculon +check1.exe CherryPicker -CODE87 +CheTeam +Chlooe +Claritybot +Cogentbot +cognitiveseo +Collector com.plumanalytics -Copier -CopyRightCheck Cosmos Craftbot crawler4j -crawl.sogou.com CrazyWebCrawler -CrunchBot -CSHttp -DIIbot +Crescent +demandbase-bot +Demon +Devil +Disco +Discobot Dispatch DittoSpyder -DnyzBot -DomainCrawler DomainSigmaCrawler -Domains Project -Drip -Ebingbong -Ecxi -EroCrawler +Download Wonder +Dragonfly +DSearch +eCatch +EirGrabber +EMail Siphon +evc-batch +Exabot Express WebPictures -ExtLinksBot -Extractor -Extreme Picture Finder EyeNetIE FHscan -FlashGet -FrontPage +Foobot +FyberSpider Fyrebot GalaxyBot Genieo GermCrawler -Gigablast +Gigabot G-i-g-a-b-o-t -Go-Ahead-Got-It -Go!Zilla -GrabNet -GrapeFX -GrapeshotCrawler +Gotit +GoZilla GT::WWW -Haansoft Harvest Havij Heritrix -Hloader -HMView +HTMLparser HTTP::Lite -Iblog IDBot +IlseBot +Image Fetch Image Sucker -Indy Library +InfoNaviRobot InfoTekies +instabid Intelliseek InterGET +Internet Ninja IRLbot -isitwp.com -Iskanie +JennyBot JetCar -JikeSpider JOC Web Spider -Joomla +Jorgee +JustView Kenjin Spider -Kozmosbot -Lanshanbot -LeechGet -Lftp +Keyword Density +LibWeb Libwhisker LieBaoFast Lightspeedsystems Likse -LinkWalker -LinqiaMetadataDownloaderBot -Litemage_walker -LNSpiderguy -lwp-request -lwp-trivial +Linkdexbot +LinkextractorPro +LinkpadBot +LinqiaScrapeBot +Lipperhey Spider +Ltx71 +Mag-Net +Mail.RU_Bot +Majestic12 Majestic-SEO MarkMonitor -Mass Downloader +MarkWatch +Masscan Mata Hari MauiBot -MeanPath Bot Mediatoolkitbot mediawords -MFC_Tear_Sample MicroMessenger -Microsoft Data Access MIIxpc -MJ12bot Morfeus Fucking Scanner -MQQBrowser +MSFrontPage MSIECrawler -muhstik-scan Name Intelligence Nameprotect -Needle +Navroad +NearSite +Nessus NetAnts -netEstate NE Crawler +Netcraft NetLyzer NetSpider +Nettrack Net Vampire -Netvibes -NICErsPRO -Nikto +NextGenSearchBot Ninja -Nmap Nutch -Offline Explorer +Octopus Offline Navigator +OnCrawl +Openfind +OpenLinkProfiler Openvas +OpenVAS OrangeBot +OutclicksBot OutfoxBot -PageAnalyzer -PageGrabber +Page Analyzer +page scorer Pandalytics Panscient -Papa Foto -Pavuk -pcBrowser -PECL::HTTP -PeoplePal -Picscout Picsearch Pimonster Pixray -PleaseCrawl -Pockey -POE-Component-Client-HTTP -polaris version -Probethenet ProWebWalker Psbot -PxBroker -RankActiveLinkBot -RankingBot -Rankivabot -RocketCrawler -RSSingBot +RankFlex +RankurBot +Recorder +RedesScrapy s1z.ru SalesIntelligent -scan.lol +satoristudio.net +Scanbot ScoutJet +Screaming ScreenerBot +SemrushBot +SentiBot SEOkicks -SEOkicks-Robot -SEOlyticsCrawler -serpstatbot -sexsearcher -Shodan +SEOprofiler +seoscanners +SEOstats Siphon +SiteCheckerBotCrawler +SiteExplorer +Siteimprove +SiteLockSpider Sitevigil +SlySearch +SmartDownload +Snake Snapbot Snoopy -SocialRankIOBot -Sociscraper -SpankBot +Sosospider +Spammen Spanner -sp_auditbot -Spinn3r -SputnikBot Sqlmap -Steeler +Sqlworm Stripper +Sucker SuperHTTP -T0PHackTeam -T8Abot +Suzuran +Swiftbot +Szukacz tAkeOut Teleport -Telesphoreo -The Intraformant -Thumbor -Titan +TeleportPro +Telesphorep +TightTwatBot Toata -Toweyabot Tracemyfile -Trendictionbot -trendiction.com -Turingos -TurnitinBot +True_Robot +TwengaBot Twice UnisterBot -Vacuum +URLy Warning +URLy.Warning +Vagabondo +VB Project VeriCiteCrawler +VidibleScraper Virusdie -WBSearchBot -Webalta +VoidEYE +Voil +Voltron WebAuto +Web Collage WebCopier WEBDAV -WebEnhancer -Web Enhancer -WebGo IS -WebLeacher +Web Fetch +Web Fuck +WebPix Web Pix -WebsiteExtractor +WebReaper +Web Sauger +Webshag Website Quester -Webster WebStripper WebSucker -Web Sucker -WebWhacker +WebZIP Whack +Whacker Whatweb Who.is Bot -WiseGuys Robot -Wotbox -Wprecon +Widow +WISENutbot +Woobot WPScan WWW-Collector-E -WWW::Mechanize -WWWOFFLE x22Mozilla +xpymep1.exe +YoudaoBot Zauba -zauba.io Zeus +zgrab Zitebot -ZumBot +ZmEu ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index 01172071d..73c8fc6de 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -bagsonsale.online -cgi2.nintendo.co.jp -ekn-art.se -moi-glazki.ru -myhealthcare.com -obsessionphrases.com -shivafurnishings.com -thefotosgratis.eu -tizanidinehcl4mgtabinfo.blogspot.com -via-energy-cumpara.com +buzz-porno.info +c-gainsbourg.com +directrev.com +ezigarettekaufen1.hpage.com +medtherapy.ru +na-telefon.biz +onthemarch.co +susanholtphotography.com +varbergsvind.se +zaidia.xhost.ro diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 22191380f..845440a21 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -00webcams.com -01apple.com -108shot.com -123movies.love -12masterov.com -13tabs.com -1pamm.ru -21h2o.com -2girls1cup-free.com -45en.ru -aa14ab57a3339c4064bd9ae6fad7495b5f.com -abcdefh.xyz -academia-nsk.org -accessoires-mode.in -acortarurl.es -actionnooz.com -activecampaign.dreamhosters.com -adclickthru.net -admatic.com.tr -adrenalin-stk.ru -adrunnr.com -adultgalls.com -adultnet.in -advocatemsk.ru -agahinameh.com -aghanyna.com -agysya.ru -aktobe.xkaz.org -alarmobninsk.ru -alert-hgd.xyz -alice.it -alkoravto.ru -alldubai.biz -allproblog.com -altamayoztourism.com -alveris.ru -amateurlivechat.org +00go.com +100dollars-seo.com +2728fb936f0.com +777-club.ru +883zy.com +8xv8.com +aa9d046aab36af4ff182f097f840430d51.com +aadroid.net +abbanreddy.com +abctoppictures.net +abovetherivernc.com +acheterviagrafr24.com +adamoads.com +adclickservice.com +adhome.biz +adloads.com +adobereader-free.ru +adult3dgames.com +advancedmassagebysara.com +adventureparkcostarica.com +adviceforum.info +advokateg.ru +aero2.ru +agecheckadult.com +aibolita.com +aimayangzhi.com +akita.kz +aksonural.ru +alert-fjg.xyz +all-streaming-media.com +alpharma.net amazingninja.com -amazon-adsystem.com -am-se.com -andpolice.com -anonymizeme.pro -aosheng-tech.com -apioutoboxnet-a.akamaihd.net -app5.letmacworkfaster.world -appleid-verification.com -appsaurus.com -appsecurityr.com -apptester.tk -apxeo.info -arendatora.ru -artavenuegardenstudios.com -aruplighting.com -asrvvv-a.akamaihd.net -atley.eu.pn +an-donut.com +apartmentratings.com +apessay.com +apiholdingmypage-a.akamaihd.net +appfastplay.com +ap.senai.br +asacopaco.tk +atofilms.com ausergrubhof.info -autolombard-krasnodar.ru -aversis.net -aviav.ru.com -avkzarabotok.info -bagcionderlab.com -battleforupdating.site -bayareaaeroclub.org -bestprofits-there.com -best-seo-software.xyz -bestserials.com -bestwebsitesawards.com -bezsporno.ru -bio-optomarket.ru -bitnote.co -blackwitchcraft.ru -blog.f00kclan.de -blogrankers.com -bmsco.co -boc.kz -bongiornos.info -bookhome.info -bristolhostel.com -brokergid.ru -buigas.00it.com -burn-fat.ga +autobrennero.it +avek.ru +b00kmarks.com +bablonow.ru +babosas.com +baikaleminer.com +barbourjackets.online +bbsoldes.fr +beachpics.com +best-coupon-offer.com +bestimagecoollibrary.com +best-mam.ru +bestplacetobuyeriacta.jw.lt +bestwrinklecreamnow.com +beta.hotkeys.com +bigames.online +big-cash.net +bikini-image.com +billiard-classic.com.ua +binomo.com +binomo.kz +bio-japan.net +bird1.ru +bizfly.info +bjgugu.net.ua +blavia.00author.com +blue-square.biz +bmw-ark.ru +boazpower.com +bolezniorganov.ru +bonkers.name +brainboosting.club +break-the-chains.com +bristolhotel.com +b-styles.xyz +buehne-fuer-menschenrechte.de +burger-tycoon.com business-suggested.tk -buyclomidonlaine.com -buyfriend.ru -buypillsorderonline.com -buypuppies.ca -cakemediahq.com.au -canacopegdl.com -canrioloadm.gq -cartierlove2u.xyz -cayado.snn.gr -cbox.ws -ccjp.eu -cenoval.ru -chatroulette.si -cheapestjerseysonwholesale.com -cheap-pills-norx.com -check-host.net -chocolateslim-original.com -cinemaenergy-hd.ru -classiquebijoux.ru -clayvasedesigns.tk -cleansearch.net -clipartnew.com -cntravelre.com -collegerentals.com -compliance-alex.top -compliance-checker.info +buyessayonline19.blogspot.ru +buzzurl.jp +camsex.xxx-cam.webcam +cartechnic.ru +cartierlove.xyz +cartujano-pre.de +cascadelink.org +cejewelry.xyz +cenokos.ru +centre-indigo.org.ua +cheapcigarettesc.info +cheapjerseysap.com +cheapjerseysbizwholesale.us +checktext.org +chelnytruck.ru +chimiver.info +cialisndbrx.com +cleanallvirus.com +clickaider.com +clickbanksites.info +clickprotects.com +clodo.ru +cognitiveseo.com +coldfilm.ru +cometorussia.net +community.allhiphop.com +compliance-brian.xyz +compliance-irvin.xyz +compliance-olga.top compucelunlock.net -connectingsingles.com -constantaservice.net -converse.ddsoldes.fr -cookingmeat.ru -cracksplanet.com -cupday.com -custodieva.ru -d0t.ru -dailystrength.org -dame-ns.kz -danceuniverse.ru -darmebel.com.ua -data1.scopich.com -date-withme.com -deda-moroza-zakaz.ru -defenderxtactical.com -delta-line.men -dengi-pod-zalog-nedvizhimosti.ru -deutschland123.de -dickssportinggoods.com -digital-video-processing.com -div.as -dms-sw.ru -dogsrun.net -dollartree.info -domoysshop.ru -dotmass.top -dptaughtme.com -driving.kiev.ua -drugspowerstore.com -easync.io -efnor-ac.com -electronicadirect.com -elektroniksigaraankara.info -elektronischezi.livejournal.com -elektrozigarettekaufen1.devhub.com -elektrozigaretten2.yn.lt -enews.tech -englishdictionaryfree.com -eng-lyrics.com -enskedesquashclub.se -eurosamodelki.ru -evogarage.com -evrotekhservis.ru -executehosting.com -eyemagination.com +cookie-law-enforcement-hh.xyz +copyrightclaims.org +cornerstone-countertops.com +costablue.xyz +crackzplanet.com +cubs-jerseys.us +culmimg.pw +custom-electric-guitar.com +dairyindia.in +dandiyabeats.in +dangphoto.trade +darkbooks.org +d-black.bz +detalizaciya-tut.biz +dignitasdata.se +dinkolove.ya.ru +dostavimvdom.ru +dostavka-v-krym.com +dotnetdotcom.org +drugstoreforyou.com +duplicashapp.com +eandsgallery.com +e-avon.ru +edge.sharethis.com +edudocs.net +eduserver.net +ekobata.ru +elektrischezigarette1.blog.pl +elektrozigaretten1.webs.com +engines-usa.com +erotische-geschichten-xxl.com +escortplius.com +etm-consult.de +eu-cookie-law.info +europages.com.ru +evaashop.ru +everytalk.tv +exoneration-project.us +expdom.com +eyelike.com.ua +ezigarettekaufen2.yolasite.com ezigarettenkaufen1.deviantart.com -f5mtrack.com -famix.xyz -faracontrol.ir -fashionindeed.ml -faz99.com -femmesdenudees.com -ff30236ddef1465f88547e760973d70a.com +ezigarettenkaufen1.pagina.gr +f07.de +fable.in.ua +failingmarriege.blogspot.com +farmingworm.com +fdzone.org +femdom.twiclub.in +fertilitetsradgivningen.se +fickluder69.com +filefilter.weebly.com +film-one.ru +find1friend.com findanysex.com fitnesspiks.com -formseo.com -forpostlock.ru -free411games.com -freejabs.com -free-social-buttons.xyz -freewhatsappload.com -fullgirl.ru -funponsel.com -gameonasia.com +fivedwld.cf +flagcounter.me +flashlarevista.com +floridahuntingfishingadventures.com +fm-upgrade.ru +fodelsedagspresenter.nu +forexunion.net +forsex.info +freelinkbuilding.website.tk +freesitetest.com +freeuploader.com +freshsuperbloop.com +front.to +gamblingnerd.com +game300.ru +gameplexcity.com +gamerextra.com game-top.su -gay-site.store -gazporno.com -gdebestkupit.ru -girls-ufa.ru -glastecfilms.com.my -go2album.com -go2jump.org +gateway.zscalerone.net +genericlowlatencyasiodriverhq.aircus.com +genericviagrasildenafiled.net +gepezz.info +getoutofdebtfree.org +get-seo-domain.com +gilsonchiro.xyz +golden-praga.ru +goodnightjournal.com +gopixdatabase.com +gox.com.ua gpms.org.my -greenidesign.co -gsasearchengineranker.top -gsasearchengineranker.xyz +gratis-sexkontakte.com +green-tea.tv +grupografico-pilar.com.ar gsmlab.pl -gsmtlf.ru -gymi.name -handicapbathtubarea.com -hawaiisurf.com -headpharmacy.com -heartofpayne.xyz -hem.passagen.se -hermesbelts.xyz -hikesearch.net -hildinghr.se -hobbyhuren-datenbank.com -honyaku.yahoofs.jp -hostingclub.lk -hotloans.ru -hundejo.com -ihc2015.info -ilo134uloh.com -iloveitaly.ru -ilte.info -imadedinner.net -images-free.net -imk.com.ua -incest-ru.com -indo-export.ru -infokonkurs.ru -istripper.com -ix20.ru -javrip.net -jerseychinabizwholesale.us -jerseyssportsshop.com +gta-top.ru +guiadeserraazul.com +headpress.ru +healmytrauma.info +hentai-manga.porn +hermesbracelets.xyz +hermesreplica.pw +hesteel.pl +hit-kino.ru +hol.es +hotblognetwork.com +hotshoppymac.com +hoverboard360.se +hoztorg-opt.ru +htmlcorner.com +https-legalrc.biz +hyiphunter.org +igithab.com +ikearugs.xyz +iklysha.ml +imfamous.info +imgarcade.com +img.wallpaperstock.net +imitex-plus.ru +imperia31.ru +inet-traffic.com +infektsii.com +infodocsportal.com +innodgfdriverhm.aircus.com +insider.pro +interior-stickers.ru +investsuccess.org +investyb.com +ispaniya-costa-blanca.ru +istanbulit.com +javstock.com +jeremyeaton.co jerseywholesalebizchina.com -jetsli.de -juliadiets.com -kaidalibor.de +job.icivil.ir +jongose.ninja +journalhome.com +kakadu-interior.com.ua kambasoft.com -karting196.ru -katushka.net -kfon.eu -kicknights.gq -kinofak.net -kinozapas.com -komp-pomosch.ru -koptims.tiu.ru -kountrylife.com -koversite.info -ktotut.net +karpun-iris.ru +katjimej.blog.fc2.com +keki.info +kerwinandcariza.com +keyhantercume.com +keywordhut.com +kidd.reunionwatch.com +kimcurlrvsms.com +kinobest.pl +kinopolet.net +klin3952.ru +kolotiloff.ru +kongruan.com +koronirealestate.gr +kotaku.com +kvartira-sutochno.com lacloop.info -ladsblue.com -lakomka.com.ua -lapolis.it -law-check-nine.xyz -lawyers.cafe -leadn.pl -lida-ru.com -liky.co.ua -lion.conboy.us -lion.snapmap.us -livingroomdecor.info -lonely-mature.com -longgreen.info -longlifelomilomi.info -mabdoola.blogspot.com -macresource.co.uk -mansparskats.com -manualterap.roleforum.ru -manyresultshub-a.akamaihd.net -marketland.ml -medicines-choice.com -medicineseasybuy.com -megakino.net -menstennisforums.com -mesbuta.info -mex-annushka.ru -michaelkorsoutlet.store +lalalove.ru +lancheck.net +lexaprogeneric.link +lezbiyanki.net +linkbux.ru +liquimondo.com +lisque.batcave.net +live-sexcam.tk +liveshoppersmac.com +lob.com.ru +lol-smurfs.com +lombia.co +long-beach-air-conditioning.com +lovasszovetseg.hu +lyngdalhudterapi.no +lyrster.com +macotool.com +magnetpress.sk +marcoislandvacations.net +maridan.com.ua +master-muznachas.ru +mcar.in.ua +mearns-tractors.co.uk +medispainstitute.com.au +meetingrainstorm.bid mielec.pl -mikrobiologies.ru -mindbox.co.za -minecraft-rus.org -minharevisao.com -misandesign.se -mmog-play.ru -mobicover.com.ua -moje-recenze.cz -moonci.ru -moxo.com -moz.com -mp3downloadhq.com -msk-diplomat.com -mturkcontent.com -muabancantho.info -muschisexbilder.com -music7s.me -musicspire.online -musicstock.me +moivestiy.biz +monitorwebsitespeed.com +moroccosurfadventures.com +moscow.xrus.org +muizre.ru +musicdaddy.net musicvidz.ru -muz-tracker.net -myfreetutorials.com -mypets.by -mypornfree.ru -na-telefon.biz -naturalbreakthroughsresearch.com -new7ob.com +mvpicton.co.uk +mydearest.co +mydownloadengine.com +mykings.pw +mylida.org +negociosdasha.com +newstraveller.ru newyorkhotelsmotels.info -nfljerseysforsalewholesaler.com -ningessaybe.me -nodup.ru -nonameread45.live -norththeface.store -nowtorrents.com -oakridgemo.com -oda.as -office-windows.ru -ohmyrings.com -oil-td.ru -olympescort.com -onemantrip.com -onlinefilmz.net -onlineku.com -onlineporno.site -panchro.co.uk +nfljerseyscheapbiz.us +nightvision746.weebly.com +no-fuel.org +no-rx.info +oivcvx.website +oneminutesite.it +onescreen.cc +onlinemegax.com +online-podarki.com +onlyforemont.ru +opinionreelle.com +orel-reshka.net +orgasmatrix.com +outrageousdeal-a.akamaihd.net parajumpersjakkesalgnorge.info -park.above.com -pcgroup.com.uy -pcimforum.com +parajumpersstore.online +partner-stop.men +partnerworkroom.men +pdns.cz pepperstyle.ru -perm.xrus.org -pewit.pw -photorepair.ru -pillscheap24h.com -pizdopletka.club +petrovka-online.com +phobia.us +photostudiolightings.com +piccdata.com +pinsdaddy.com +pipki.r.acdnpro.com +pivka.xyz plaff-go.ru -plusnetwork.com -pl-vouchers.com -podshipniki-ntn.ru -poems.com.ua -poffet.net -pogruztehnik.ru -porn555.com -pornofiljmi.com -pornosexrolik.com -porno.simple-image.com.ua -portside.cc -posible.net -pozdrawleniya.ru -prizesbook.online -produkto.net -promodj.com -promover.org -prosperent.com -prostitutki-kiev.org -prostitutki-tyumeni.org -pts163.ru -qualitymarketzone.com -quickcashlimited.com -quickchange.cc -razorweb-a.akamaihd.net -redhotfreebies.co.uk -reelheroes.net -reklama-i-rabota.ru -rentehno.ru -ring4rhino.com -rogervivierforsale.com -rome2rio.com -rossanasaavedra.net -rubbed.us -ruex.org.ua -rusexy.xyz -russian-postindex.ru -rvtv.ru -rybalka-opt.ru -sajatvelemeny.com -sanidumps.com -saugatuck.com -sayyoethe.blogspot.co.za -sbricur.com -scalerite.co.za -scanner-brian.top -scanner-don.top -scopich.com -score-ads.men -secret.xn--oogle-wmc.com -securityallianceservices.com -seeresultshub-a.akamaihd.net -segol.tv -seo-2-0.com -seoanalyses.com -servicecenter.co.ua -service-core.ru -sex-sex-sex5.com +pojdelo.weebly.com +pomoc-drogowa.cba.pl +popup-fdm.xyz +porndairy.in +pornogratisdiario.com +poshiv-chehol.ru +pravoholding.ru +preventheadacheguide.info +privatefx-in.ru +professionaldieselcare.com +proposal-engine.com +prpops.com +rapevideosmovies.com +razleton.com +recipedays.ru +rednise.com +ribieiendom.no +rock-cafe.info +rocketchange.ru +rollercoin.com +rospromtest.ru +roznica.com.ua +ruspdd.com +rutor.vip +s1z.ru +safe-app.net +sampleletters.net +sanatorrii.ru +sape.top +sashagreyblog.ga +satoristudio.net +scanner-olga.top +scanner-viktor.top +scoopquest.com +searchimage.co +searchinquire.com +se.bnt-team.com +sei80.com +semprofile.com +semrush.com +seo-platform.com +seorussian.ru +sexad.net +sexphoto.site +sex-pr.net sexstream.pl -sexxdate.net -shama-rc.net -shemalegalls.blogporn.in -sherlock.se -shiksabd.com -shtora66.ru -sirpornogratis.xxx -skuteczna-dieta.co.pl -sky-mine.ru -skytraf.xyz -sluganarodu.ru -social-buttons.com +sfd-chess.ru +sharebutton.to +shopvilleroyboch.com.ua +shymkent.xkaz.org +signal03.ru +silvercash.com +simplepooltips.com +sispe.com.br +sitevaluation.org +s-iwantyou.com +slimcdn.com +slonechka.ru +slutloadlive.com +smallseotools.com +sms2x2.ru social-fun.ru social-s-ggg.xyz -socialtrade.biz -sodexo.com -softomix.net softwaretrend.net -sovetogorod.ru -sovinsteel.ru -spl63.fr +sonata-arctica.wz.cz +soundfrost.org +spaceshipad.com sport7777.net -sports-supplements.us +spravka-medosmotr.ru +spy-app.info +spymac.net spywarespy.com -srecorder.com +squidoo.com stard.shop -stats-public.grammarly.io -stoki.ru -sukirgenk.dvrlists.com -summerlinhomes411.info -supermesta.ru -sureone.pro -svarkagid.com -svnuppsalaorebro.se -swagbucks.com -swinger-mobil.net -swinginwithme.ru -tabakur77.com -taboola.com -tagil.zrus.org -telegraf.by -thaisamkok.com -thegioixekhach.com -theguardlan.com -thruport.com +stromerrealty.com +styro.ru +superoboi.com.ua +supers.com.ua +surintech.ac.th +svetodiodoff.ru +szamponrevita.pl +tattomedia.com +technopellet.gr +telefonsexi.com +telefonsex-ohne0900.net +tengohydar.tk +terrafootwear.us tizanidinecost.blogspot.com -tizanidinehydrochloride2mg.blogspot.com -tizanidinepills.blogspot.com -tizanidinerecreationaluse.blogspot.com -tizanidinesleepaid.blogspot.com +tizanidineshowupondrugtest.blogspot.com +tizanidinesideeffectsweightloss.blogspot.com tkanorganizma.ru -tnaionline.org -tomck.com -toondinsey.com -toonfamilies.net -tovaroboom.vast.ru -traderzplanet.in -traffixer.com -trahic.ru -trahvid.com -trainoffend.ml -treasuretrack-a.akamaihd.net -truck-addzilla.life -trustaffs.com -try-rx.com +tn811.us +tongkatmadura.info +top1-seo-service.com +topclickguru.com +totu.info +traffic2money.com +tripper.de tsstcorpcddvdwshbbdriverfb.aircus.com -tytoona.com -ukkala.xyz +tvoystartup.ru +tvteleport.ru +uginekologa.com uk-zheu20.ru -ultimateclassicrock.com -ultimatesetnewfreeallsoftupgradesystems.pw -usacasino.com -validccseller.com -vashsvet.com +umityangin.net +univerfiles.com +unpredictable.ga +uptime-delta.net +uptime-gamma.net +uroffer.link +vacances-voyages.info +valsalud.com +vancleefreplica.pw +vapeface.club +vavilone.com vbikse.com -venerologiya.com -viagengrarx.com -viagroid.ru -videositename.com -vipsiterip.org -viteonlusarezzo.it -viven.host.sk -vizitki.net -vodkoved.ru -vonradio.com +v-doc.co +vgoloveboli.net +videogamesecrets.com +videotuber.ru +villakohlanta.nu +vip.51.la +visa-pasport.ru +vvpg.ru vykupavto-krasnodar.ru -wapsite.me -wareseeker.com -waterefficiency.co -webads.co.nz -website-analytics.online -westermarkanjou.se -wheelchairliftsarea.com -whengirlsgowild.com -wikes.20fr.com -winterclassichockeyjerseys.com -wosik-dach.service-for-web.de -writersgroup580.web.fc2.com +vzube.com +wallpapers-best.com +warezkeeper.com +web.cvut.cz +wechatdownload10.com +westen-v.life +winwotgold.pl +withstandingheartwarming.com +wnoz.de +woodyguthrie.se +world-mmo.com +worldmusicfests.com +worst-sites.online +wpsecurity.website wstroika.ru -www.arenda-yeisk.ru -wygraj-skiny.win -xgames-04.com -xjrul.com -x-lime.com +www.event-tracking.com +x5market.ru +xjlottery.com +xkaz.org +xlolitka.com +xmladserver.com +xmronta.com +xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai xn------7cdbapdecfd4ak1bn0amjffj7afu3y.xn--p1ai -xn--80ahvj9e.xn--p1ai -xn----9sbebi2bvzr7h.xn--p1ai -xn--oogle-wmc.com -x-true.info -xzlive.com -yellowfootprints.com -zaidia.xhost.ro -zakonobosago.ru -zaobao.com.sg -zarepta.com -zebradudka.com -zed21.net -zeikopay.com -zeroredirect10.com -zigarettenonlinekaufen1.bloog.pl -znaniyapolza.ru -zrelaya.pw +xn----7sbaphztdjeboffeiof6c.xn--p1ai +xn--80ahdheogk5l.xn--p1ai +xn--80aikhbrhr.net +xn----itbkqkfiq.xn--p1ai +xolodremont.ru +xtremeeagles.net +xxxrus.org +yachts-cruise.info +yaoguangdj.com +ynymnwbm.bloger.index.hr +yoshkarola.zrus.org +your-bearings.com +youtubologia.it +yugo-star.ru +zaimite.ru +zakazvzloma.com +zeroredirect7.com +zirondelli.it +zlatnajesen.com +zojirushi-products.ru +zophim.me zscalerone.net +zskdla.site +zvuker.net diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index 81c49c42a..785374a1a 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -00webcams.com -0lovespells0.blogspot.com -123cha.com -123movies.love -2girls1cup-free.com -3-letter-domains.net -4replicawatch.net -5kstore.com -7search.com -9icmzvn6.website -9med.net -aa81bf391151884adfa3dd677e41f94be1.com -abcdefh.xyz -abclauncher.com -absolutelyanalog.com -adultnet.in -airmaxshoes-2016.com -alf-img.com -all4wap.ru -alltheviews.com -alot.com -anaksma.info -ananumous.ru -android-style.com -anti-virus-removal.info -apelsinnik.website -ascat.porn -asmxsatadriverin.aircus.com -avto-oligarh.ru -b2b-lounge.com +1-free-share-buttons.com +1xbet4you.com +2rich4bitches.com +4rent.ru +7zap.com +8jn.dating-app.ru +a96527gi.beget.tech +aa625d84f1587749c1ab011d6f269f7d64.com +adamoads.com +adimmix.com +adultmeetonline.info +adultwebhosting.info +affordablewebsitesandmobileapps.com +ahrefs.com +akusherok.ru +alfa-medosmotr.ru +allforminecraft.ru +allnews24.in +alloysteel.ru +allstatesugarbowl.org +aloofly.com +arclk.net +asscenihotosidea.blogspot.com +autoblogger24.info +autowebmarket.com.ua +azartmsl.com backlink4u.net -bagsonsale.online -balook.com -bestbuy.ca +backuperwebcam.weebly.com +beclean-nn.ru bezlimitko.xyz -big-cash.net -bkgr.se -blavia.00author.com -blogs.rediff.com -bobba.dzaba.com -bonzibuddi.com -brakehawk.com -brus.city -burn-fat.ga -buyhoverboard.com -californianews.cf -cascadelink.org -cayado.snn.gr -cbcseward.com -chimiver.info -chlooe.com -chuckguilford.com -ciceros.co -clickcash.com +bfz.biz +bigames.online +bjetjt.com +bkns.vn +blackcurranthumidifiers.site +blockety.co +bonzbuddy.com +brainboostingsupplements.org +bscodecs.com +buqyxa.rincian.info +buyk.host.sk +bvlgariwallet.xyz +call-of-duty.info +canoncdriverq3.pen.io +cartierbraceletsreplica.pw +cheapwebsitehoster.com +cheatcode-lita12.rhcloud.com +chelyabinsk.xrus.org +classiquebijoux.ru +cleanallspyware.com clubfashionista.com +cocyq.inwtrade.com +compliance-willy.top com-supportcenter.website -copblock.org -covadhosting.biz -creditnation.ru -cubook.supernew.org -culmimg.pw -czat.wp.pl -dafatiri.com -dairyindia.in -dame-ns.kz -demian.kz -diarioaconcagua.com -dichvuvesinhhanoi.com -discountliv.com -doctissimo.fr -dostavimvdom.ru -doublepimp.com -downloadkakaotalk.com -e705.net -easycommerce.cf -edwinkonijn.com.au -enternet.ee -erotische-geschichten-xxl.com -escortplius.com -europeanwatches.ru +cookingmeat.ru +coslab.club +counter.yadro.ru +crd.clan.su +cubs-jerseys.us +daydream-studio.ru +dekorkeramik.ru +dgfitness.co +dipstar.org +dirtpics.pw +domainsatcost.com +dotmass.top +download-wallpaper.net +drunkmoms.net +druzhininevgeniy63.blogspot.com +eandsgallery.com +earn-from-articles.com +ekto.ee +elitepcgames.com +engines-usa.com +eonpal.com everytalk.tv -expolicenciaslatam.co -extlinks.com -eywords-monitoring-your-success.com -ezigarettenkaufen1.deviantart.com -ezigaretteshop.webs.com -ezrvrentals.com -fenoyl.batcave.net -financetip.eu -folowsite.com -forex-indextop20.ru -fortevidyoze.net -freewebs.com -freewlan.info -g33.org -galaxyflowers.ru -gamerscorps.com -gcup.ru -get-your-social-buttons.info -gktt.ru -go2jump.org -gogalleryawesome.com -goodwinmetals.co -guigyverpo.cf -havepussy.com -hentaiheroes.com -hermesreplica.win -hesteel.pl -highspeed5.net -himgaws.pw -hol.es -howmuchdoestizanidinecost.blogspot.com -hspline.com -iboss.com -ilovevitaly.xyz -images.gyffu.com +exoneration-project.us +eyessurgery.ru +fable.in.ua +failingmarriege.blogspot.com +fashionavenuegame.com +fettefrauen.net +fickluder69.com +filkhbr.com +findclan.org +fitness-video.net +fiverr.com +flavors.me +forum69.info +forums.toucharcade.com +fotosfotos.eu +freejabs.com +free-social-buttons.xyz +fxtips.ru +gambarxkata.co +gdcentre.ru +germetiki.com.ua +getridofstretchmarks.org +gettpromos.com +gk170.ru +glopages.ru +gotwebsite1.com +gsmlab.pl +helpmymacfaster.trade +homelygarden.com +hoverboardforsaledirect.com +howlongdoestizanidinestayinyoursystem.blogspot.com +hplaserjetpdriver8y.pen.io +humanorightswatch.org +igithab.com +iinstalll-fii1leis.jus0wil.pp.ua +imediadesk.com i-midias.net.br -instakink.com -instasexyblog.com +immigrational.info +infokonkurs.ru +infowarcraft.ru in-tandem.co -iomoio.net -istripper.com -izoll.ru -janavibekken.no -jeremyeaton.co -jerseysbizwholesalecheap.com -jjbabskoe.ru -jmat.cn -joingames.org -kimcurlrvsms.com -kino-filmi.com -klikbonus.com -languagecode.com -larose.jb2c.me -law-check-two.xyz -ledis.top -lenvred.org -letmacworkfaster.site -letslowbefast.today -liveinternet.ro -ljusihus.se -lomza.info -lubetube.com -marmitaco.cat -matchpal-a.akamaihd.net -matrixalchemy.com -medizinreisen.de -meetlocalchicks.com -me-ke.com -miniature.io -mobstarr.com -mojowhois.com -mp3ringtone.info -my-bc.ru -narosty.com -nerudlogistik.ru -newsperuse.com -nextrent-crimea.ru -nfvsz.com -nodup.ru -offf.info -officedocuments.net -okonich.com.ua -o.light.d0t.ru -oneclickfiles.com -oneshotdate.com -ooo-gotovie.ru -optitrade24.com -osoznanie-narkotikam.net -ozoz.it -paceform.com -paleohub.info -panchro.co.uk -pcimforum.com -pc-test.net -pladform.ru -play-movie.pl -pogosh.com -popunder.ru -powitania.pl -prize44.com -pro-okis.ru -prostitutki-yaroslavlya.org -pukaporn.com -puteshestvennik.com -rapidgator-porn.ga -rethinkwasteni.info -richinvestmonitor.com -ru-dety.ru -ruf777.com -salmonfishingsacramentoriver.com -scanner-donald.top -scanner-julianna.top -scopich.com -sdsjweb.com -securesmrt-dt.com -sendearnings.com -seocheckupx.com -serptehnika.ru -sexpornotales.net -sharpchallenge.com -sibecoprom.ru -simplepooltips.com -sindragosa.comxa.com -s.lollypopgaming.com -softtor.com -soserfis.com -space-worry.ml -spb.ru -spin2016.cf -sport7777.net -steame.ru -stroilka.info -strv.se -sygraem.com -tecspb.ru -tehngr.ru -telemetryverification.net -templates.radiodigital.co -thebluffs.com -thedownloadfreeonlinegames.blogspot.com -thefotosgratis.eu -thfox.com -threecolumnblogger.com -time-japan.ru -tomck.com -torrentgamer.net +intimshop-fantasy.ru +int.search.mywebsearch.com +iptool.xyz +it-max.com.ua +japan-bearings.ru +jumptap.com +jurajskie.info +justprofit.xyz +kambasoft.com +karaganda.xkaz.org +kasino-money.pw +kazan.xrus.org +kaz.kz +keywordblocks.com +keywordhut.com +kinostorm.net +krynica.info +lacasamorett.com +lflash.ru +libertybilisim.com +likesdesign.com +linkrr.com +machicon-ueno.info +maju.bluesalt.co +mamasuper.prom.ua +mantramusic.ru +microsoftportal.net +mielec.pl +minneapoliscopiers.com +moncleroutletonline.xyz +moneymaster.ru +moscow-region.ru +mydeathspace.com +myphotopipe.com +myplaycity.com +narkologiya-penza.ru +networkad.net +newhairstylesformen2014.com +niroo.info +oakridgemo.com +offside2.5v.pl +okayimage.com +olympescort.com +onetravelguides.com +optom-deshevo.ru +oren-cats.ru +ostrovtaxi.ru +paparazzistudios.com.au +pawli.eu +pdamods.ru +perfectpracticeweb.com +perl.dp.ua +php-market.ru +physfunc.ru +picturesmania.com +piter.xrus.org +pliks.pl +porn555.com +pornzone.tv +preparevideosafesystem4unow.site +prodvigator.ua +prospekt-st.ru +prostitutki-almata.org +psa48.ru +pst2017.onlinewebshop.net +pts163.ru +purplesphere.in +qld10000.net +qwesa.ru +redbottomheels.xyz +redhotfreebies.co.uk +reimageplus.com +remont-mobile-phones.ru +rootandroid.org +rubanners.com +ruscopybook.com +sale-japan.com +sawin.beth.webd.pl +scanner-willy.top +seadragonherbery.com +sharebutton.org +sigmund-freud.co.uk +simul.co +site3.free-share-buttons.com +snip.to +sofit-dmd.ru +softxaker.ru +sotechco.co +special-porn.com +speedup-my.site +sribno.net +sstroy44.ru +suggest-keywords.com +sukarame.net +supernew.org +t3chtonic.com +tacbibirfa.tk +telegraf.by +thebitcoincode.com +thegolfclub.info +thelottosecrets.com +themeforest.net +thenews-today.info +timer4web.com +tizanidine4mgstreetprice.blogspot.com +tizanidinefibromyalgia.blogspot.com +tizanidinevscyclobenzaprine.blogspot.com +topdownloads.ru treasuretrack-a.akamaihd.net -truck-addzilla.life -trudogolik.net -uamtrk.com -uhdtv.website -ussearche.cf -vasileostrovsky-rayon.ru -vduplo.ru -viagroid.ru -vkonche.com -waterpurifier.club -wildworld.site -wingsoffury2.com -wordpress-crew.net -xclicks.net -xfire.com -xn----ctbbcjd3dbsehgi.xn--p1ai -xn--e1afanlbnfckd7c3d.xn--p1ai -yaminecraft.ru -yourporn.com -zarabotok--doma.ru -zeg-distribution.com -zolotoy-lis.ru +trustl.life +typimga.pw +umg-stroy.ru +url2image.com +urlcut.ru +varbergsvind.se +vellings.info +venta-prom.ru +warning.or.kr +webextract.profound.net +websiteaccountant.de +weburok.com +worst-sites.online +xcombear.ru +xdoza.com +xlolitka.com +xlovecam.com +x-musics.com +yottos.com +yourmovies.pl +zaobao.com.sg +zarabiaj-dzis.pl +znakomstvaonlain.ru +zvetki.ru diff --git a/.dev-tools/test_units/whitelist-ips.conf b/.dev-tools/test_units/whitelist-ips.conf index cc2baf875..dd5291a4a 100644 --- a/.dev-tools/test_units/whitelist-ips.conf +++ b/.dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 0; +104.154.182.187 0; 127.0.0.1 0; diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index ff0ed3477..4502f7cc6 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index a6dbe5bf1..b6f16959a 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index ab53dab49..a2d8b4cec 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.01.2229 +#### Version: V4.2021.01.2230 #### Bad Referrers Blocked: [6853](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [583](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [587](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index f80c38c09..803d0dbe8 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.01.2229 +#### Version: V4.2021.01.2230 #### Bad Referrers Blocked: [6853](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [583](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [587](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index bdb3319df..55b8cbe30 100755 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.01.2229 +#### Version: V4.2021.01.2230 #### Bad Referrers Blocked: [6853](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [583](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [587](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index 40a74f19a..5abfb27a3 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,7 +1,3 @@ -Buck -ZoominfoBot -Petalbot -SentiBot 360Spider 404checker 404enemy @@ -62,6 +58,7 @@ Bolt BotALot Brandprotect Brandwatch +Buck Buddy BuiltBotTough BuiltWith @@ -358,6 +355,7 @@ Pavuk pcBrowser PECL::HTTP PeoplePal +Petalbot PHPCrawl Picscout Picsearch @@ -412,6 +410,7 @@ Searchestate SearchmetricsBot Semrush SemrushBot +SentiBot SEOkicks SEOkicks-Robot SEOlyticsCrawler @@ -583,5 +582,6 @@ Zeus zgrab Zitebot ZmEu +ZoominfoBot ZumBot ZyBorg diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 718652724..7eb1438e3 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.01.2229 +#### Version: V4.2021.01.2230 #### Bad Referrers Blocked: [6853](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [583](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [587](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 4efc70554..a4ec81574 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2229 -### Updated: Sat Jan 2 14:16:54 SAST 2021 +### Version: V4.2021.01.2230 +### Updated: Sat Jan 2 15:38:24 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index f2cc431bf..ee9f42d80 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.01.2229 -### Updated: Sat Jan 2 14:16:55 SAST 2021 +### Version: V3.2021.01.2230 +### Updated: Sat Jan 2 15:38:25 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 4efc70554..a4ec81574 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2229 -### Updated: Sat Jan 2 14:16:54 SAST 2021 +### Version: V4.2021.01.2230 +### Updated: Sat Jan 2 15:38:24 SAST 2021 ### Bad Referrer Count: 6853 -### Bad Bot Count: 583 +### Bad Bot Count: 587 ################################################### ### VERSION INFORMATION ## @@ -207,6 +207,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)BotALot(?:\b)" 3; "~*(?:\b)Brandprotect(?:\b)" 3; "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buck(?:\b)" 3; "~*(?:\b)Buddy(?:\b)" 3; "~*(?:\b)BuiltBotTough(?:\b)" 3; "~*(?:\b)BuiltWith(?:\b)" 3; @@ -503,6 +504,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)pcBrowser(?:\b)" 3; "~*(?:\b)PECL::HTTP(?:\b)" 3; "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)Petalbot(?:\b)" 3; "~*(?:\b)PHPCrawl(?:\b)" 3; "~*(?:\b)Picscout(?:\b)" 3; "~*(?:\b)Picsearch(?:\b)" 3; @@ -557,6 +559,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SentiBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; @@ -728,6 +731,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 201d3f0b9..1e19ba421 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2021.01.2229 -### Updated: Sat Jan 2 14:16:56 SAST 2021 -### Bad Bot Count: 583 +### Version: V4.2021.01.2230 +### Updated: Sat Jan 2 15:38:26 SAST 2021 +### Bad Bot Count: 587 ################################################### ### Version Information ## @@ -135,6 +135,8 @@ User-agent: Brandprotect Disallow:/ User-agent: Brandwatch Disallow:/ +User-agent: Buck +Disallow:/ User-agent: Buddy Disallow:/ User-agent: BuiltBotTough @@ -727,6 +729,8 @@ User-agent: PECL::HTTP Disallow:/ User-agent: PeoplePal Disallow:/ +User-agent: Petalbot +Disallow:/ User-agent: PHPCrawl Disallow:/ User-agent: Picscout @@ -835,6 +839,8 @@ User-agent: Semrush Disallow:/ User-agent: SemrushBot Disallow:/ +User-agent: SentiBot +Disallow:/ User-agent: SEOkicks Disallow:/ User-agent: SEOkicks-Robot @@ -1177,6 +1183,8 @@ User-agent: Zitebot Disallow:/ User-agent: ZmEu Disallow:/ +User-agent: ZoominfoBot +Disallow:/ User-agent: ZumBot Disallow:/ User-agent: ZyBorg