diff --git a/.dev-tools/_robots_input/robots-input.txt b/.dev-tools/_robots_input/robots-input.txt index 04cc54a72..448edc61e 100644 --- a/.dev-tools/_robots_input/robots-input.txt +++ b/.dev-tools/_robots_input/robots-input.txt @@ -80,6 +80,7 @@ Claritybot Cliqzbot Cloud mapping coccocbot-web +Cocolyzebot CODE87 Cogentbot cognitiveseo @@ -244,6 +245,7 @@ Libwhisker LieBaoFast Lightspeedsystems Likse +Linkbot Linkdexbot LinkextractorPro LinkpadBot @@ -505,6 +507,7 @@ Vacuum Vagabondo VB Project VCI +VelenPublicWebCrawler VeriCiteCrawler VidibleScraper Virusdie @@ -582,6 +585,7 @@ Zeus zgrab Zitebot ZmEu +ZoomBot ZoominfoBot ZumBot ZyBorg diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 32430a294..7b6055cbc 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.01.2238 -### Updated: Mon Jan 4 11:10:47 SAST 2021 +### Version: V3.2021.01.2239 +### Updated: Mon Jan 4 12:00:45 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 180bdc087..18b642a16 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 1; +104.154.182.187 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index bb50a4a64..dd5291a4a 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 0; +104.154.182.187 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 32430a294..7b6055cbc 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.01.2238 -### Updated: Mon Jan 4 11:10:47 SAST 2021 +### Version: V3.2021.01.2239 +### Updated: Mon Jan 4 12:00:45 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 180bdc087..18b642a16 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 1; +104.154.182.187 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index bb50a4a64..dd5291a4a 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 0; +104.154.182.187 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 32430a294..7b6055cbc 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.01.2238 -### Updated: Mon Jan 4 11:10:47 SAST 2021 +### Version: V3.2021.01.2239 +### Updated: Mon Jan 4 12:00:45 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 180bdc087..18b642a16 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 1; +104.154.182.187 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 6f08a6b3d..0688107c1 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -141,6 +141,7 @@ "~*(?:\b)Cliqzbot(?:\b|)" 0; "~*(?:\b)Cloud\ mapping(?:\b|)" 0; "~*(?:\b)coccocbot-web(?:\b|)" 0; + "~*(?:\b)Cocolyzebot(?:\b|)" 0; "~*(?:\b)CODE87(?:\b|)" 0; "~*(?:\b)Cogentbot(?:\b|)" 0; "~*(?:\b)cognitiveseo(?:\b|)" 0; @@ -305,6 +306,7 @@ "~*(?:\b)LieBaoFast(?:\b|)" 0; "~*(?:\b)Lightspeedsystems(?:\b|)" 0; "~*(?:\b)Likse(?:\b|)" 0; + "~*(?:\b)Linkbot(?:\b|)" 0; "~*(?:\b)Linkdexbot(?:\b|)" 0; "~*(?:\b)LinkextractorPro(?:\b|)" 0; "~*(?:\b)LinkpadBot(?:\b|)" 0; @@ -566,6 +568,7 @@ "~*(?:\b)Vagabondo(?:\b|)" 0; "~*(?:\b)VB\ Project(?:\b|)" 0; "~*(?:\b)VCI(?:\b|)" 0; + "~*(?:\b)VelenPublicWebCrawler(?:\b|)" 0; "~*(?:\b)VeriCiteCrawler(?:\b|)" 0; "~*(?:\b)VidibleScraper(?:\b|)" 0; "~*(?:\b)Virusdie(?:\b|)" 0; @@ -643,6 +646,7 @@ "~*(?:\b)zgrab(?:\b|)" 0; "~*(?:\b)Zitebot(?:\b|)" 0; "~*(?:\b)ZmEu(?:\b|)" 0; + "~*(?:\b)ZoomBot(?:\b|)" 0; "~*(?:\b)ZoominfoBot(?:\b|)" 0; "~*(?:\b)ZumBot(?:\b|)" 0; "~*(?:\b)ZyBorg(?:\b|)" 0; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index bb50a4a64..dd5291a4a 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 0; +104.154.182.187 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 32430a294..7b6055cbc 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.01.2238 -### Updated: Mon Jan 4 11:10:47 SAST 2021 +### Version: V3.2021.01.2239 +### Updated: Mon Jan 4 12:00:45 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 32430a294..7b6055cbc 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.01.2238 -### Updated: Mon Jan 4 11:10:47 SAST 2021 +### Version: V3.2021.01.2239 +### Updated: Mon Jan 4 12:00:45 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 04d512a2a..8623b4e60 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2238 -### Updated: Mon Jan 4 11:10:46 SAST 2021 +### Version: V4.2021.01.2239 +### Updated: Mon Jan 4 12:00:44 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 52422de58..22793dd10 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.154.182.187 0; +35.193.184.18 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 04d512a2a..8623b4e60 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2238 -### Updated: Mon Jan 4 11:10:46 SAST 2021 +### Version: V4.2021.01.2239 +### Updated: Mon Jan 4 12:00:44 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 701efaf43..180bdc087 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 1; +34.122.173.211 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index d99c8cd8c..bb50a4a64 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 0; +34.122.173.211 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 04d512a2a..8623b4e60 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2238 -### Updated: Mon Jan 4 11:10:46 SAST 2021 +### Version: V4.2021.01.2239 +### Updated: Mon Jan 4 12:00:44 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 701efaf43..180bdc087 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 1; +34.122.173.211 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index d99c8cd8c..bb50a4a64 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 0; +34.122.173.211 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 04d512a2a..8623b4e60 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2238 -### Updated: Mon Jan 4 11:10:46 SAST 2021 +### Version: V4.2021.01.2239 +### Updated: Mon Jan 4 12:00:44 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 701efaf43..180bdc087 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 1; +34.122.173.211 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 6f08a6b3d..0688107c1 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -141,6 +141,7 @@ "~*(?:\b)Cliqzbot(?:\b|)" 0; "~*(?:\b)Cloud\ mapping(?:\b|)" 0; "~*(?:\b)coccocbot-web(?:\b|)" 0; + "~*(?:\b)Cocolyzebot(?:\b|)" 0; "~*(?:\b)CODE87(?:\b|)" 0; "~*(?:\b)Cogentbot(?:\b|)" 0; "~*(?:\b)cognitiveseo(?:\b|)" 0; @@ -305,6 +306,7 @@ "~*(?:\b)LieBaoFast(?:\b|)" 0; "~*(?:\b)Lightspeedsystems(?:\b|)" 0; "~*(?:\b)Likse(?:\b|)" 0; + "~*(?:\b)Linkbot(?:\b|)" 0; "~*(?:\b)Linkdexbot(?:\b|)" 0; "~*(?:\b)LinkextractorPro(?:\b|)" 0; "~*(?:\b)LinkpadBot(?:\b|)" 0; @@ -566,6 +568,7 @@ "~*(?:\b)Vagabondo(?:\b|)" 0; "~*(?:\b)VB\ Project(?:\b|)" 0; "~*(?:\b)VCI(?:\b|)" 0; + "~*(?:\b)VelenPublicWebCrawler(?:\b|)" 0; "~*(?:\b)VeriCiteCrawler(?:\b|)" 0; "~*(?:\b)VidibleScraper(?:\b|)" 0; "~*(?:\b)Virusdie(?:\b|)" 0; @@ -643,6 +646,7 @@ "~*(?:\b)zgrab(?:\b|)" 0; "~*(?:\b)Zitebot(?:\b|)" 0; "~*(?:\b)ZmEu(?:\b|)" 0; + "~*(?:\b)ZoomBot(?:\b|)" 0; "~*(?:\b)ZoominfoBot(?:\b|)" 0; "~*(?:\b)ZumBot(?:\b|)" 0; "~*(?:\b)ZyBorg(?:\b|)" 0; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index d99c8cd8c..bb50a4a64 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 0; +34.122.173.211 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 04d512a2a..8623b4e60 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2238 -### Updated: Mon Jan 4 11:10:46 SAST 2021 +### Version: V4.2021.01.2239 +### Updated: Mon Jan 4 12:00:44 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 361f3a3d5..c8638035f 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.193.184.18 0; +104.154.182.187 0; 34.122.173.211 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index cd8f0c28d..7cf21c13b 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2238 -### Updated: Mon Jan 4 11:10:46 SAST 2021 +### Version: V4.2021.01.2239 +### Updated: Mon Jan 4 12:00:44 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 2d9c3c5be..c4667b0ac 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.193.184.18 0; -104.154.182.187 0; +34.122.208.80 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 0b8f391bd..8ad49e015 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2238 -### Updated: Mon Jan 4 11:10:46 SAST 2021 +### Version: V4.2021.01.2239 +### Updated: Mon Jan 4 12:00:44 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index 3d512ce96..a398860de 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.122.173.211 0; +35.193.184.18 0; +34.122.208.80 0; 104.154.182.187 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 04d512a2a..8623b4e60 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2238 -### Updated: Mon Jan 4 11:10:46 SAST 2021 +### Version: V4.2021.01.2239 +### Updated: Mon Jan 4 12:00:44 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 6cb558faf..361f3a3d5 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.154.182.187 0; 35.193.184.18 0; -34.122.208.80 0; +34.122.173.211 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 04d512a2a..8623b4e60 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2238 -### Updated: Mon Jan 4 11:10:46 SAST 2021 +### Version: V4.2021.01.2239 +### Updated: Mon Jan 4 12:00:44 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 180bdc087..18b642a16 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 1; +104.154.182.187 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/blacklist-user-agents.conf b/.dev-tools/test_units/blacklist-user-agents.conf index 6f08a6b3d..0688107c1 100644 --- a/.dev-tools/test_units/blacklist-user-agents.conf +++ b/.dev-tools/test_units/blacklist-user-agents.conf @@ -141,6 +141,7 @@ "~*(?:\b)Cliqzbot(?:\b|)" 0; "~*(?:\b)Cloud\ mapping(?:\b|)" 0; "~*(?:\b)coccocbot-web(?:\b|)" 0; + "~*(?:\b)Cocolyzebot(?:\b|)" 0; "~*(?:\b)CODE87(?:\b|)" 0; "~*(?:\b)Cogentbot(?:\b|)" 0; "~*(?:\b)cognitiveseo(?:\b|)" 0; @@ -305,6 +306,7 @@ "~*(?:\b)LieBaoFast(?:\b|)" 0; "~*(?:\b)Lightspeedsystems(?:\b|)" 0; "~*(?:\b)Likse(?:\b|)" 0; + "~*(?:\b)Linkbot(?:\b|)" 0; "~*(?:\b)Linkdexbot(?:\b|)" 0; "~*(?:\b)LinkextractorPro(?:\b|)" 0; "~*(?:\b)LinkpadBot(?:\b|)" 0; @@ -566,6 +568,7 @@ "~*(?:\b)Vagabondo(?:\b|)" 0; "~*(?:\b)VB\ Project(?:\b|)" 0; "~*(?:\b)VCI(?:\b|)" 0; + "~*(?:\b)VelenPublicWebCrawler(?:\b|)" 0; "~*(?:\b)VeriCiteCrawler(?:\b|)" 0; "~*(?:\b)VidibleScraper(?:\b|)" 0; "~*(?:\b)Virusdie(?:\b|)" 0; @@ -643,6 +646,7 @@ "~*(?:\b)zgrab(?:\b|)" 0; "~*(?:\b)Zitebot(?:\b|)" 0; "~*(?:\b)ZmEu(?:\b|)" 0; + "~*(?:\b)ZoomBot(?:\b|)" 0; "~*(?:\b)ZoominfoBot(?:\b|)" 0; "~*(?:\b)ZumBot(?:\b|)" 0; "~*(?:\b)ZyBorg(?:\b|)" 0; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 7e97ee20e..255ed20f4 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -80legs -GermCrawler -GetRight -IDBTE4M -NetSpider -OrangeSpider -Semrush -Spbot -VidibleScraper -Zermelo +DomainCrawler +ExtractorPro +LinqiaRSSBot +Ninja +Scrapy +Sitevigil +sysscan +trendiction.com +Web Sauger +WebZIP diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index b91094489..4c3633f1b 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ 360Spider 404enemy -Aboundex -Aboundexbot -AfD-Verbotsverfahren -AhrefsBot -AIBOT -AiHitBot -AllSubmitter -Anarchie -Ankit -Anthill +80legs +Alexibot +Apexoo archive.org_bot -ASPSeek +arquivo-web-crawler +autoemailspider +AwarioSmartBot +Backlink-Ceck backlink-check BacklinkCrawler -BackWeb -BatchFTP -BetaBot -Black Hole -BlowFish -Brandprotect +BackStreet +Battleztar Bazinga +BDFetch +BLEXBot +Brandwatch Buck -Buddy -BuiltBotTough BuiltWith BunnySlippers Calculon -CATExplorador +CazoodleBot Cegbfeieh check1.exe -CheeseBot +CheTeam +ChinaClaw Chlooe Cloud mapping +CODE87 +Copier +CopyRightCheck Cosmos Craftbot crawler4j -CrazyWebCrawler +crawler.feedback Crescent -CrunchBot -CSHttp Curious Custo -CyotekWebCopy -Demon +DBLBot Deusu -DigitalPebble +Devil +Digincore DIIbot -Disco -Discobot -Discoverybot Dispatch -DittoSpyder -DnyzBot +DomainAppender DomainSigmaCrawler -Domains Project -Dragonfly -EasyDL -EirGrabber -EMail Siphon -EMail Wolf -EroCrawler -evc-batch -ExtLinksBot -Extractor -ExtractorPro -FHscan -Fimap -Firefox/7.0 -FyberSpider -Gigabot -GoZilla -Go!Zilla +domainsproject.org +Dotbot +Drip +DSearch +eCatch +Ecxi +Evil +Exabot +Express WebPictures +Ezooms +FlashGet +Foobot +Freeuploader +FrontPage +Fyrebot +Genieo +GermCrawler +GetRight +G-i-g-a-b-o-t +Go-Ahead-Got-It +Gotit +GrabNet Grafula -GrapeFX GridBot -Haansoft -Havij +HEADMasterSEO +heritrix +Hloader +HMView HTMLparser -Humanlinks HybridBot -IDBTE4M -IlseBot -Image Fetch +Iblog +Id-search +Image Sucker +IndeedBot +instabid +InternetSeer ips-agent -IRLbot isitwp.com IstellaBot Jbrofuzz JennyBot JetCar -JikeSpider -JOC Web Spider -Joomla Jorgee -Kinza -Kozmosbot -Lanshanbot -LeechGet -Libwhisker +JustView +Jyxobot +Larbin +LeechFTP +LibWeb LieBaoFast -Lightspeedsystems -LinqiaMetadataDownloaderBot -LinqiaRSSBot +Linkbot +LinksManager Lipperhey Spider -Litemage_walker -Lmspider LNSpiderguy LWP::Simple -Mag-Net +lwp-trivial +Magnet +magpie-crawler MarkMonitor +MarkWatch Masscan Mass Downloader Mata Hari -meanpathbot +Mb2345Browser MeanPath Bot -Mediatoolkitbot MegaIndex.ru Metauri -MIDown tool +MicroMessenger +Microsoft Data Access +Microsoft URL Control MIIxpc -Mister PiX Mojeek +Mojolicious Morfeus Fucking Scanner -MQQBrowser +Mr.4x3 MSFrontPage MSIECrawler Msrabot +muhstik-scan +Musobot +Name Intelligence Navroad -NearSite -Needle +NetAnts Netcraft +NetLyzer +NetMechanic Nettrack -Net Vampire -NetZIP +Netvibes NextGenSearchBot +Nibbler +NICErsPRO +Niki-bot NimbleCrawler Ninja -NPbot -Nutch +Nmap oBot -OnCrawl +Openfind OpenLinkProfiler OutclicksBot -OutfoxBot PageAnalyzer Papa Foto -Picsearch +PECL::HTTP +Picscout +Pi-Monster +PleaseCrawl +plumanalytics Probethenet -PyCurl -QueryN Metasearch -RankFlex +ProWebWalker +Pump +Quick-Crawler +RankActiveLinkBot RankingBot -Rankivabot -RankurBot -RealDownload -Reaper -RedesScrapy +RankingBot2 +RebelMouse RepoMonkey -Ripper -Rogerbot -s1z.ru +RocketCrawler +RSSingBot +satoristudio.net ScanAlert scan.lol Scrapy -Searchestate SearchmetricsBot SemrushBot SentiBot SEOkicks SEOkicks-Robot -SEOlyticsCrawler Seomoz SEOprofiler seoscanners -serpstatbot -sexsearcher +SEOstats Shodan -SiteCheckerBotCrawler +Siphon +Sitebeam sitechecker.pro SiteExplorer -Siteimprove -SiteLockSpider -SiteSnagger +SiteSucker Site Sucker -Sitevigil -SlySearch +SMTBot Snake -Snapbot -SocialRankIOBot -Sogou web spider -Sottopop -SpaceBison -Spammen SputnikBot -spyfu Sqlmap -Sqlworm Sqworm -Steeler -Stripper -Sucuri -SuperBot -SuperHTTP +Sucker SurveyBot sysscan +Szukacz T0PHackTeam T8Abot -tAkeOut Teleport -TeleportPro The Intraformant TheNomad +Thumbor Titan Toata -Turingos +Trendictionbot +trendiction.com +trendiction.de +True_Robot Turnitin +TwengaBot +Twice +UnisterBot +URLy Warning URLy.Warning -VeriCiteCrawler +VCI +VelenPublicWebCrawler +VidibleScraper Virusdie -WallpapersHD +Wallpapers/3.0 WBSearchBot -Web Auto +Webalta +WebBandit WebCollage -WebCopier WebEnhancer -Web Enhancer -Web Fetch -Web Fuck +WebGo IS WebImageCollector -Web Pix -WebsiteQuester -Website Quester -Webster +WebLeacher +WebmasterWorldForumBot +webmeup-crawler WebStripper -Web Sucker -WebWhacker +WebSucker +WebZIP +WeSEE +Whacker Who.is Bot Widow +WinHTTrack WiseGuys Robot -WISENutbot -Wonderbot +Woobot Wotbox +Wprecon WWW-Collector-E -xpymep1.exe -Zade +WWW-Mechanize +WWW::Mechanize +Xaldon WebSpider +YoudaoBot zauba.io +Zermelo Zeus +zgrab +Zitebot +ZmEu +ZoomBot +ZoominfoBot ZumBot -ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 06ae46236..e31b9bb1b 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -360Spider -404checker -404enemy -Acunetix -ADmantX -AhrefsBot -AiHitBot -Alexibot +80legs +Abonti +Aipbot AlphaBot Anarchie -Ankit Apexoo arquivo-web-crawler -Aspiegel -ASPSeek Asterias -Attach +autoemailspider AwarioSmartBot -BackDoorBot -BacklinkCrawler -BackWeb -Bandit +Backlink-Ceck +backlink-check +BackStreet +Badass BatchFTP -BBBike -BDCbot Bitacle -BLEXBot -Brandprotect -Brandwatch -BuiltWith -Bullseye -BunnySlippers -CATExplorador -CazoodleBot -Cloud mapping -coccocbot-web -Cogentbot +Blackboard +Black Hole +Blow +BlowFish +Boardreader +Bolt +Buddy +BuiltBotTough +Cegbfeieh +CheeseBot +CherryPicker +Claritybot +Cliqzbot +CODE87 cognitiveseo -com.plumanalytics Copier +CopyRightCheck +Copyscape Cosmos -Craftbot crawler4j -crawler.feedback -CrazyWebCrawler -Crescent CrunchBot Curious Custo -CyotekWebCopy DataCha0s +Deusu Devil Digincore -DigitalPebble +Dirbuster +Disco Discoverybot +Dispatch +DittoSpyder +DomainAppender DomainSigmaCrawler -Download Wonder Dragonfly Drip +DSearch EasyDL -Ebingbong -ECCP/1.0 -EirGrabber EMail Siphon EroCrawler evc-batch -ExtLinksBot -Extreme Picture Finder -EyeNetIE +Exabot +Express WebPictures Ezooms -Fimap -Firefox/7.0 +facebookscraper +FHscan Foobot FrontPage -FyberSpider -Genieo -GermCrawler -GetRight -GetWeb +Fyrebot +GalaxyBot +Gigablast +Gigabot GoZilla -Go!Zilla -Grabber GrabNet -GrapeshotCrawler +Grafula +GrapeFX GridBot -Haansoft Harvest HEADMasterSEO +heritrix +Hloader HTTrack -Iblog +Humanlinks Image Sucker -IndeedBot -Indy Library -InfoTekies +Intelliseek +InterGET InternetSeer internetVista monitor -IRLbot isitwp.com Iskanie JamesBOT Jbrofuzz -JennyBot Jetty JikeSpider -JOC Web Spider -JustView +Joomla +Jorgee +Jyxobot Kenjin Spider -Keyword Density Kinza -Lanshanbot -Larbin +Kozmosbot LeechFTP LeechGet LexiBot Lftp LibWeb -Linkdexbot +LieBaoFast +Lightspeedsystems +Likse +Linkbot LinkextractorPro -LinkScan +LinkpadBot +LinksManager +LinkWalker +LinqiaMetadataDownloaderBot +LinqiaScrapeBot Lipperhey -Lipperhey Spider -Litemage_walker +Lmspider LNSpiderguy -Magnet -magpie-crawler -Mail.RU_Bot -MarkWatch +lwp-request +LWP::Simple +Mag-Net +Majestic12 Masscan +Mass Downloader Mata Hari -MauiBot -Mediatoolkitbot +Mb2345Browser mediawords MicroMessenger Microsoft URL Control +MIDown tool +MIIxpc MJ12bot -Mojolicious -Morfeus Fucking Scanner -Mozlila +Mojeek MSIECrawler Msrabot -Musobot -Name Intelligence +muhstik-scan Needle -Nessus -netEstate NE Crawler NetMechanic NetSpider -Nettrack -Nimbostratus +NextGenSearchBot +Nibbler +Niki-bot Nmap NPbot oBot -OnCrawl +Offline Explorer Openfind +OpenVAS +OPPO A33 +OrangeSpider +OutclicksBot +OutfoxBot PageAnalyzer Page Analyzer -Pandalytics -Pavuk -Petalbot -PHPCrawl -Picscout -Picsearch +page scorer +Papa Foto +pcBrowser +PictureFinder Pimonster -PleaseCrawl -plumanalytics -POE-Component-Client-HTTP -Probethenet -RankActive -RankFlex -RankingBot2 -Rankivabot -RebelMouse +Pixray +Psbot +Pump +PxBroker +RealDownload +Reaper ReGet -RepoMonkey -RSSingBot +RocketCrawler +Rogerbot s1z.ru SalesIntelligent -SBIder ScanAlert +scan.lol +ScoutJet Scrapy -ScreenerBot -SemrushBot +Screaming +Searchestate +SearchmetricsBot +Semrush +SEOkicks SEOlyticsCrawler SEOprofiler -Shodan -SiteCheckerBotCrawler -SiteExplorer -Sitevigil -SlySearch -SmartDownload +seoscanners +SeoSiteCheckup +SEOstats +Sitebeam +sitechecker.pro +Siteimprove +SiteSnagger +SiteSucker +Site Sucker SMTBot -Snoopy +Snake SocialRankIOBot +Sociscraper +sogouspider +Sosospider +Spammen SpankBot -sp_auditbot +Spanner Spinn3r -Sqworm +Sqlmap +Stripper +Sucker Sucuri SuperBot +SuperHTTP Surfbot -Suzuran -sysscan -T8Abot -tAkeOut -TeleportPro -Telesoft -TheNomad -TightTwatBot -Toata -Tracemyfile -Trendiction +T0PHackTeam +Thumbor +Titan Trendictionbot Turingos -Turnitin -TwengaBot +TurnitinBot +Twice Upflow URLy Warning VB Project -VeriCiteCrawler VidibleScraper -Virusdie -VoidEYE -Voil -WallpapersHD -WebBandit +Wallpapers/3.0 +WASALive-Bot +WBSearchBot +WebCollage Web Collage WebCopier -WEBDAV -WebFetch -WebGo IS -WebImageCollector -WebLeacher +Web Fetch +WebmasterWorldForumBot webmeup-crawler -Web Pix WebSauger +Web Sauger WebsiteExtractor -Webster +WebsiteQuester +WebStripper Web Sucker WebWhacker WebZIP -Whack -Whacker -Who.is Bot WinHTTrack +WISENutbot +Wonderbot Woobot -Wotbox -WPScan +Wprecon +WWW-Collector-E +WWWOFFLE +x09Mozilla +Xaldon WebSpider +Xaldon_WebSpider +Xenu xpymep1.exe +zauba.io +zgrab Zitebot +ZmEu +ZoomBot +ZoominfoBot ZumBot diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index 9df30b9bb..2c4727bc7 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -24-ak.ru -codq.info -cutalltheshit.com -fashion-mk.net -fenoyl.batcave.net -hmmm.cz -lafourmiliaire.com -slavic-magic.ru -summerlinhomes411.info -vote-up.ru +absugars.com +dieswaene.com +eduinfosite.com +genericviagrasildenafiled.net +laudit.ru +prizefestival.mobi +rightenergysolutions.com.au +royalvegascasino.com +sexy-screen-savers.com +uhdtv.website diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index de0525b45..0165a72da 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -101billion.com -10bet.com -123any.com -1bet.com -1kinobig.ru -24-ak.ru -2ads.co.uk -2daytrendingnews.com -45en.ru -4replicawatch.net -4ureyesonly.com -6128786.com -8lufu.com -absugars.com -academiacsmendoza.org -actionnooz.com -actulite.com -adcash.com -adf.ly -adinterax.com -adpremium.org -ads.gold -adultfriendfinder.com -aero2.ru -a.frcls.fr -agapovdg.ru -ahrefs.com -akama.com +101lesbian.xyz +101raccoon.ru +123locker.com +123movies.love +2345.com +256bit.by +3dracergames.com +4kepics.com +4webmasters.org +55wmz.ru +91abcw.com +aa14ab57a3339c4064bd9ae6fad7495b5f.com +adamoads.com +adclickservice.com +adconscious.com +advancedsoftwaresupport.com +advertex.info +ahmedabadwebs.com +alert-hgd.xyz algarveglobal.com -algerianembassy.co.in -alice.it -all4invest.info -all4invest.ru -allcredits.su -aloofly.com -amateurmatch.com -amung.us -android-systems.ru +allwidewallpapers.com +alot.com +alpharma.net +alveris.ru +amanda-porn.ga +amateurlivechat.org +android4fun.org +android-style.com angigreene.com -animali.deagostinipassion.it -anonymizeme.pro +anniemation.com +anonymous-redirect.com +aosheng-tech.com apibrowsesmartne-a.akamaihd.net -apiportalmorecom-a.akamaihd.net -apiqualitinknet-a.akamaihd.net -apiwebwebgetcom-a.akamaihd.net -apiwoodensealcom-a.akamaihd.net -appsecurityr.com -arcadeplayhouse.com +apiholdingmypage-a.akamaihd.net +apishops.ru +app5.letmacworkfaster.world +aquarium-pro.ru +arius.tech +arpelsreplica.xyz +asmxsatadriverin.aircus.com asociatia-tipografilor-transilvania.ro -atelielembrancaqueencanta.com.br -audiofree.ru -autobrennero.it -avkzarabotok.com -avtoarenda.by -avtointeres.ru -azartclub.org -azlex.uz -babosas.com -babyguns.ru -backlink4u.net -bank.uz -barbourjackets.online -battleforupdating.site -bdsmgalls.net +audiobangout.com +avramstroy.ru +azartmix.com +azartniy-bonus.com +backuperwebcam.weebly.com beauty-things.com -bestofferswalkmydogouteveryday.gq -bestofpicture.com -biketank.ga -billiard-classic.com.ua -binomo.com -binomo.kz -bio-japan.net -biteg.xyz -blacktwhite.com -bloglag.com -bolitgorlo.net -bracketsmackdown.com -brainboostingsupplements.org -braintobe.win -brainzpod.win -brandbucket.com -brg8.com -brokergid.ru -buigas.00it.com -buqyxa.rincian.info -buy-cheap-pills-order-online.com -cardiosport.com.ua +best-businessman.ru +best-mam.ru +bestmarriages.com +betonka.pro +bezcmexa.ru +big-cash.net +binaryoptionscops.info +bioinnovate.co +bizfly.info +blavia.00author.com +blogsfunky672.weebly.com +bobba.dzaba.com +boeuklad.com +bongacams.com +bonvillan.com +bookmaker-bet.com +brendbutik.ru +brimstonehillfortress.org +buckspar.gq +buttons-for-website.com +buyk.host.sk +bwlx.prepedu.cn +carabela.com.do cardul.ru -ccjp.eu -centraletermice.us -cheapjerseysap.com -cheapmoncler.win -checkhit.com -chocolateslim-en-espana.com -chorus.terakeet.com -cialischmrx.com -cialisndbrx.com -cibpenonptib.flu.cc -cleanpcnow.com -clicksor.net -clknsee.com -coindirect.io -coinsspb.com -colobit.biz -compliance-don.xyz -compliance-willy.top -copenergo.ru -cosmetic.donna7753191.ru -covetnica.com -csgo4.win +cartierbraceletsreplica.pw +cartierreplica.win +cashonads.com +cbox.ws +cctva.tv +cdncash.net +celebsopics.com +cercacamion.it +cfjump.com +cheapbelstaff.online +cheapestjerseys-wholesale.com +cheapmoncler.xyz +cheap-trusted-backlinks.com +cheatcode-lita12.rhcloud.com +chelnytruck.ru +chocolateslim-original.com +cialischstgerts.com +cigarpass.com +ciproandtizanidine.blogspot.com +citetick.com +clarithromycin500mg.com +claytransformations.info +clickbank.net +clickbanksites.info +clicksotrk.com +coaching-netz.info +compliance-george.xyz +coolingoods.com +copypaste.traderzplanet.in +counterbot.com +cowblog.fr +cowlmash.com +cpajunkies.com +crd.clan.su culmimg.pw -d2jsp.org -daooda.com -data1.scopich.com -data.vtc.pw -ddospanel.com -demon-tweeks.com -dentalpearls.com.au +custodieva.ru +custom-electric-guitar.com +dandiyabeats.in +ddsoldes.fr +decoratinghomes.ga destinationrealestate.com -dignitasdata.se -dimfour.com -dktr.ru -doctormakarova.ru -dogsrun.net -dominterior.org -downloader12.ru -downloads-whatsapp.com -drev.biz -dynainbox.com -dyshagi.ru -earn-from-articles.com -e-biznes.info -ecookna.com.ua -edge.sharethis.com -ekspertmed.com -e-kwiaciarz.pl -elektrischezigarette2.devhub.com -eloxal.ru +dev33.dioniqlabb.se +dev-seo.blog +diarioaconcagua.com +dikx.gdn +divku.ru +docs4all.com +dogbreedspicture.net +domainroam.win +domik-derevne.ru +dosugrostov.site +driving.kiev.ua +droidlook.net +drunkenstepfather.com +e-buyeasy.com +e-commerce-seo.com +eleimgo.pw +elektrischeziga.livejournal.com +elektrischezigarette1.blog.pl +elvel.com.ua emediate.eu -engines-usa.com -epicbrogaming.com -erotische-geschichten-xxl.com -escort-russian.com -essaypro.com -etotupo.ru -everytalk.tv -excaliburfilms.com -exchanges-bet.com -excitacao.com -expertnaya-ocenka.ru -extlabs.io -extremepornos.net -eyelike.com.ua -ezigarettekaufen1.hpage.com -ezigarettekaufen.myblog.de -f1nder.org -familienzahnaerzte.com -farmprofi.net -feeriaclub.ru -fefo.gdn -fickkontakte.org -filmi-v.online -financeloan.us -financepoints.eu -firstdrugmall.ru -focalink.com -forum-engineering.ru -fotosfotos.eu -freakycheats.com -freecamdollars.com -free-share-buttons.com -freewareseek.com -freewebs.com -freewlan.info -freshmac.space -freza-sverlo.ru -fr.netlog.com -fsakhalin.ru -fun2cell.net -gallery.rennlist.com -gdebestkupit.ru -generalporn.org -gg.zzyjxs.com -gktt.ru -glicol.kz -globalsurfari.com -gobongo.info -goldpanningtools.com -golmau.host.sk -googlemare.com -gorodservis.ru -gov.yanao.ru -grupografico-pilar.com.ar +en.altezza.travel +enginebay.ru +eng-lyrics.com +erotikchat-24.com +eshop4u.jp +espaceinventoristes.com +essayassist.com +estelight.ru +exoclick.com +expolicenciaslatam.co +eyes-on-you.ga +face.hostingx.eu +falllow.gq +familyphysician.ru +fashionindeed.ml +fealq.com +fickenprivat.info +findacheaplawyers.com +findthe.pet +fishingwholesale.us +fitfloponline.store +flashlarevista.com +floating-share-buttons.com +flowwwers.com +forex-procto.ru +formulaantiuban.com +foto-sisek.porngalleries.top +free-fbook-traffic.com +freesitetest.com +free-social-buttons.com +freeuploader.com +freshberry.com.ua +funtoonez.com +fz139.ttk.ru +galleryawesome.com +game300.ru +gamedayassist.com +generic-pills-online.com +global-ics.co.za +globetrotting-culture.ru +go2album.com +golaya.pw +goodnightjournal.com +goosefishpost.bid +gpirate.com +gratuitbaise.com +greendream.com.ua +greenidesign.co +greenzaim.ru +gsasearchenginerankerdiscount.com gsasearchengineranker.site -gsmtlf.ru -gstatey.net -gwebtools.com -habermetre.com -hausfrauensex18.com -hdapp1008-a.akamaihd.net -healgastro.com -heatpower.ru -helvetia.com.ua -hermesbracelets.xyz -himgaws.pw -hkdiiohi.skyrock.com -hoholikik.club -houseofgaga.ru -hoverboard360.de -humanorightswatch.org -hvd-store.com -ideoworld.org -iflycapetown.co.za -ilovevitaly.xyz +gta-club.ru +gtopstats.com +gz2.bbsoldes.fr +harmonyglen.us +hauleddes.com +havepussy.com +hccoder.info +hd-film.pl +heliko.no +highstairs-a.akamaihd.net +hobbyhuren24.net +holidaypics.org +house.sieraddns.com +ibb.com.ua +icaseclub.ru +ie.57883.net images.gyffu.com -imagui.eu -innoslicon.com -inome.com.ua -inspiring-desperate.tk -interior-stickers.ru -intermesh.net -intim-uslugi.info -ipchicken.com -iwanttodeliver.com -iweblist.info -jamiembrown.com -jmat.cn -job.icivil.ir -jobmarket.com.ua -jolic2.com -jovencitas.gratis -justucalling32211123456789.tk -jwss.cc +img.wallpaperstock.net +incanto.in.ua +infazavr.ru +infogame.name +infoupdate.org +insta-add.pro +irkutsk.zrus.org +islandminingsupply.wordpress.com +jagg.info +javcoast.com +javrip.net +jerseychinabizwholesale.com +jerseywholesalebizchina.com +justbcause.com +kam-dom.ru +karadene.com +kartiniresto.com katadhin.co -katjimej.blog.fc2.com +kazrent.com keki.info -keywords-monitoring-your-success.com -kinopolet.net -konoplisemena.com -krasula.pp.ua -ks1234.com -kupit-adenu.ru -kvartira-sutochno.com -larchik.net +kfon.eu +kiwe-analytics.com +konica.kz +kostenloser-sex.com +kountrylife.com +krasnodar.xrus.org +kreativperlen.ch +kvartir-remont.biz +landoftracking.com +larutti.ru law-check-eight.xyz -lazymanyoga.com ledpolice.ru -link.ac -linkbux.ru -liquimondo.com -livingcanarias.com -localmatchbook.com -lotto6888.com -love-baby.cz -lsp-awak-perikanan.com -m3gadownload.pl -m5home.ru -maglid.ru -meble-bogart.info -mecash.ru -medanestesia.ru -medi-fitt.hu +lernur.net +letslowbefast.today +life.biz.ua +lincolntheatre.com +lirunet.ru +littleberry.ru +littlesexdolls.com +liumimgx.pw +liupis.com +livetsomudvekslingstudent.bloggersdelight.dk +liveu.infoteka.hu +lostfilm-online.ru +lovi-moment.com.ua +lumb.co +lyrics.home-task.com +magazintiande.ru +mainlinehobby.net +maju.bluesalt.co +mansparskats.com +marinetraffic.com +maxthon.com +mcar.in.ua +medicovi.com +medizinreisen.de medosmotr-ufa.ru -metaxalonevstizanidine.blogspot.com -militarysale.pro -miniature.io -mmgq.ru -mnimmigrantrights.net -mobifunapp.weebly.com -mobile-appster.ru -mokrayakiska.com -moneymaster.ru -moomi-daeri.com -morefastermac.trade -mountainstream.ms -musezone.su -mybinaryoptionsrobot.com -myonigroup.com -myonlinepayday.co -myperiod.club -myyour.eu -naj-filmy24.pl -nalogovyy-kodeks.ru -nationalbreakdown.com -naturalshair.site -naturtreenspicerx.pw -next-dentists.tk -nicovideo.jp -nightvision746.weebly.com -nomuos.it -noscrapleftbehind.co -notfastfood.ru -november-lax.com -nudejapan.net -nuker.com -offers.bycontext.com -offtime.ru -okonich.com.ua -olvanto.ru -omsk.xrus.org -onefilms.net -onlyporno.ru -optibuymac.com -osb.se11.ru -ownshop.win -oxotl.com -painting-planet.com -pandarastore.top -paparazzistudios.com.au -pastaleads.com -paydayonlinecom.com -peekyou.com -php-market.ru -picturesmania.com -pijoto.net -pinwallpaper.xyz -piski.top -pivka.xyz -pizdopletka.club -plastjulgranar.se -platezhka.net -polybuild.ru -popugaychiki.com -pornodojd.ru -pornoreino.com -pornosee.info -pornpost.in -portnoff.od.ua -praisong.net -preparevideosafesystem4unow.space -pricheski-video.com -priscilarodrigues.com.br -profit-opportunity.com +medtherapy.ru +mega-bony2017.pl +megahdporno.net +miniads.ca +minneapoliscopiers.com +mirzonru.net +mitrasound.ru +mobplayer.net +monclercheap.xyz +moncleroutletonline.pw +mp3ringtone.info +mrvideospornogratis.xxx +musflashtv.com +music7s.me +my-bc.ru +mydirtyhobby.com +mylida.org +netanalytics.xyz +networkad.net +newfilmsonline.ru +nodding-passion.tk +nowtorrents.com +nufaq.com +nvssf.com +oballergiya.ru +o-dachnik.ru +oecnhs.info +office2web.com +officedocuments.net +okout.ru +olympescort.com +onclickpredictiv.com +one-gear.com +ontargetseo.us +oren-cats.ru +outrageousdeal-a.akamaihd.net +pacificair.com +pattersonsweb.com +perl.dp.ua +phobia.us +picsfair.com +pk-pomosch.ru +plaff-go.ru +playlott.com +playtap.us +poisk-zakona.ru +poiskzakona.ru +pokemon-go-play.online +popup.matchmaker.com +pornhub-forum.ga +pornofeuer.com +pornoforadult.com +pornoklad.net +primedice.com +priora-2.com +prizestohandle.club +proekt-mos.ru projectforte.ru -prokotov.com -promoheads.com -promover.org -prostitutki-odessa.org -prostitutki-tyumeni.org -proxyelite.biz -quangcaons.com -questionmarque.ch -quickchange.cc -ranksonic.com -rapidsites.pro -regionshop.biz -registrydomainservices.com -reimageplus.com -remont-mobile-phones.ru -renewablewealth.com -repeatlogo.co.uk -rfserial.net -rosbalt.com.ua -rospromtest.ru -rubanners.com -ruclicks.com -rutor.vip -sa-rewards.co.za -satellite.maps.ilovevitaly.com -saveriopiazza.it -sbornik-zakonov.ru -scmor.ilxc.cc -scoopquest.com -scripted.com -searchaddis.com -search-goo.com -searchlock.com -seocheckupx.com -seotoolsagency.com -seo-tools-optimizing.com -servingnotice.com -sexflirtbook.com +purchasepillsnorx.com +putitin.me +qpypcx.com +qwesa.ru +rangoman.date +rapevideosmovies.com +razleton.com +realtytimes.com +recinziireale.com +redhotfreebies.co.uk +redirectingat.com +redirectme.net +reeyanaturopathy.com +registratciya-v-moskve.ru +registrationdomainsite.com +reining.lovasszovetseg.hu +responsinator.com +reversing.cc +rfid-locker.co +ritlweb.com +rubbed.us +ruspdd.com +rus-pornuha.com +russia-tao.ru +rutor.group +sakhboard.ru +samolet.fr +saneyes.com +sape.top +saugatuck.com +scanner-walt.top +seadragonherbery.com +seminarykansas904.ml +seokicks.de +servisural.ru +sexphoto.site sexreliz.com -sex-watch.com -shikiso.info -ship-marvel.co.ua -shisha-swag.de +sexy.babes.frontend-stack.top +shell-pmr.ru shitmovs.com -shoesonlinebuy.xyz -shopfishing.com.ua -shopperifymac.com -sideeffectsoftizanidine.blogspot.com -simple-image.com.ua -siteaero.com -site-auditor.online -sitebeam.net +shtora66.ru +silktide.com +sitevalued.com +sk.golden-praga.ru skincrate.net -skuteczna-dieta.co.pl -sledstvie-veli.net -s.lollypopgaming.com +slowmacfaster.trade smartadserver.com +smart-scripts.com smichovbike.cz +snsdeainavi.info +soaksoak.ru +soblaznu.net +socas.pluto.ro +socialbookmarksubmission.org socialbuttons.xyz -social-s-ggg.xyz -sodexo.com -solinf.co -sonata-arctica.wz.cz +socialmadesimple.com +softxaker.ru +sogimlecal.tk +sonnikforme.ru +souvenirua.com spacash.com -spaceshipad.com -spb-plitka.ru -specialfinanceoffers.com -spyfu.com -stathat.com -statoutlook.info -stop-zavisimost.com -stpolice.com -strana-solnca.ru +special-porn.com +speedup-my.site +sprttrack.com +sptslmtrafms.com +spywarebegone.com +spywareit.com +sstroy44.ru +stairliftstrue.com +stal-rulon.ru +starwars.wikia.com +steamoff.net +stickers-market.ru +stpicks.com +strongholdsb.ru suhanpacktech.com -summerlinhomes411.info -suralink.com -surffoundation.nl -sweepstakes.rewardit.com -takeprofitsystem.com -tarad.com -t-bygg.com -tehngr.ru -testingads.pro -tgtclick.com -themeforest.net -the-torrent-tracker.blogspot.com -tiandeural.ru -tizanidinedosageforsleep.blogspot.com -tizanidinehclsideeffects.blogspot.com -tizanidinenarcotic.blogspot.com -tizanidineorflexeril.blogspot.com -toonfamilies.net -torrent-to-magnet.com -traffic2cash.org -trk-4.net -trustedmaccleaner.com -tula.howotorg.ru -turist-strani.ru -turizmus.us -turvgori.ru -tvory.predmety.in.ua -twelvevisionspartyofcolorado.com -uamtrk.com -uggsale.online +superinterstitial.com +superlist.biz +superstarfloraluk.com +survival.betteroffers.review +susanholtphotography.com +svensk-poesi.com +symphonyintegratedhealthcare.com +tabletkinaodchudzanie.com.pl +taximytishi.ru +terraclicks.com +tfxiq.com +theallgirlarcade.com +thedownloadfreeonlinegames.blogspot.com +tizanidine4mguses.blogspot.com +tizanidineformigraines.blogspot.com +tizanidinehcl2mgsideeffects.blogspot.com +tnaionline.org +tnctrx.com +tooplay.com +topdownloads.ru +topsiteminecraft.com +toyota.7zap.com +traffic-club.info +trafficstars.com +traffictrade.life +trion.od.ua +tt-ipd.info +ttrraacckkrr.com +tuckpointingmasonrysystems.com +tvand.ru +typimga.pw +u17795.netangels.ru +ultimateclassicrock.com +ultimatesetnewfreeallsoftupgradesystems.pw +umnovocaminho.com +uni.me unionmarkt.de -unisexjewelry.org -uogonline.com +univerfiles.com +unrealcommander.org upstore.me uptimebot.net -uptime.com -vacuumscleaner.com -valuado.com -vandrie-ict.nl -vapeface.club +ural-buldozer.ru +urengoy.pro +usdx.us +vadimkravtcov.ru varikozdok.ru -velpanex.ru -videositename.com -viktoria-center.ru -vip2ch.com -vipsexfinders.com -viteonlusarezzo.it -vmnmvzsmn.over-blog.com -voloslove.ru -vote-up.ru -vseigru.one -wallpaperaccess.com -watchmyfb.pl -webnode.me -webshoppermac.com -website-audit.com.ua +vbikse.com +vekzdorov.ru +venerologiya.com +viagraneggrx.com +via-gra.webstarts.com +viandpet.com +videochat.bz +videochat.life +video-production.com +voditeltrezviy.ru +voluumtrk.com +vzube.com +w7s.ru +wallpapers-best.com +wallpaperstock.net +watchmygf.net +webalan.ru +webix.me +webscouter.net +website-analyzer.info +websiteexplorer.info website-speed-check.site +websites-reviews.com website-stealer.nufaq.com -weburlopener.com -weprik.ru -westvilletowingservices.co.za -whatistizanidine2mg.blogspot.com -wieseversa.no -winx-play.ru -writingservices17.blogspot.ru -www.jbetting.com -www.kabbalah-red-bracelets.com -xcc24.pl -xev.ru -xn----8sbdbjgb1ap7a9c4czbh.xn--p1acf -xn----9sbubg3ambdfl1j.xn--p1ai -xn--d1aifoe0a9a.top -xn--e1afanlbnfckd7c3d.xn--p1ai -x-porno.video -yellocloud.be -yellowproxy.net -yellowstonesafaritours.com -yougotanewdomain.com -youporn-ru.com -yourporn.com -youtubologia.it +while.cheapwebsitehoster.com +wholesalejerseys-cheapest.com +windowssearch-exp.com +winterclassichockeyjerseys.com +womama.ru +woodyguthrie.se +world-mmo.com +woweb.com.ua +wtsindia.in +www.souvenirua.com +xchat26.myfreecams.com +xclicks.net +xn--80ajbshivpvn2i.xn--p1ai +xn-----8kcatubaocd1bneepefojs1h2e.xn--p1ai +xn----9sbebi2bvzr7h.xn--p1ai +xn--b1adccaf1bzj.xn--p1ai +xn--b1ag5cfn.xn--p1ai +xxart.ru +yaderenergy.ru +youandcredit.ru +youbrainboost.asia +youdesigner.kz +youjizz.com +youradexchange.com +youradulthosting.com +yourserverisdown.com +youthreaders.com +youtuhe.com +yugo-star.ru +zaapplesales.blogspot.com zagadki.in.ua -zaidia.xhost.ro -zajm-zalog-krasnodar.ru +zaim-pod-zalog-krasnodar.ru +zaobao.com.sg +zastenchivosti.net zebradudka.com -zero1.it -zhuravlev.info -zigarettenonlinekaufen2.drupalgardens.com -zigzog.ru -zone-kev717.info -zoogdiesney.com -zoominfo.com +zerocash.msk.ru +zeroredirect7.com +zoogdinsney.com +zytpirwai.net diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index dffcba71c..39d6a5c31 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -03p.info -1001watch.com.ua -123locker.com -2kata.ru -3th.co.in -98oi.ru -a96527gi.beget.tech -aa8780bb28a1de4eb5bff33c28a218a930.com -acads.net -adanih.com -adsfresh.men +21h2o.com +2728fb936f0.com +51unlim.ru +70casino.online +aanapa.ru +abcguru.xyz +absugars.com +abwa.tk +addtoadd.men +adssafeprotected.com +adzpower.com agroeconom.kz -alert-fdm.xyz -alf-img.com -ameblo.top -androids-store.com -apccargo.com -apimegabrowsebiz-a.akamaihd.net -apisecretsaucebi-a.akamaihd.net -apple.com-cleaner.systems -appmsr.org -arabseyes.com +alienwheels.de +all4invest.ru +allforminecraft.ru +apiqualitinknet-a.akamaihd.net +arendatora.ru aruplighting.com -asdfz.pro -asiavirtualsolutions.com -asmxsatadriverin.aircus.com -atlant-auto.info -aviav.ru -azbukafree.com -balkanfarma.org -bayareaaeroclub.org -bildsuche.ru -bjanshee.ru -bkns.vn -bluesman.nu -boostmyppc.com -breastaugmentation.co.za -brothers-smaller.ru -buigas.00it.com -builtwith.com -buron.pw -buynorxx.com -cakesplus.com.au -calcularpagerank.com -carsdined.org -cartierbracelet.xyz -cartierlove2u.xyz -cartierreplica.top -ccbill.com -chcu.net -chelyabinsk.xrus.org -cindymatches.com -civilwartheater.com -clayvasedesigns.tk -clean-virus-mac.com -communisave.co.za -community.allhiphop.com -com-onlinesupport.site -compliance-don.xyz -compliance-elena.top -coslab.club -culmimg.pw -de.zapmeta.com -dfwu1013.info -dokfilms.net -doktoronline.no -domaineaneblanc.com -doska-vsem.ru -ee77ee.com -essaypro.com -eu-cookie-law.blogspot.com -event-tracking.com -ewebarticle.info -eyessurgery.ru -fable.in.ua -fapgon.com -feargames.ru +asseenontv.ru +autovideobroadcast.com +bestcurs.org +bestfortraders.com +besthomemadepornsites.com +best-seo-offer.com +betune.onlinewebshop.net +bikini-image.com +bloke.com +braintobe.win +brainzpod.win +buybest1.biz +candlehandmade.com +cartierbraceletsreplica.pw +celebsopics.com +cheapbarbour.online +cheapjerseysa.com +chocolateslim-original.com +compliance-julia.top +contextualyield.com +cougfan.info +cymbaltaandtizanidine.blogspot.com +d2jsp.org +dailyrank.net +danashop.ru +dataloading.net +deutschland123.de +digilander.libero.it +dividendo.ru +djekxa.ru +dnepropetrovsk.mistr-x.org +domainsatcost.com +dominterior.org +downloadkakaotalk.com +druzhbany.ru +easync.io +ekto.ee +encodable.com +evehemming.blogspot.com.au +everytalk.tv +excitacao.com +f5mtrack.com +fbdownloader.com femdom.twiclub.in -filmgo.ru -floridamhca.org -forumprofi.de -foto-telok.net -foxweber.com -frcls.fr -free-gluten.ru -free-social-buttons.com +filesclub.net +firesub.pl +firma-legion.ru +fishingwholesale.us +fix-website-errors.com +flauntyoursite.com +flex4launch.ru +focalink.com +foodcrafts.website +forexgb.ru +freejabs.com +free-stock-illustration.com +free-today.com frvo.alptandem.ru -fusoradio.info -gamblingnerd.com -gameplexcity.com -gayxperience.com -gidonline.one -gilbertbanda.net -girlporn.ru -globetrotting-culture.ru -goldandcard.ru -grtyi.com -gtopstats.com -gwebtools.com -hasshe.com -hdpixion.com -helicalpile.us -heygidday.biz -hikesearch.net -housediz.com +g7m.pl +gambarkataku.co +getoutofdebtfree.org +goblacked.com +gsasearchenginerankerhelp.com +guigyverpo.cf +hdwallpapers-free.com +headpress.ru +heliko.no houseofgaga.ru -ibb.com.ua -idegenvezeto.eu -iklysha.ml +hundejo.com iloveitaly.ru -imagecoolpub.com -imediadesk.com -imgata.com +images-graphics-pics.com intervsem.ru -izhevsk.zrus.org -jackwolfskinoutlet.online -kartiniresto.com -klosetkitten.com -knowsitall.info +ipostroika.ru +istizanidineanarcoticdrug.blogspot.com +itag.pw +kazan.zrus.org +keywordglobal.co.za +kinoplen.ru +kinopolet.net +kometars.xyz +koral.se kovrenok.ru kraljeva-sutjeska.com -lampokrat.ws -landinez.co -languagecode.com -larose.jb2c.me -lineshops.biz -lion.bolegapakistan.com -liumimgx.pw -lollypopgaming.com -maglid.ru -mainlinehobby.net -managerpak204.weebly.com -marketland.ml -mediaoffers.click -meds-online24.com -member-quiz.com -menhealed.net -metarip.ru -metascephe.com -middlerush-a.akamaihd.net -mindeyegames.com -miniads.ca -mirzonru.net -modabutik.ru -moskva.nodup.ru -mug-na-chas-moscow.ru -mysex21.com -narosty.com -naturalshair.site -norththeface.store -obiavo.net -o-dachnik.ru -one-gear.com -online-podarki.com +lalalove.ru +laserpen.club +letmacworkfaster.site +longlifelomilomi.info +luxup.ru +mabdoola.blogspot.com +mackeeper-center.club +makedo.ru +manifestation.betteroffers.review +manimpotence.com +manipulyator-peterburg.ru +master-muznachas.ru +matematikus.info +max-eclat.men +maximilitary.ru +mediafresh.online +medi-fitt.hu +menstennisforums.com +misterjtbarbers.com +mister-shop.com +mjchamonix.org +moivestiy.biz +moncleronline.xyz +moneyzzz.ru +monsterdivx.tv +muz-baza.net +muznachas-service.ru +mycouponizemac.com +mydirtyhobby.com +my-floor.in.ua +najaden.no +na-telefon.biz +new7ob.com +oneclickfiles.com +onlinefilmz.net o-o-8-o-o.com -orhonit.com -orsonet.ru -page2rss.com -paidonlinesites.com -panchro.xyz -paphoselectricianandplumber.com -partnerline.men -pavlodar.xkaz.org -pestomou.info -peugeot-club.org -picmoonco.pw -picurams.pw -porngalleries.top -pornpost.in -pricheskaonline.ru -primedice.com -prodvigator.ua -profolan.pl +optikremont.ru +orel-reshka.net +panchro.co.uk +pandroid.co +partner-high.men +patol01.pw +p-business.ru +pdns.download +perosan.com +picsearch.com +picswe.com +pixshark.com +pl.aasoldes.fr +plusnetwork.com +pokemon-go-play.online +popmarker.com +popup.matchmaker.com +portal-eu.ru +presleycollectibles.com +progonrumarket.ru +prohoster.info promoforum.ru -prostitutki-kiev.org -pro-tec.kz -pulse33.ru -puserving.com -remontbiz.ru -replica-watch.ru -reversing.cc -roofers.org.uk -sanatorrii.ru -scanner-irvin.top -school-diplomat.ru -schoolfiles.net -semalt.com -service.adtech.us -sexad.net -sexphoto.site -sex-sex-sex5.com -sharebutton.to -silverage.ru -skatestick.bid -slujbauborki.ru -sms2x2.ru -sodexo.com -sortthemesitesby.com -sotkal.lark.ru -steebook.com -stjamesschool.info -strana-krasoty.ru -strana-solnca.ru -summerlinhomes411.info -supermama.top -svolze.com -tamada69.com -titan-ads.life -tivolibasket.it -tizanidine4mg.blogspot.com -tobeyouday.win -torontoplumbinggroup.com -trafficjunky.net -tula.howotorg.ru -tyumen.xrus.org -uchil.net -ukkelberg.no -um-razum.ru -universals.com.ua -unmaroll.ya.ru -userequip.com -vchulkax.com -vekzdorov.ru -vepad.com -vipsiterip.org -vkgaleria.com -vladimir.xrus.org -voprosotvet24.ru -w3data.co -webcamdevochka.com -weekes.biz.tc -wheelchairliftsarea.com -wholesalejerseysgaa.com -wonderfulflowers.biz -wovis.site -www.jbetting.com -xaylapdiendanang.com -xn--80ahdheogk5l.xn--p1ai -xn--80ajbshivpvn2i.xn--p1ai -xn--e1afanlbnfckd7c3d.xn--p1ai -xn--e1aggki3c.xn--80adxhks -x-stars.ru -xtraffic.plus -yginekologa.com -you-shall-not-pass.is74.ru +prostitutki-yaroslavlya.org +pshare.biz +pussysaga.com +quality-traffic.com +quelle.ru +quickloanbank.com +rainbowice.ru +ranksonic.info +redirectingat.com +regionshop.biz +renecaovilla.online +replicaclub.ru +roma-kukareku.livejournal.com +ruscopybook.com +sandhillsonline.com +saveindex.xyz +scanner-olga.top +searchengineranker.email +searchmywindow-a.akamaihd.net +sei80.com +seoboxes.com +seofirmreviewsus.info +seo-tools-optimizing.com +service-core.ru +sfj-ror.no +shell-pmr.ru +smichovbike.cz +snow.nvr163.com +soft1.ru +softlinesolutions.me +solitaire-game.ru +special-porn.com +sport-video-obzor.ru +steame.ru +stpolice.com +streha-metalko.si +stroymonolit.su +suchenindeutschland.com +supermodni.com.ua +susanholtphotography.com +taihouse.ru +taqywu51.soup.io +taranerymagesswa.blogspot.com +t-bygg.com +telegraf.by +themestotal.com +thexart.club +tizanidineforopiatewithdrawal.blogspot.com +tizanidinenarcotic.blogspot.com +topquality.cf +trafficmania.com +traffmonster.info +trainoffend.ml +trubywriting.com +trustedhealthtips.com +turist-strani.ru +turvgori.ru +uamtrk.com +ua.tc +ublaze.ru +uggsale.online +vavilone.com +via-energy-cumpara.com +vip-parfumeria.ru +virtualbb.com +vkak.ru +vodkoved.ru +voluumtrk.com +wanker.us +watch-movies.ru +webextract.profound.net +webix.me +while.cheapwebsitehoster.com +wpsecurity.website +wufak.com +wygraj-skiny.win +xgftnlrt.bloger.index.hr +xmronta.com +x-musics.com +xn--d1aifoe0a9a.top +youbrainboost.asia +youporn-ru.com +yourothersite.com +zagadki.in.ua zakazvzloma.com -zero1.it -znakom.sibtest.ru -zoogdinsney.com -zx6.ru +zaobao.com.sg +zeroredirect7.com +zigarettenonlinekaufen1.bloog.pl +znakomstva-moskva77.ru +zootoplist.com +zrizvtrnpale.tk diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index f0474a154..b2122ba99 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -1,21 +1,11 @@ - - - -Welcome to NginX - - -
-

Welcome to the NginX Ultimate Bad Bot Blocker

-

This is merely used for testing !!!

- - + 503 Service Temporarily Unavailable

503 Service Temporarily Unavailable

nginx/1.18.0 (Ubuntu)
- +ml> 503 Service Temporarily Unavailable

503 Service Temporarily Unavailable

@@ -581,4 +571,14 @@

Welcome to the NginX Ultimate Bad Bot Blocker

This is merely used for testing !!!

+ + + +Welcome to NginX + + +
+

Welcome to the NginX Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ \ No newline at end of file diff --git a/.dev-tools/test_units/whitelist-ips.conf b/.dev-tools/test_units/whitelist-ips.conf index bb50a4a64..dd5291a4a 100644 --- a/.dev-tools/test_units/whitelist-ips.conf +++ b/.dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 0; +104.154.182.187 0; 127.0.0.1 0; diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index 33db97ee2..35000d0af 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index 8d33d32d9..4c698205c 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 6e039fa6f..bc86b8aa6 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.01.2238 +#### Version: V4.2021.01.2239 #### Bad Referrers Blocked: [6906](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [587](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [591](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index e4e597f55..69cb81b97 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.01.2238 +#### Version: V4.2021.01.2239 #### Bad Referrers Blocked: [6906](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [587](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [591](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index 538918e4a..ed20c5da8 100755 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.01.2238 +#### Version: V4.2021.01.2239 #### Bad Referrers Blocked: [6906](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [587](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [591](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 30a552a0a..fff6840ec 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.01.2238 +#### Version: V4.2021.01.2239 #### Bad Referrers Blocked: [6906](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [587](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [591](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 04d512a2a..8623b4e60 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2238 -### Updated: Mon Jan 4 11:10:46 SAST 2021 +### Version: V4.2021.01.2239 +### Updated: Mon Jan 4 12:00:44 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 32430a294..7b6055cbc 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.01.2238 -### Updated: Mon Jan 4 11:10:47 SAST 2021 +### Version: V3.2021.01.2239 +### Updated: Mon Jan 4 12:00:45 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 04d512a2a..8623b4e60 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.01.2238 -### Updated: Mon Jan 4 11:10:46 SAST 2021 +### Version: V4.2021.01.2239 +### Updated: Mon Jan 4 12:00:44 SAST 2021 ### Bad Referrer Count: 6906 -### Bad Bot Count: 587 +### Bad Bot Count: 591 ################################################### ### VERSION INFORMATION ## @@ -229,6 +229,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cocolyzebot(?:\b)" 3; "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; @@ -393,6 +394,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LieBaoFast(?:\b)" 3; "~*(?:\b)Lightspeedsystems(?:\b)" 3; "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkbot(?:\b)" 3; "~*(?:\b)Linkdexbot(?:\b)" 3; "~*(?:\b)LinkextractorPro(?:\b)" 3; "~*(?:\b)LinkpadBot(?:\b)" 3; @@ -654,6 +656,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Vagabondo(?:\b)" 3; "~*(?:\b)VB\ Project(?:\b)" 3; "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VelenPublicWebCrawler(?:\b)" 3; "~*(?:\b)VeriCiteCrawler(?:\b)" 3; "~*(?:\b)VidibleScraper(?:\b)" 3; "~*(?:\b)Virusdie(?:\b)" 3; @@ -731,6 +734,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)zgrab(?:\b)" 3; "~*(?:\b)Zitebot(?:\b)" 3; "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZoomBot(?:\b)" 3; "~*(?:\b)ZoominfoBot(?:\b)" 3; "~*(?:\b)ZumBot(?:\b)" 3; "~*(?:\b)ZyBorg(?:\b)" 3; diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index e8b0b58f5..de695a5b1 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2021.01.2238 -### Updated: Mon Jan 4 11:10:49 SAST 2021 -### Bad Bot Count: 587 +### Version: V4.2021.01.2239 +### Updated: Mon Jan 4 12:00:46 SAST 2021 +### Bad Bot Count: 591 ################################################### ### Version Information ## @@ -179,6 +179,8 @@ User-agent: Cloud mapping Disallow:/ User-agent: coccocbot-web Disallow:/ +User-agent: Cocolyzebot +Disallow:/ User-agent: CODE87 Disallow:/ User-agent: Cogentbot @@ -507,6 +509,8 @@ User-agent: Lightspeedsystems Disallow:/ User-agent: Likse Disallow:/ +User-agent: Linkbot +Disallow:/ User-agent: Linkdexbot Disallow:/ User-agent: LinkextractorPro @@ -1029,6 +1033,8 @@ User-agent: VB Project Disallow:/ User-agent: VCI Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ User-agent: VeriCiteCrawler Disallow:/ User-agent: VidibleScraper @@ -1183,6 +1189,8 @@ User-agent: Zitebot Disallow:/ User-agent: ZmEu Disallow:/ +User-agent: ZoomBot +Disallow:/ User-agent: ZoominfoBot Disallow:/ User-agent: ZumBot