diff --git a/.dev-tools/_robots_input/robots-input.txt b/.dev-tools/_robots_input/robots-input.txt index 6c007e501..096c6e075 100644 --- a/.dev-tools/_robots_input/robots-input.txt +++ b/.dev-tools/_robots_input/robots-input.txt @@ -429,6 +429,7 @@ SearchmetricsBot Semrush SemrushBot SentiBot +SeobilityBot SEOkicks SEOkicks-Robot SEOlyticsCrawler diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 97eb2dda9..9bc4b3fd4 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.05.2361 -### Updated: Thu May 27 07:27:03 SAST 2021 +### Version: V3.2021.05.2362 +### Updated: Thu May 27 08:20:22 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 0ff8f6817..3bd4c7e2e 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.91.101 1; +35.193.184.18 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 29c63bbe3..cc2baf875 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.91.101 0; +35.193.184.18 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 97eb2dda9..9bc4b3fd4 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.05.2361 -### Updated: Thu May 27 07:27:03 SAST 2021 +### Version: V3.2021.05.2362 +### Updated: Thu May 27 08:20:22 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 0ff8f6817..3bd4c7e2e 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.91.101 1; +35.193.184.18 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 29c63bbe3..cc2baf875 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.91.101 0; +35.193.184.18 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 97eb2dda9..9bc4b3fd4 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.05.2361 -### Updated: Thu May 27 07:27:03 SAST 2021 +### Version: V3.2021.05.2362 +### Updated: Thu May 27 08:20:22 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 0ff8f6817..3bd4c7e2e 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.91.101 1; +35.193.184.18 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 3557e6230..9df798c2c 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -490,6 +490,7 @@ "~*(?:\b)Semrush(?:\b|)" 0; "~*(?:\b)SemrushBot(?:\b|)" 0; "~*(?:\b)SentiBot(?:\b|)" 0; + "~*(?:\b)SeobilityBot(?:\b|)" 0; "~*(?:\b)SEOkicks(?:\b|)" 0; "~*(?:\b)SEOkicks-Robot(?:\b|)" 0; "~*(?:\b)SEOlyticsCrawler(?:\b|)" 0; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 29c63bbe3..cc2baf875 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.91.101 0; +35.193.184.18 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 97eb2dda9..9bc4b3fd4 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.05.2361 -### Updated: Thu May 27 07:27:03 SAST 2021 +### Version: V3.2021.05.2362 +### Updated: Thu May 27 08:20:22 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 97eb2dda9..9bc4b3fd4 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.05.2361 -### Updated: Thu May 27 07:27:03 SAST 2021 +### Version: V3.2021.05.2362 +### Updated: Thu May 27 08:20:22 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index f26e60914..cf4f606ef 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.05.2361 -### Updated: Thu May 27 07:27:02 SAST 2021 +### Version: V4.2021.05.2362 +### Updated: Thu May 27 08:20:21 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 91396c033..22793dd10 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.66.200.49 0; +35.193.184.18 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index f26e60914..cf4f606ef 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.05.2361 -### Updated: Thu May 27 07:27:02 SAST 2021 +### Version: V4.2021.05.2362 +### Updated: Thu May 27 08:20:21 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 66808fb4d..180bdc087 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.188.15.155 1; +34.122.173.211 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 82a85f46f..bb50a4a64 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.188.15.155 0; +34.122.173.211 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index f26e60914..cf4f606ef 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.05.2361 -### Updated: Thu May 27 07:27:02 SAST 2021 +### Version: V4.2021.05.2362 +### Updated: Thu May 27 08:20:21 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 66808fb4d..180bdc087 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.188.15.155 1; +34.122.173.211 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 82a85f46f..bb50a4a64 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.188.15.155 0; +34.122.173.211 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index f26e60914..cf4f606ef 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.05.2361 -### Updated: Thu May 27 07:27:02 SAST 2021 +### Version: V4.2021.05.2362 +### Updated: Thu May 27 08:20:21 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 66808fb4d..180bdc087 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.188.15.155 1; +34.122.173.211 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 3557e6230..9df798c2c 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -490,6 +490,7 @@ "~*(?:\b)Semrush(?:\b|)" 0; "~*(?:\b)SemrushBot(?:\b|)" 0; "~*(?:\b)SentiBot(?:\b|)" 0; + "~*(?:\b)SeobilityBot(?:\b|)" 0; "~*(?:\b)SEOkicks(?:\b|)" 0; "~*(?:\b)SEOkicks-Robot(?:\b|)" 0; "~*(?:\b)SEOlyticsCrawler(?:\b|)" 0; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 82a85f46f..bb50a4a64 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.188.15.155 0; +34.122.173.211 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index f26e60914..cf4f606ef 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.05.2361 -### Updated: Thu May 27 07:27:02 SAST 2021 +### Version: V4.2021.05.2362 +### Updated: Thu May 27 08:20:21 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 183ac90e0..fc6f8997a 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.197.122.201 0; -34.66.25.221 0; +34.122.208.80 0; +35.193.184.18 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index fe3c79e7e..4e1c8e417 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.05.2361 -### Updated: Thu May 27 07:27:02 SAST 2021 +### Version: V4.2021.05.2362 +### Updated: Thu May 27 08:20:21 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 37407911b..361f3a3d5 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.91.101 0; +35.193.184.18 0; +34.122.173.211 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 98bb7354a..3b89eeb40 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.05.2361 -### Updated: Thu May 27 07:27:02 SAST 2021 +### Version: V4.2021.05.2362 +### Updated: Thu May 27 08:20:21 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index afea80d5d..1e08a0a2d 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.202.245.105 0; -34.66.50.208 0; -35.192.91.101 0; +34.122.208.80 0; +35.193.184.18 0; +104.154.182.187 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index f26e60914..cf4f606ef 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.05.2361 -### Updated: Thu May 27 07:27:02 SAST 2021 +### Version: V4.2021.05.2362 +### Updated: Thu May 27 08:20:21 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 634331777..c4667b0ac 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.188.15.155 0; -34.66.25.221 0; +34.122.208.80 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index f26e60914..cf4f606ef 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.05.2361 -### Updated: Thu May 27 07:27:02 SAST 2021 +### Version: V4.2021.05.2362 +### Updated: Thu May 27 08:20:21 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 0ff8f6817..3bd4c7e2e 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.91.101 1; +35.193.184.18 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/blacklist-user-agents.conf b/.dev-tools/test_units/blacklist-user-agents.conf index 3557e6230..9df798c2c 100644 --- a/.dev-tools/test_units/blacklist-user-agents.conf +++ b/.dev-tools/test_units/blacklist-user-agents.conf @@ -490,6 +490,7 @@ "~*(?:\b)Semrush(?:\b|)" 0; "~*(?:\b)SemrushBot(?:\b|)" 0; "~*(?:\b)SentiBot(?:\b|)" 0; + "~*(?:\b)SeobilityBot(?:\b|)" 0; "~*(?:\b)SEOkicks(?:\b|)" 0; "~*(?:\b)SEOkicks-Robot(?:\b|)" 0; "~*(?:\b)SEOlyticsCrawler(?:\b|)" 0; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index b1b228df8..9dea0adb1 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ +Copyscape Evil -ExtractorPro -PECL::HTTP -POE-Component-Client-HTTP -RealDownload -SMTBot -VB Project -WallpapersHD -WEBDAV -Wprecon +FDM +HTTP::Lite +Nimbostratus +Pavuk +probe-image-size +SalesIntelligent +SISTRIX +Sitevigil diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 3c01344b1..d583a4f25 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,249 +1,249 @@ -01h4x.com -404checker -404enemy +80legs Abonti -ADmantX +Aboundexbot AhrefsBot +AIBOT Alligator +AllSubmitter AlphaBot -Anarchie -Ankit -Apexoo -arquivo-web-crawler +Anarchy +Anarchy99 +archive.org_bot +arquivo.pt ASPSeek Asterias -Attach +autoemailspider AwarioRssBot -AwarioSmartBot -BackDoorBot -BacklinkCrawler -BDFetch -Blow +Backlink-Ceck +backlink-check +BackWeb +BDCbot +Black Hole +BlackWidow BlowFish Brandprotect +Buddy BuiltBotTough -CATExplorador -CCBot -Cegbfeieh -CheTeam +BuzzSumo +Calculon +check1.exe +CheeseBot Cocolyzebot -CODE87 +com.plumanalytics Copier +Copyscape +Cosmos Craftbot -crawler4j -CSHttp -DBLBot -Demon +Curious +Custo +DataCha0s +demandbase-bot Deusu +Devil Digincore -Dirbuster Disco -Discobot -Discoverybot -DittoSpyder +DnyzBot DomainAppender -DomCopBot -Download Wonder -Dragonfly +Dotbot +Drip DSearch -Ebingbong -Ecxi -EirGrabber -EMail Siphon -EMail Wolf +EasyDL +ECCP/1.0 +Evil Exabot Express WebPictures -EyeNetIE -facebookscraper +Extractor FDM -FHscan -Fimap +FemtosearchBot +Flunky Foobot -FrontPage Fuzz -FyberSpider +GalaxyBot +Genieo Getintent +GetRight GetWeb +Gigablast Gigabot +Go-Ahead-Got-It +GoZilla GrabNet +Grafula GrapeFX +GrapeshotCrawler GT::WWW -Haansoft -Harvest -HEADMasterSEO +HaosouSpider +Havij heritrix -Heritrix Hloader HMView -Iblog +HTTP::Lite +Humanlinks IDBot IlseBot -InfoTekies instabid -InterGET -Internet Ninja -InternetSeer -internetVista monitor -Iria -Iskanie -JamesBOT +Intelliseek +ips-agent +IRLbot +isitwp.com +IstellaBot +Jbrofuzz JennyBot -Jetty -Jorgee +JetCar JustView -Jyxobot -Kenjin Spider +Kinza Kozmosbot -Larbin -LeechFTP -LieBaoFast -Linkbot -Linkdexbot -LinksManager -LinqiaMetadataDownloaderBot -LNSpiderguy +Lanshanbot +Lftp +Libwhisker +Lightspeedsystems +Likse +LinkextractorPro +LinkpadBot +LinkWalker +Lipperhey +Lipperhey Spider Ltx71 Mag-Net -Majestic12 +Mail.RU_Bot Majestic SEO -Majestic-SEO MarkMonitor -MarkWatch Masscan -Mb2345Browser meanpathbot -mediawords -MegaIndex.ru -Metauri -MFC_Tear_Sample -MIIxpc +Meanpathbot +MeanPath Bot +MicroMessenger +Microsoft Data Access +Microsoft URL Control Mister PiX -Mojeek +Morfeus Fucking Scanner MQQBrowser -Mr.4x3 -MSIECrawler -Msrabot muhstik-scan Musobot -Nameprotect -Nessus +Name Intelligence +NearSite NetAnts Netcraft -netEstate NE Crawler -Net Vampire +NetLyzer +NetMechanic +NetSpider +NextGenSearchBot +NICErsPRO Nikto -Offline Explorer +Nimbostratus +Ninja +Nmap +Offline Navigator +OnCrawl +OpenLinkProfiler +Openvas OpenVAS -OrangeBot +OPPO A33 OrangeSpider OutclicksBot OutfoxBot -Page Analyzer PageGrabber -PageScorer -PageThing.com +page scorer +Pandalytics +Panscient Pavuk -Petalbot +pcBrowser +PeoplePal Picscout -Pimonster -Pixray -POE-Component-Client-HTTP -ProPowerBot -PxBroker -PyCurl +Picsearch +Piepmatz +plumanalytics +Probethenet +ProWebWalker +Psbot +Pu_iN +Pump QueryN Metasearch -RankActiveLinkBot -RankingBot2 +RankActive +RankFlex +RankingBot +Reaper +RebelMouse ReGet +RepoMonkey Re-re -Rogerbot -RSSingBot +RocketCrawler +SBIder ScanAlert -Scanbot -ScreenerBot -Searchestate +scan.lol +Scrapy +SearchmetricsBot +Semrush +SemrushBot SentiBot -SEOkicks -seoscanners +SeobilityBot +SEOlyticsCrawler +SEOprofiler SeoSiteCheckup -SEOstats -serpstatbot -Shodan -Siphon -SiteCheckerBotCrawler -SiteLockSpider -siteripz +SISTRIX +SiteExplorer +Siteimprove +SiteSnagger SiteSucker -Site Sucker SlySearch -Snapbot +SMTBot +Snake Snoopy -Sogou web spider +Sociscraper +Sosospider +SpaceBison Spammen -SpankBot +sp_auditbot +Spbot Spinn3r -SputnikBot -spyfu Sqlworm -Sqworm -Steeler -Stripper Sucuri -SuperHTTP +SuperBot +Surfbot Suzuran -sysscan +Swiftbot Szukacz -T0PHackTeam -Teleport TeleportPro +Telesphorep The Intraformant -TheNomad +Thumbor TightTwatBot Titan Toata -Toweyabot -Trendictionbot trendiction.com trendiction.de -Turingos +True_Robot Turnitin Twice -UnisterBot Upflow -URLy.Warning +URLy Warning +Vacuum +Vagabondo VB Project -VCI -VeriCiteCrawler +VelenPublicWebCrawler VidibleScraper -VoidEYE -Voil -Voltron +Wallpapers/3.0 Webalta +WebAuto WebBandit -WebCopier -Web Fetch -WebFuck +WebEnhancer +Web Enhancer WebLeacher -WebPix -Web Pix Web Sauger Webshag -WebsiteQuester -Webster -Web Sucker -WebWhacker +WebStripper Whack +Widow WinHTTrack -WISENutbot Wotbox WPScan -WWW-Mechanize x09Mozilla +x22Mozilla +Xaldon_WebSpider Xenu -Zade -Zauba -Zermelo +zauba.io zgrab +Zitebot ZmEu ZoomBot ZoominfoBot diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 0254aca74..5d70f2867 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -360Spider 404checker -Aboundexbot -Acunetix +404enemy ADmantX +AIBOT AiHitBot -Aipbot +Alexibot Alligator +AllSubmitter +AlphaBot Anarchie +Anarchy +Anarchy99 +Ankit Anthill -Apexoo +archive.org_bot +arquivo.pt +ASPSeek Asterias -Attach -AwarioRssBot AwarioSmartBot BackDoorBot Backlink-Ceck -backlink-check -BacklinkCrawler +BackStreet BackWeb -Bandit -Barkrowler -BDCbot +Badass +BatchFTP BDFetch -Bigfoot -Blackboard +BetaBot +Bitacle +Black Hole BLEXBot -BotALot +BlowFish +Brandwatch +Buck Buddy -BuiltWith BunnySlippers -BuzzSumo -CATExplorador -CensysInspect -CheTeam +CazoodleBot +CCBot +Cegbfeieh +check1.exe +CheeseBot +ChinaClaw Chlooe +Claritybot coccocbot-web Cocolyzebot -CODE87 -Collector -Copier +Cogentbot +cognitiveseo CopyRightCheck +Copyscape Cosmos -CrazyWebCrawler -Crescent +crawler.feedback +crawl.sogou.com Curious -Demon +DatabaseDriverMysqli +DBLBot +Deusu +Digincore +DIIbot Dirbuster -Dispatch -DnyzBot -Domains Project -DomainStatsBot +Disco +Discoverybot +DomainCrawler +DomainSigmaCrawler DomCopBot +Dotbot +Download Wonder Dragonfly Drip -DSearch -Ebingbong -Ecxi +DTS Agent +EirGrabber EMail Wolf -EroCrawler -evc-batch -Extractor -ExtractorPro -Extreme Picture Finder +Evil +Exabot +Express WebPictures Ezooms -facebookscraper -FDM +Fimap +FlashGet Flunky FrontPage FyberSpider -Fyrebot -Gigablast -Gigabot -Go-Ahead-Got-It -GoZilla -Go!Zilla -Grafula -GrapeFX -GT::WWW -heritrix +Gotit +Grabber +GrabNet +GrapeshotCrawler +Harvest +Havij +HEADMasterSEO HMView -HTMLparser HTTP::Lite -HybridBot +HTTrack Iblog -Indy Library +IDBTE4M +Id-search +Image Fetch +InfoNaviRobot Intelliseek -Internet Ninja +InternetSeer +ips-agent +IRLbot isitwp.com Iskanie -IstellaBot -JamesBOT -JetCar -Jetty +Jbrofuzz JikeSpider Joomla -Jyxobot -Kenjin Spider +Keyword Density +Kinza +Kozmosbot +Lanshanbot Larbin LeechFTP +LeechGet Lftp -Libwhisker -LieBaoFast +Lightspeedsystems Linkbot -Linkdexbot -LinqiaRSSBot -LinqiaScrapeBot -Lipperhey Spider +LinkScan +LinksManager +LinqiaMetadataDownloaderBot +Litemage_walker LNSpiderguy Ltx71 -lwp-request LWP::Simple lwp-trivial -Mag-Net -Majestic12 -Majestic SEO -Mata Hari -MauiBot -Mb2345Browser +Mail.RU_Bot +Mass Downloader Meanpathbot -MeanPath Bot mediawords -MegaIndex.ru MFC_Tear_Sample -MicroMessenger +Microsoft URL Control +MIDown tool +MIIxpc MJ12bot Mojeek -Mojolicious -Mozlila +Morfeus Fucking Scanner Mr.4x3 -Nameprotect -Nessus +Navroad NetAnts -Netcraft -NetSpider -Net Vampire +NetLyzer +NetMechanic NetZIP -Nibbler -Niki-bot Nikto -Nimbostratus Ninja +Nmap +NPbot +Nutch oBot -Octopus Offline Explorer -Openvas -PageGrabber -PageScorer -PageThing.com +Openfind +OpenLinkProfiler +OpenVAS +OPPO A33 +OrangeSpider +OutfoxBot +Page Analyzer +page scorer Panscient -PHPCrawl -Picscout +pcBrowser +PECL::HTTP +PeoplePal +Petalbot Picsearch -Piepmatz -Pimonster -Pixray -PleaseCrawl -POE-Component-Client-HTTP Probethenet -ProWebWalker -Pu_iN +ProPowerBot PxBroker -Quick-Crawler +PyCurl +RankFlex +RankingBot RealDownload Reaper RebelMouse -Ripper +Re-re ripz -RocketCrawler -s1z.ru +Rogerbot SalesIntelligent -satoristudio.net -SBIder scalaj-http Scanbot +scan.lol +Screaming ScreenerBot SearchmetricsBot -Semrush +SentiBot +SeobilityBot SEOkicks +SEOkicks-Robot SEOlyticsCrawler -serpstatbot -sexsearcher +Seomoz +SeoSiteCheckup +Shodan SISTRIX -Sitebeam -SiteCheckerBotCrawler sitechecker.pro -SiteLockSpider -SiteSucker +siteripz +SiteSnagger +Site Sucker +Sitevigil +SlySearch +SmartDownload SMTBot -Snoopy -Sociscraper +Snapbot +SocialRankIOBot sogouspider Sogou web spider Sottopop +SpaceBison Spanner +Spbot Spinn3r -Sqworm +Sqlmap +Stripper Sucker -SuperBot -Surfbot -sysscan -tAkeOut -Telesphorep -TheNomad +Sucuri +SuperHTTP +SurveyBot +Suzuran +Telesoft Thumbor TightTwatBot Tracemyfile -Turingos -Turnitin -Typhoeus -URLy.Warning +Trendiction +True_Robot +TwengaBot +Twice +Upflow +URLy Warning Vacuum -Vagabondo -VB Project -VCI VeriCiteCrawler -Voil -Wallpapers/3.0 +Virusdie +Webalta Web Auto -WebCollage -WebCopier -WEBDAV -WebEnhancer -Web Enhancer -Web Fetch -WebFuck +Web Collage WebGo IS WebImageCollector +WebLeacher WebmasterWorldForumBot webmeup-crawler +Web Pix WebReaper -Web Sauger -Webshag +WebsiteExtractor WebsiteQuester +Webster WebStripper -WebSucker WeSEE Whacker WinHTTrack -Wonderbot -Wprecon +Woobot +Wotbox WWW-Collector-E WWW-Mechanize -WWWOFFLE -x22Mozilla +Zade zauba.io -Zermelo -Zeus -zgrab +Zitebot ZoomBot diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index c8cc1a785..9fcdf106a 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -diusyawiga.tk -e-biznes.info -glass-msk.ru -hawaiisurf.com -lawyersinfo.org -medictube.ru -pro-tec.kz -qexyfu.bugs3.com -siteripz.net -uchetunet.su +discovertreasure-a.akamaihd.net +iklysha.ml +inboundlinks.win +iopeninghours.co.uk +kihi.gdn +leadn.pl +pirateday.ru +profolan.pl +silktide.com +zzlgxh.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 0803e1bc8..77bd87359 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -000free.us -10bet.com -1kdailyprofit.me -3-letter-domains.net -3rbseyes.com -51unlim.ru -5-steps-to-start-business.com -9icmzvn6.website -9med.net -academia-nsk.org -actulite.com +0912701309f8ce.com +1001desert.com +101raccoon.ru +108shot.com +1j7740kd.website +45en.ru +4inn.ru +5elementov.ru +7a2rnuey1tw9ar.ru +7zap.com +aa9d046aab36af4ff182f097f840430d51.com +abbanreddy.com +acheterviagrafr24.com +activepr.ru +addtoadd.men +adexprts.com adidas.frwebs.fr +adminshop.com adtech.de -adviceforum.info +adultactioncam.com +afmuseum.com +a.frcls.fr air-edem.ru -alldezservice.kz +alekseevec.ru allfinweb.com -alphaforum.ru -amateurgalls.com -amazingpic.net -ameblo.top -anal-acrobats.com anal-acrobats.hol.es -anlimebel.kiev.ua -anniemation.com -apartmentbay.ru +anaseracresar.tk +angigreene.com +animebox.com.ua +anonymizeme.pro apartmentratings.com -apiitsthirteende-a.akamaihd.net -apimossnetinfo-a.akamaihd.net -apispringsmartne-a.akamaihd.net -apple.com-cleaner.systems -applyneedy.xyz -asociatia-tipografilor-transilvania.ro -atley.eu.pn -auto.rusvile.lt +apifasterlightin-a.akamaihd.net +app5.letmacworkfaster.world +apparel-offer.com +appearance-cool.com +arendatora.ru +arpelsreplica.xyz +as5000.com +asia-forum.ru +asrv-a.akamaihd.net +atleticpharm.org +audiofree.ru +autolombard-krasnodar.ru +autonew.biz aviapanda.ru -aviav.org -avirasecureserver.com -aviva-limoux.com -avkzarabotok.com -ayerbo.xhost.ro -azartmix.com -azinoofficial777.ru -bagcionderlab.com -balkanfarma.org -balla-boo.se -bankmib.ru -bannerspace.com -bedcapdealers.com -bestempresas.es +aviav.eu +avtocredit-legko.ru +azbaseimages.net +azbukadiets.ru +azbukafree.com +bablonow.ru +bbtec.net +bedandbreakfast.com +begalka.xyz +belinka.com.ua +benchmarkcommunications.co.uk +berdasovivan.ru +best-businessman.ru +best-deals-products.com +best-games.today bestmouthwash.club -bestofpicture.com -bestwebsiteawards.com +bestofferswalkmydogouteveryday.gq +bestpornuha.com +best-seo-solution.com +beta.hotkeys.com betshuckclean.com -betwinservice.com -bezsporno.ru -biznesrealnost.ru -bjorkbacken.nu -bloggen.be -blog.koreadaily.com -bluesalt.co -blue-square.biz -bochemit.com.ua +bidr.trellian.com +bio-optomarket.ru +blog-hits.com +blogos.kz +bmw-ark.ru +bnt-team.com bolezniorganov.ru -bonzibuddi.com -brillianty.info -budilneg.xyz -bumascloset.com -business-made-fun.com -buyantiviralwp.com -cactussoft.biz -candelluxsklep.pl -cardsharp1.ru +brothers-smaller.ru +buigas.00it.com +buy-forum.ru +buyk.host.sk +buypuppies.ca +buzz-porno.info +buzzsumo.com +cacheimages.com +cakesplus.com.au +candlehandmade.com +candytiner.com +carsplate.com cartierlove2u.xyz -cartierloveringreplica.xyz -cartierlovestore.xyz -cascadelink.org -cdnnetwok.xyz -chatroulette.world -cheapjerseysbizwholesale.us -cheapmoncler.xyz -cheatcode-lita12.rhcloud.com +cartierreplica.win +cdncash.com +celejihad.info cherrypointplace.ca -chocoslim.pro -cienum.fr -cityadspix.com -cjmc.info -classicakuhni.ru -clickso.com -colobit.biz -com-onlinesupport.site -compliance-george.xyz -compliance-julianna.top -com-secure.download -conboy.us -cookie-law-enforcement-hh.xyz -cornerstone-countertops.com -costablue.xyz -countercrazy.com -cranly.net -credit-respect.ru +christianlouboutinsaleoutletonline.us +cialischsrx.com +civilwartheater.com +clipartnew.com +club-lukojl.ru +codq.info +communisave.co.za +compliance-alexa.xyz +compliance-alex.xyz +compliance-nelson.top +connectingsingles.com +conocer-sanabria.com +cookielawblog.wordpress.com +counter.yadro.ru +courtshipgift.com +crackzplanet.com +creditwell.ru crest3d.ru -customcatchcan.com -cwetochki.ru -cyprusbuyproperties.com -daptravel.com -designdevise.com -detoxmed24.ru -dev33.dioniqlabb.se -discountliv.com -dividendo.ru -docstoc.com -domain2008.com -domainsatcost.com -domini.cat -draniki.org -drugspowerstore.com +crest-poloski.ru +crynet.cc +cubs-jerseys.us +cymbaltaandtizanidine.blogspot.com +d2jsp.org +dairyindia.in +datadepths.com +ddsoldes.fr +dekoration.us +delayreferat.ru +dengi-pod-zalog-nedvizhimosti.ru +detskie-zabavi.ru +dev.citetick.com +dev-seo.blog +dfwu1019.info +dimaka.info +discovertreasurenow.com +divci-hry.info +djihispano.com +dmmspy.com +dms-sw.ru +dojki365.online +doktoronline.no +dollartree.info +dotmass.top druzhininevgeniy63.blogspot.com -dscaas.website -duawitchrarato.tk -dynainbox.com -dyt.net -eaplay.ru -earncash.com.ua -easync.io -ecxtracking.com -educatemetv.com -edwinkonijn.com.au -ege-russian.ru -ekaterinburg.xrus.org -ekn-art.se -ekobata.ru +earl-brown.info +ecommerce-seo.com +eduinfosite.com +elektrischezigaretten1.blogse.nl +elektronischezigarettekaufen1.beeplog.com elektrozigaretten1.tumblr.com -elkacentr.ru -elmifarhangi.com -englishgamer.com -erot.co -erotik-kostenlos.net -es5.com -esopini.com -euromasterclass.ru +elidelcream.weebly.com +eloxal.ru +enge-fotzen.info +enginebay.ru +engines-usa.com +enskedesquashclub.se european-torches.ru -extractorandburner.com -eyemagination.com -ezigarettekaufen2.yolasite.com -failingmarriege.blogspot.com -faithe.top -familienzahnaerzte.com -family1st.ca -fanoboi.com -fettefrauen.net -filesdatabase.net -financepoints.eu +exmasters.com +exportshop.us +extlinks.com +extremepornos.net +f0815.de +facebook-mobile.xyz +familyphysician.ru +fanpagerobot.com +fatfasts-4tmz.com +feargames.ru +ferrotodo.com +fetroshok.ru +fighrofacciufreesig.ga +filmfanatic.com +filunika.com.ru findclan.org -findpik.com -finuse.com -fireads.men -firma-legion.ru -fivedwld.cf -f-loaded.de -formulaf1results.blogspot.com -fotzen-ficken.com -franklinfire.co -freegamesplay.online -free-social-buttons.xyz -frequiry.com -frvo.alptandem.ru -fuckingawesome.com -funnypica.com -fxtips.ru -gameonasia.com -gameplexcity.com -gameprimary.com -gdcentre.ru -geileweiber.tk -getresponse.com -gettpromos.com -glasof.es -gogps.me -gotorussia.com -grizzlysgrill.com -guge.io -gwhwpxbw.bloger.index.hr -hacktougroup.ru -hamptonoaks.ca -haveinc.xyz -health-medical-portal.info -heatpower.ru -hoholikik.club -holidaypics.org -homedecorpicture.us -hostnow.men -hotgirlhdwallpaper.com -hotkeys.com -hotsocialz.com -houtings.xyz -htmlcorner.com -hully.altervista.org -humbmosquina.tk -hundejo.com -hyhj.info -igrovyeavtomaty777.ru -images-graphics-pics.com -imfamous.info -impisr.edunsk.ru +flavors.me +flowwwers.com +fok.nl +formularz-konkurs.tk +free-fb-traffic.com +freelifetimefuckbook.com +fres-news.com +frustrated-favorable.gq +gallictures.com +gamblingnerd.com +gdebestkupit.ru +generousdeal-a.akamaihd.net +germes-trans.com +get-seo-domain.com +gidonline.one +girls-ufa.ru +glicol.kz +gobongo.info +goodhumor24.com +googlefeud.com +goosefishpost.bid +graphid.com +greatgrace.ru +gribokstop.com +groupmoney.ru +gsasearchengineranker.top +gta-club.ru +guarrasdelporno.xxx +guiadeserraazul.com +hao123.com +healbio.ru +heartofpayne.xyz +herehloadibs.cf +hmmm.cz +hosting-tracker.com +hotchatdate.com +hothor.se +hotloans.ru +houston-vikings.com +howtoclean.club +huimin764128.com +i-hobot.ru +imgarit.pw +i-midias.net.br +imperiafilm.ru inboxdollars.com -innoslicon.com +incest-ru.com +incolors.club +insta-add.pro +instasexyblog.com +insurple.com intellego.info -internet-apteka.ru -internetproviderstucson.com -int.search.mywebsearch.com -iopeninghours.co.uk -iqoption.com -i-service.kz -ispaniya-costa-blanca.ru -ix20.ru +iomoio.net +iqs.biz.ua +irkutsk.zrus.org +itbc.kiev.ua +itsdp3.com +itservicesthatworkforyou.com +ivearchenceinflu.cf +iyasimasennka.com j33x.com -jamiembrown.com -javidol.site -javitas.info +james13prix.info javrip.net -je7.us jrpmakati.com -kerwinandcariza.com -keywordspay.com -kidd.reunionwatch.com -kinobaks.com +jwss.cc +kaac.ru +kadashihotel.com +karpun-iris.ru +kchaxton.com +kinoduh.ru kinogolos.ru -kinogonew.ru -kiprinform.com -klejonka.info +kinotorka.ru kmgamex.cf -knowsitall.info -knowyournextmove.com -koleso24.com.ua -komputers-best.ru -konkursov.net -ktotut.net -kumuk.info +kol-energo.ru +kotaku.com +krasivoe-hd.net +krasnodar.ru +kreativperlen.ch +ks1234.com +kwzf.net laapp.com -labafydjxa.su -lalalove.ru -lawyers.cafe +labelwater.se +landinez.co +larutti.ru +leboard.ru lenpipet.ru -likesuccess.com -linerdrilling.com -linkarena.com +lexixxx.com +linkwithin.com +live-cam6.info +lmrauction.com loadingpages.me -lonfon.xyz -lostfilm-online.ru -lubetube.com -mafcards.ru -magento-crew.net -mainlinehobby.net -maju.bluesalt.co -mall.uk -marblestyle.ru -marcogrup.com -marketingvici.com -maslenka.kz +locatellicorretor.com.br +loftdigital.eu +m5home.ru +mactechinfo.info +mahnwachen-helfen.info +managerpak204.weebly.com +mararoom.ru masterseek.com -mbiologi.ru -mcnamaratech.com -mearns-tractors.co.uk +matb3aa.com mebel-alait.ru -mediafresh.online +mediawhirl.net mejoresfotos.eu -meltwater.com -menetie.ru -mensandals.xyz -mexicotravelnet.com -mhtr.be -michaelkorsoutlet.store -mil-stak.com -mindbox.co.za -moetomnenie.com -moscow-clining.ru -motherboard.vice.com -mp3ritm.top -musicspire.online -musicvidz.ru -muzaporn.com -mydirtystuff.com +mesbuta.info +m-google.xyz +missclub.info +missvietnam.org +mmofreegames.online +mmog-play.ru +mnimmigrantrights.net +mnogolok.info +mockupui.com +monarchfind-a.akamaihd.net +moneyzzz.ru +moscow.xrus.org +moskva.nodup.ru +muycerdas.xxx +my-big-family.com +mydoctorok.ru +my-floor.in.ua myghillie.info -mykings.pw -myperiod.club -nakozhe.com -narco24.me -narkologiya-orel.ru -navalwiki.info -newhairstylesformen2014.com -nextlnk12.com -nudejapan.net -nullrefer.com -nw-servis.ru -onlineporno.site -o-o-8-o-o.com -ooo-gotovie.ru -ooo-olni.ru -openfrost.com -ororodnik.goodbb.ru -oxotl.com -park.above.com -paydayonlinecom.com +mymercy.info +myplaycity.com +nalogovyykodeks.ru +national-today-winning-winner.club +naughtyconnect.com +nbsproject.ru +newyorkhotelsmotels.info +nhl09.ru +nightvision746.weebly.com +njkmznnb.ru +nonameread45.live +northfacestore.online +nosecret.com.ua +oivcvx.website +online-hd.pl +online-zaymy.ru +ons-add.men +ooomeru.ru +oops-cinema.ru +opinionreelle.com +optikremont.ru +oralsexfilme.net +osagonline.ru +pagesense.com +painting-planet.com +palvira.com.ua +pandarastore.top +parvezmia.xyz +patterntrader-en.com +pdamods.ru pdn-4.com -pestomou.info -pflexads.com -picscout.com -picsearch.com -pinapchik.com -piratecams.com -pivka.xyz -pixell.club -pl-vouchers.com +pelfind.me +perederni.net +perm-profnastil.ru +pirateday.ru +plastgran.com +plastweb.ru +playlott.com +pl.id-forex.com +podshipniki-nsk.ru poems.com.ua -poisk-zakona.ru -popads.net -popander.mobi -pornogad.com -pornoreino.com -pornosee.info +pogruztehnik.ru +pokemongooo.ml +polytopesexempt.com +popcash.net +porndairy.in +pornodojd.ru +pornohd1080.online +pornolook.net pornosexrolik.com -pornoxxx.com.mx -porn-w.org -porodasobak.net -portal-eu.ru -portside.xyz -posible.net -pozdrawleniya.ru -prize44.com -prizestohandle.club -prod2016.com -prom23.ru -promgirldresses.xyz -pronekut.com -pronorm.fr -prosperent.com -pufip.com -purchasepillsnorx.com -puserving.com -pussyspace.net -quelle.ru -questionmarque.ch -quickloanbank.com -quick-offer.com -ranking2017.ga -rankingchart.de -rebelmouse.com -reklama-i-rabota.ru +pornoted.com +powitania.pl +pozdravleniya-c.ru +pp-budpostach.com.ua +privat-girl.net +prizesbook.online +proekt-mos.ru +projefrio.com.br +prostitutki-kiev.org +proxyelite.biz +puzo2arbuza.ru +pxhdwsm.com +quickchange.cc +rabotaetvse.ru +rangjued.com +rasteniya-vs-zombi.ru +razleton.com +registry-clean-up.net +renecaovilla.online renhacklids.tk -rfid-locker.co -rfserial.net -rialp.getenjoyment.net -rockma.se +rep-am.com +reversing.cc +rniaeba.ga +rocketchange.ru roll123.com +rootandroid.org rospromtest.ru -rozalli.com -rrutw.com -runstocks.com -rusenvironmental.net -s1z.ru -safe-app.net -saitevpatorii.com -salmonfishingsacramentoriver.com -santasgift.ml -sardinie.us -satoristudio.net -scanner-elena.top -scanner-josh.top -scanspyware.net -scrnet.biz.ua -search.1and1.com -semp.net -sendearnings.com -seoboxes.com -seocheki.net -seogadget.ru -seojokes.net -seorankinglinks.xyz -seotoolsagency.com -servicecenter.co.ua -sexblog.pw -sextracker.com -sexy-pings.com -s-forum.biz -ship-marvel.co.ua -shisha-swag.de -shitmovs.com -shivafurnishings.com -shlyahten.ru -shopvilleroyboch.com.ua -simpoed.ufop.br -sirpornogratis.xxx -skylta.com -slomm.ru -slutloadlive.com -smartshoppymac.com -snowplanes.com -sobecjvuwa.com.ru -socialmonkee.com -spinnerco.ca -sport-video-obzor.ru -starpages.net -statustroll.com -stickers-market.ru -stoki.ru -stpolice.com -stromerrealty.com -studworks.org -suchenindeutschland.com +royalads.net +savingsslider-a.akamaihd.net +sbetodiodnye-lampy.ru +sbt-aqua.ru +scat.porn +sc-specialhost.com +sdelatmebel.ru +searchengineranker.email +search-goo.com +seccioncontrabajo.com +secretscook.ru +semrush.com +seo18.su +seofirmreviewsus.info +seoheap.com +sexgalleries.top +sexiporno.net +sexkontakte-seite.com +sexsaoy.com +sexy.babes.frontend-stack.top +sexystrippe.info +sexytrend.ru +shodanhq.com +sitebeam.net +site.ru +skutecznetabletkinaporostwlosow.pl +slavic-magic.ru +smadihome.com +snip.tw +snjack.info +snworks.com +snyatie-lomki-v-stacionare.ru +socialtrade.biz +soheavyblog.com +splendorsearch-a.akamaihd.net +srdrvp.com +stair.registrydomainservices.com +stockspmb.info +stroiminsk.org sucsesofinspiration.com -sudexpert66.ru -takeflyte.com -tam-gde-more.ru -tanieaukcje.com.pl -tasteidea.com -tds-advert005.info -tedxrj.com -teenporn18.net -telefonsexsofort.tk -telsis.com -testbotprocessor44.com -thebitcoincode.com +svarkagid.com +svbur.ru +symphonyintegratedhealthcare.com +szucs.ru +tarad.com +tattoo33.ru +taximytishi.ru +templates.franklinfire.co +thebestphotos.eu +thedownloadfromwarez.blogspot.com +thegolfclub.info +themeforest.net thepokertimer.com -thetardistimes.ovh -tizanidine4mgstreetprice.blogspot.com -tizanidinedruginteractions.blogspot.com -tizanidinehcl2mg.blogspot.com tizanidinehcl2mgsideeffects.blogspot.com -tmm-kurs.ru -tongkatmadura.info -top-study.work -toyota.7zap.com -trafaret74.ru -trafficmonetize.org -trion.od.ua -truemfilelj.gq -tsan.net -tubo360.com -tuckermktg.com -twittruth.com -ukkelberg.no -ukrtvory.in.ua -ultimatesetnewfreeallsoftupgradesystems.pw -unece.org -univerfiles.com -uptimechecker.com -urdoot.win -usswrite.com -ustion.ru -valuado.com -v-doc.co -vegascosmetics.ru -venta-prom.ru -vesnatehno.ru -via-energy-order.com -viagra-soft.ru -video-chat.cn -videochaty.ru -videtubs.pl -vita.com.hr -vodkoved.ru -vriel.batcave.net -vseuznaem.com -vucms.com -wapsite.me -warningzscaler.heraeus.com -waterpurifier.club -webix.me +tizanidineusedfor.blogspot.com +tizanidinevsvalium.blogspot.com +tjkckpytpnje.com +topcasinoratings.ru +topshef.ru +topsiteminecraft.com +torrent-newgames.com +torrent-to-magnet.com +tracksurf.daooda.com +traderzplanet.in +trafficreceiver.club +tula.howotorg.ru +typer.one +tytoona.com +uchebavchehii.ru +ukrup.com +ultramart.biz +um-razum.ru +unblocksit.es +undergroundcityphoto.com +unpredictable.ga +uptime-as.net +uptime-gamma.net +urccvfmc.bloger.index.hr +vacances-voyages.info +vadimkravtcov.ru +velobikestock.com +vepad.com +via-energy-cumpara.com +viandpet.com +viberdownload10.com +videochat.world +videogamesecrets.com +videooko.weebly.com +video-woman.com +vigrx-original.ru +vip2ch.com +vipms.ru +vipsiterip.org +vostoktrade.info +vse-pesni.com +walpaperlist.com +watchmyfb.pl +webnode.me +webpromotion.ae websitevaluebot.com -wejdz-tu.pl -whatsappdownload10.com +weburlopener.com +welcomeauto.ru whatzmyip.net wheelchairliftsarea.com -wikes.20fr.com -wnoz.de -workle.website -world-mmo.com -www.arenda-yeisk.ru -www.event-tracking.com -xfire.com -xn--1-8sbcpb0bdm8k6a.xn--p1ai -xn----8sbdbjgb1ap7a9c4czbh.xn--p1acf -xtubeporno.net -xxxsiterips.xyz -xxx-treker.ru -yourporn.com -yugo-star.ru -yurgorod.ru -zaapplesales.blogspot.com -zaimite.ru -zaloro.com -zdesformula.ru +wholesalecheapjerseysfree.com +wholesalejerseychinaoutlet.com +womama.ru +wpthemedetector.co.uk +www.ehscloud.cn +x5market.ru +xaylapdiendanang.com +xblog.in +xep.info +xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai +xn-----7kcabaipgeakzcss7bjdqdwpfnhv.xn--p1ai +xn--80aagddcgkbcqbad7amllnejg6dya.xn--p1ai +xn----8sbhefaln6acifdaon5c6f4axh.xn--p1ai +xn--d1acah0c.xn--p1ai +xn--h1ahbi.com.ua +xn--l1aengat.xn--p1ai +xpresscare.ru +xxx-cam.webcam +yellowproxy.net +yoluxuryevents.com +youdesigner.kz +youtubologia.it +zaimhelp.ru +zaim-pod-zalog-krasnodar.ru +zaloadi.ru +zaobao.com.sg zebramart.ru -zeikopay.com -zerocash.msk.ru -zionstar.net -zoodrawings.com -zophim.me -zzlgxh.com +zeroredirect10.com +zeroredirect5.com +zeroredirect6.com +zigarettenonlinekaufen.tumblr.com +zreloeporno.tv +zryydi.com +zvetki.ru +zvooq.eu diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index b7e84aacc..21130a053 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -007angels.com -2x2fan.ru -4ur.click -991mostfm.co.id +11pikachu.ru +18ps.ru +777-club.ru +aa625d84f1587749c1ab011d6f269f7d64.com +aab94f698f36684c5a852a2ef272e031bb.com +academiacsmendoza.org +adclickservice.com addblueoff.com.ua -admitad.com -advokat-grodno.by -albamargroup.com -aliexpress.com -amazon-adsystem.com -amehdaily.com -amospalla.es +adloads.net +alfabot.xyz +alldownload.pw +allwidewallpapers.com +alt-servis.ru ampower.me -analytics-ads.xyz -angigreene.com angkortours.vn -animal36.com -apelsinnik.website -apibatbrowsecom-a.akamaihd.net -appfastplay.com -arius.tech -arpelsreplica.xyz -artparquet.ru -artpicso.com -ayeartoforget.com -babyguns.ru -bankiem.pl -battleforupdating.site -bestcoin.cc -bestdooz.com -betune.onlinewebshop.net -bezlimitko.xyz -blackle.com -browsepulse-a.akamaihd.net -bumascloset.com -buytizanidineonlinenoprescription.blogspot.com -calcularpagerank.com -canvas.gloverid.site -carson.getenjoyment.net -cartierbraceletsreplica.pw -casinopinup-wins.com -c-english.ru -chidporn.com -cleanmypc.com -clickpapa.com -cocolyze.com -codysbbq.com -compliance-brian.top -compliance-john.top -cowlmash.com -crazyboost.pro -creativehutindia.com -data1.scopich.com -decorazilla.com -deluxedumps.com -den-noch24.ru -dieta-personalna.pl -dmmspy.com -dotmass.top -education-cz.ru -ekn-art.se -elektrischezi.canalblog.com -elvenar.com -enskedesquashclub.se -evaashop.ru -evening-dating-club.info -exoneration-project.us -extrabot.com -ez8motelseaworldsandiego.com -fanrto.com -fast-torrent.ru -fealq.com -finstroy.net -forex-procto.ru -free-share-buttons.top -fxgallery.com -getmyads24.com -globatur.ru -gne8.com -good-mummy.ru -gosarhivrt.ru -gotomontenegro.net -graphid.com -gsasearchenginerankerhelp.com -gstatey.net -gyffu.com -hanwei.us -hem.passagen.se -hidefiles.org -hmmm.cz -homeart.space -hornymatches.com -humbmosquina.tk -i-hobot.ru -ilikevitaly.com -impotentik.com -indiakino.net -iscblog.info -james13prix.info -javxxx18.com -keywordsuggests.com -kinomaniatv.pl -kinopolet.net -kinosed.net -klitimg.pw -kopihijau.info -kosmetyki.tm.pl -koversite.info -kreativperlen.ch -ktotut.net -ladov.ru -landoftracking.com -lapitec.eu +arate.ru +arclk.net +arewater.com +as5000.com +asacopaco.tk +asscenihotosidea.blogspot.co.za +astrochicks.com +atout-energie-69.com +ayanaspa.com +berrymall.ru +bg6s0.com +bif-ru.info +bigames.online +big-boards.info +big-cash.net +blue-square.biz +boanonihaca.tk +board.f00d.de +bonanza-fortune.men +bongacams.com +bonvillan.com +bonzbuddy.com +braindaily.xyz +brg8.com +brus.city +buzzzg.men +chelyabinsk.xrus.org +chiblackhawks-jerseys.com +classicakuhni.ru +cl.s7.exct.net +cmhomestayagency.com +cookie-law-enforcement-ff.xyz +coolwallpapers-hd.com +coprofam.org +counterbot.com +credit-respect.ru +creditwell.ru +csgo4.win +custom-electric-guitar.com +customsua.com.ua +directrev.com +disruptingdinnerparties.com +djekxa.ru +domashneeruporno.com +drillsaw.ru +edwinkonijn.com.au +electro-prom.com +elektrozigaretten1.postbit.com +elektrozigaretten2.yn.lt +en.altezza.travel +enbersoft.com +ereko.ru +escortslet.net +europages.com.ru +exchangeit.gq +exhibitionplus.eu +ezigarettenkaufen1.pagina.gr +f0815.de +ferretsoft.com +fillmewithhappiness.com +filmfanatic.com +find1friend.com +fishtauto.ru +flashbannernow.com +forms-mtm.ru +fortwosmartcar.pw +freelifetimefuckbook.com +free-social-buttons.com +fr.netlog.com +funnymama.com +gael-s.ru +get-free-social-traffic.com +gfaq.ru +goldenggames.com +googlefeud.com +googlemare.com +graphics8.info +g.starmoe.xyz +hackers-crackers.tk +hebr.myddns-flir.com +helpmymacfaster.trade +hentai-manga.porn +homedo.fabpage.com +humanorightswatch.org +imk.com.ua +incanto.in.ua +istizanidineanarcoticdrug.blogspot.com +izismile.com +jagg.info +kanimage.com +kazan.zrus.org +kazrent.com +kemerovo.zrus.org +ketoanhanoi.info +kollekcioner.ru +kollesa.ru +komputernaya-pomosh-moscow.ru +kredit-pod-zalog-krasnodar.ru +ks1234.com +lalalove.ru +lancheck.net lasvegaslockandsafe.com -le-clos-des-alouettes.com -lernur.net -liky.co.ua -lincolntheatre.com -linkwithin.com -lirunet.ru -luciddiagnostics.in -macdamaged.space -mazda-roadsters.com -mcnamaratech.com -medmajor.ru -mmog-play.ru -monclercheap.xyz -moneyviking-a.akamaihd.net -mp3ringtone.info -msfsaar.de +lawyers.cafe +laxdrills.com +ldrtrack.com +linkredirect.biz +lion.misba.us +lkbennettoutlet.online +lovi-moment.com.ua +lubetube.com +luchshie-topcasino.ru +m1media.net +magnetpress.sk +malls.com +marmitaco.cat +mebel-ekb.com +me-ke.com +micasainvest.com +mielec.pl +mir-business-24.ru +monsterdivx.tv +montredemarque.nl +moroccosurfadventures.com +music7s.me +musicas.baixar-musicas-gratis.com +mxgetcode.com +mydownlodablefiles.com myhitmp3.club -narco24.me -nash-krym.info -news-speaker.com -nextconseil.com -nicefloor.co.uk -oballergiya.ru -odoratus.net -of-ireland.info -o.light.d0t.ru -onlainbesplatno.ru +mysex21.com +natali-forex.com +netanalytics.xyz +nisuturnetdgu.tk +november-lax.com +obiavo.kz +obiavo.su +offer.camp +oneminutesite.it onlinadverts.com -owohho.com -pacificair.com -paclitor.com -pateaswing.com -pc-virus-d0l92j2.pw -photo-clip.ru -php-market.ru -pic2fly.com -picturesify.com -pinsdaddy.com +onlinewritingjobs17.blogspot.ru +paintingplanet.ru +panouri-solare-acoperis.com +parajumpersjakkesalgnorge.info +partnerads.men +pcimforum.com +pdn-4.com +pechikamini.ru +perfection-pleasure.ru +perosan.com +pewit.pw +phobia.us +pixgood.com play-mp3.com -pochemychka.net -pomoc-drogowa.cba.pl -popander.mobi -porto.abuilder.net -profitkode.com -prom23.ru -prostitutki-belgoroda.org -proxyradar.com -purchasepillsnorx.com -pussyspace.net -rapevideosmovies.com -rating-bestcasino.com -realting-moscow.ru -rebrand.ly -relax.ru -rentaremotecomputer.com -risparmiocasa.bz.it -rus-pornuha.com -russkie-gorki.ru -rvtv.ru -rvzr-a.akamaihd.net -ryetaw.com -santasgift.ml -sbt-aqua.ru -scat.porn -schalke04fc.info +poem-paying.gq +pogosh.com +pokerniydom.ru +pons-presse.com +popserve.adscpm.net +popup-hgd.xyz +pornhub-ru.com +pornoblood.com +pornosee.info +pornoslive.net +preconnubial.usuby.site +privatefx.all4invest.info +qitt.ru +quebec-bin.com +quelle.ru +r-control.ru +renecaovillasale.online +renewablewealth.com +rfd-split.hr +robertefuller.com +roznica.com.ua +saltspray.ru +salutmontreal.com sdelai-prosto.ru -sell-fb-group-here.com -serialsway.ucoz.ru -serw.clicksor.com -sethrollins.net -sex-pr.net -silverage.ru -similardeals.net -skatestick.bid -slavia.info -soft-terminal.ru -softxaker.ru -stauga.altervista.org -strongsignal-a.akamaihd.net -sugarkun.com -surffoundation.nl -swinginwithme.ru -swinon.site -szamponrevita.pl -tasteidea.com -td-33.ru -thebitcoincode.com -tiandeural.ru -titslove.yopoint.in -tizanidinedosage.blogspot.com -tizanidinehydrochloride4mgstreetvalue.blogspot.com -token-lab.org -toondinsey.com +sdi-pme.com +searchengineranker.email +seoexperimenty.ru +seogadget.ru +sexxdate.net +shama-rc.net +shop.acim.org +shtaketniki.ru +slomm.ru +social-search.me +sogimlecal.tk +solmarket.by +souvenir.cc +splendorsearch-a.akamaihd.net +sport-video-obzor.ru +startwp.org +stop-zavisimost.com +styro.ru +suture.co +svarbit.com +t3chtonic.com +tilido.com +tizanidinenarcotic.blogspot.com topflownews.com -topkarkas.com -traffixer.com +topquality.cf +torrent-newgames.com +traffic100.com +trafficshaper.com trustedhealthtips.com -tubeoffline.com -tupper-posuda.ru -tvoystartup.ru -udsgame.online -umekana.ru -usswrite.com -valmetrundan.se -vbikse.com -videochat.ph -videositename.com -vietimgy.pw -villakohlanta.nu -vinsit.ru -vostoktrade.info -vote-up.ru -vsdshnik.com -vykup-avto-krasnodar.ru -watchmygf.net -we-are-gamers.com -webshoppermac.com -website-speed-up.top -wesharepics.com -wesharepics.site -womens-journal.net -wzgyyq.com -xblog.in -xchangetrak.com -xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai -xn--80aaaks3bbhabgbigamdr2h.xn--p1ai -xn--80aagddcgkbcqbad7amllnejg6dya.xn--p1ai +ua-company.ru +ucsol.ru +unece.org +univerfiles.com +urengoy.pro +uzungil.com +vadimkravtcov.ru +v-doc.co +verymes.xyz +videochat.mx +viktoria-center.ru +vitalads.net +vkgaleria.com +vzube.com +wdrake.com +webix.me +weblibrary.win +whereverdesperate.gq +wholesalejerseysgaa.com +windowssearch-exp.com +woman-orgasm.ru +wpsecurity.website +wurr.voila.net xn-----8kcatubaocd1bneepefojs1h2e.xn--p1ai -xn----8sbdbjgb1ap7a9c4czbh.xn--p1acf -xn--hxazdsfy.blogspot.com -xolodremont.ru -xxart.ru -you-shall-not-pass.is74.ru -youtuhe.com -zakazvzloma.com -zvuker.net +yhit.press +yourads.website +youtoner.it +zazagames.org +zerocash.msk.ru +zx6.ru diff --git a/.dev-tools/test_units/whitelist-ips.conf b/.dev-tools/test_units/whitelist-ips.conf index 29c63bbe3..cc2baf875 100644 --- a/.dev-tools/test_units/whitelist-ips.conf +++ b/.dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.91.101 0; +35.193.184.18 0; 127.0.0.1 0; diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index 0db95b635..cab12945b 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index b83cc521c..a5564c47b 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 31ae3a8eb..70db4b172 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.05.2361 +#### Version: V4.2021.05.2362 #### Bad Referrers Blocked: [7050](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [608](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [609](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index f556660a5..2e243b56e 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.05.2361 +#### Version: V4.2021.05.2362 #### Bad Referrers Blocked: [7050](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [608](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [609](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index 2842e6676..05552a824 100755 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.05.2361 +#### Version: V4.2021.05.2362 #### Bad Referrers Blocked: [7050](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [608](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [609](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index b477b096a..3848dd13e 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,4 +1,3 @@ -SeobilityBot 01h4x.com 360Spider 404checker @@ -430,6 +429,7 @@ SearchmetricsBot Semrush SemrushBot SentiBot +SeobilityBot SEOkicks SEOkicks-Robot SEOlyticsCrawler diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index f9f20640f..6ccfc1348 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.05.2361 +#### Version: V4.2021.05.2362 #### Bad Referrers Blocked: [7050](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [608](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [609](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index f26e60914..cf4f606ef 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.05.2361 -### Updated: Thu May 27 07:27:02 SAST 2021 +### Version: V4.2021.05.2362 +### Updated: Thu May 27 08:20:21 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 97eb2dda9..9bc4b3fd4 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.05.2361 -### Updated: Thu May 27 07:27:03 SAST 2021 +### Version: V3.2021.05.2362 +### Updated: Thu May 27 08:20:22 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index f26e60914..cf4f606ef 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.05.2361 -### Updated: Thu May 27 07:27:02 SAST 2021 +### Version: V4.2021.05.2362 +### Updated: Thu May 27 08:20:21 SAST 2021 ### Bad Referrer Count: 7050 -### Bad Bot Count: 608 +### Bad Bot Count: 609 ################################################### ### VERSION INFORMATION ## @@ -578,6 +578,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)SEOkicks(?:\b)" 3; "~*(?:\b)SEOkicks-Robot(?:\b)" 3; "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index c11104892..06e8b659d 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2021.05.2361 -### Updated: Thu May 27 07:27:04 SAST 2021 -### Bad Bot Count: 608 +### Version: V4.2021.05.2362 +### Updated: Thu May 27 08:20:23 SAST 2021 +### Bad Bot Count: 609 ################################################### ### Version Information ## @@ -877,6 +877,8 @@ User-agent: SemrushBot Disallow:/ User-agent: SentiBot Disallow:/ +User-agent: SeobilityBot +Disallow:/ User-agent: SEOkicks Disallow:/ User-agent: SEOkicks-Robot