diff --git a/.dev-tools/_robots_input/robots-input.txt b/.dev-tools/_robots_input/robots-input.txt index c16c13525..ee0b8277e 100644 --- a/.dev-tools/_robots_input/robots-input.txt +++ b/.dev-tools/_robots_input/robots-input.txt @@ -79,6 +79,7 @@ Claritybot Cliqzbot Cloud mapping coccocbot-web +CODE87 Cogentbot cognitiveseo Collector @@ -195,6 +196,7 @@ Humanlinks HybridBot Iblog IDBot +IDBTE4M Id-search IlseBot Image Fetch diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index a5914add1..4f32983e7 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.12.2186 -### Updated: Fri Dec 4 09:59:13 SAST 2020 +### Version: V3.2020.12.2187 +### Updated: Fri Dec 4 11:03:34 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 180bdc087..3bd4c7e2e 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 1; +35.193.184.18 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index bb50a4a64..cc2baf875 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 0; +35.193.184.18 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index a5914add1..4f32983e7 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.12.2186 -### Updated: Fri Dec 4 09:59:13 SAST 2020 +### Version: V3.2020.12.2187 +### Updated: Fri Dec 4 11:03:34 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 180bdc087..3bd4c7e2e 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 1; +35.193.184.18 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index bb50a4a64..cc2baf875 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 0; +35.193.184.18 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index a5914add1..4f32983e7 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.12.2186 -### Updated: Fri Dec 4 09:59:13 SAST 2020 +### Version: V3.2020.12.2187 +### Updated: Fri Dec 4 11:03:34 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 180bdc087..3bd4c7e2e 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 1; +35.193.184.18 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 852c5cb39..4e7a334b2 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -140,6 +140,7 @@ "~*(?:\b)Cliqzbot(?:\b|)" 0; "~*(?:\b)Cloud\ mapping(?:\b|)" 0; "~*(?:\b)coccocbot-web(?:\b|)" 0; + "~*(?:\b)CODE87(?:\b|)" 0; "~*(?:\b)Cogentbot(?:\b|)" 0; "~*(?:\b)cognitiveseo(?:\b|)" 0; "~*(?:\b)Collector(?:\b|)" 0; @@ -256,6 +257,7 @@ "~*(?:\b)HybridBot(?:\b|)" 0; "~*(?:\b)Iblog(?:\b|)" 0; "~*(?:\b)IDBot(?:\b|)" 0; + "~*(?:\b)IDBTE4M(?:\b|)" 0; "~*(?:\b)Id-search(?:\b|)" 0; "~*(?:\b)IlseBot(?:\b|)" 0; "~*(?:\b)Image\ Fetch(?:\b|)" 0; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index bb50a4a64..cc2baf875 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 0; +35.193.184.18 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index a5914add1..4f32983e7 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.12.2186 -### Updated: Fri Dec 4 09:59:13 SAST 2020 +### Version: V3.2020.12.2187 +### Updated: Fri Dec 4 11:03:34 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index a5914add1..4f32983e7 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.12.2186 -### Updated: Fri Dec 4 09:59:13 SAST 2020 +### Version: V3.2020.12.2187 +### Updated: Fri Dec 4 11:03:34 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 882f8d061..7981d4484 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.12.2186 -### Updated: Fri Dec 4 09:59:12 SAST 2020 +### Version: V4.2020.12.2187 +### Updated: Fri Dec 4 11:03:33 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 882f8d061..7981d4484 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.12.2186 -### Updated: Fri Dec 4 09:59:12 SAST 2020 +### Version: V4.2020.12.2187 +### Updated: Fri Dec 4 11:03:33 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 701efaf43..3bd4c7e2e 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 1; +35.193.184.18 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index d99c8cd8c..cc2baf875 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 0; +35.193.184.18 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 882f8d061..7981d4484 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.12.2186 -### Updated: Fri Dec 4 09:59:12 SAST 2020 +### Version: V4.2020.12.2187 +### Updated: Fri Dec 4 11:03:33 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 701efaf43..3bd4c7e2e 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 1; +35.193.184.18 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index d99c8cd8c..cc2baf875 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 0; +35.193.184.18 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 882f8d061..7981d4484 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.12.2186 -### Updated: Fri Dec 4 09:59:12 SAST 2020 +### Version: V4.2020.12.2187 +### Updated: Fri Dec 4 11:03:33 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 701efaf43..3bd4c7e2e 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 1; +35.193.184.18 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 852c5cb39..4e7a334b2 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -140,6 +140,7 @@ "~*(?:\b)Cliqzbot(?:\b|)" 0; "~*(?:\b)Cloud\ mapping(?:\b|)" 0; "~*(?:\b)coccocbot-web(?:\b|)" 0; + "~*(?:\b)CODE87(?:\b|)" 0; "~*(?:\b)Cogentbot(?:\b|)" 0; "~*(?:\b)cognitiveseo(?:\b|)" 0; "~*(?:\b)Collector(?:\b|)" 0; @@ -256,6 +257,7 @@ "~*(?:\b)HybridBot(?:\b|)" 0; "~*(?:\b)Iblog(?:\b|)" 0; "~*(?:\b)IDBot(?:\b|)" 0; + "~*(?:\b)IDBTE4M(?:\b|)" 0; "~*(?:\b)Id-search(?:\b|)" 0; "~*(?:\b)IlseBot(?:\b|)" 0; "~*(?:\b)Image\ Fetch(?:\b|)" 0; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index d99c8cd8c..cc2baf875 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 0; +35.193.184.18 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 882f8d061..7981d4484 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.12.2186 -### Updated: Fri Dec 4 09:59:12 SAST 2020 +### Version: V4.2020.12.2187 +### Updated: Fri Dec 4 11:03:33 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index c8638035f..a40be186b 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.154.182.187 0; 34.122.173.211 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 1f482e530..77107c10b 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.12.2186 -### Updated: Fri Dec 4 09:59:12 SAST 2020 +### Version: V4.2020.12.2187 +### Updated: Fri Dec 4 11:03:33 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index a40be186b..5e80f4199 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.122.173.211 0; +35.193.184.18 0; +34.122.208.80 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 1993cb5a8..6e39a8dee 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.12.2186 -### Updated: Fri Dec 4 09:59:12 SAST 2020 +### Version: V4.2020.12.2187 +### Updated: Fri Dec 4 11:03:33 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 882f8d061..7981d4484 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.12.2186 -### Updated: Fri Dec 4 09:59:12 SAST 2020 +### Version: V4.2020.12.2187 +### Updated: Fri Dec 4 11:03:33 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index aab6db649..3bd8bebbb 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST +35.193.184.18 0; 34.122.173.211 0; 104.154.182.187 0; -34.122.208.80 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 882f8d061..7981d4484 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.12.2186 -### Updated: Fri Dec 4 09:59:12 SAST 2020 +### Version: V4.2020.12.2187 +### Updated: Fri Dec 4 11:03:33 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 180bdc087..3bd4c7e2e 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 1; +35.193.184.18 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/blacklist-user-agents.conf b/.dev-tools/test_units/blacklist-user-agents.conf index 852c5cb39..4e7a334b2 100644 --- a/.dev-tools/test_units/blacklist-user-agents.conf +++ b/.dev-tools/test_units/blacklist-user-agents.conf @@ -140,6 +140,7 @@ "~*(?:\b)Cliqzbot(?:\b|)" 0; "~*(?:\b)Cloud\ mapping(?:\b|)" 0; "~*(?:\b)coccocbot-web(?:\b|)" 0; + "~*(?:\b)CODE87(?:\b|)" 0; "~*(?:\b)Cogentbot(?:\b|)" 0; "~*(?:\b)cognitiveseo(?:\b|)" 0; "~*(?:\b)Collector(?:\b|)" 0; @@ -256,6 +257,7 @@ "~*(?:\b)HybridBot(?:\b|)" 0; "~*(?:\b)Iblog(?:\b|)" 0; "~*(?:\b)IDBot(?:\b|)" 0; + "~*(?:\b)IDBTE4M(?:\b|)" 0; "~*(?:\b)Id-search(?:\b|)" 0; "~*(?:\b)IlseBot(?:\b|)" 0; "~*(?:\b)Image\ Fetch(?:\b|)" 0; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 77f0409d8..c5a49bfe9 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Abonti -autoemailspider -Buddy -Dotbot -HTMLparser -Pump -Screaming -Sqworm -WebGo IS -WWW-Collector-E +Asterias +Backlink-Ceck +EasyDL +EMail Wolf +GermCrawler +Humanlinks +Meanpathbot +Rogerbot +SemrushBot +T0PHackTeam diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 76e113a7b..c68580130 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,75 +1,73 @@ -Aboundex +360Spider +404checker +80legs +Abonti Acunetix ADmantX -AfD-Verbotsverfahren AhrefsBot -AIBOT +AiHitBot Alligator -AlphaBot -Anarchie Ankit -archive.org_bot -Attach +Anthill +Apexoo +arquivo-web-crawler +Aspiegel +Asterias AwarioSmartBot -Backlink-Ceck +backlink-check BacklinkCrawler BackStreet BackWeb +Badass BatchFTP -Battleztar Bazinga BDCbot +BDFetch +BetaBot Bigfoot -Bitacle Blackboard -Black Hole -BlackWidow -Blow -BlowFish -Brandprotect -BuiltWith +Boardreader +Bolt +Buddy +BuiltBotTough Bullseye -CazoodleBot -CCBot -CherryPicker +Calculon +CATExplorador +check1.exe CheTeam ChinaClaw -Chlooe -Cliqzbot -Cogentbot +Claritybot Collector -Copier +com.plumanalytics CopyRightCheck Copyscape -Cosmos -Craftbot crawler4j -CyotekWebCopy -DatabaseDriverMysqli +crawler.feedback +crawl.sogou.com +Curious +DBLBot demandbase-bot Demon +Devil Digincore -DIIbot -DomainAppender -DomainCrawler +Discoverybot +Dispatch +DomainSigmaCrawler +Domains Project +domainsproject.org +Drip +DSearch DTS Agent Ebingbong +eCatch Ecxi EirGrabber -EroCrawler -evc-batch -Exabot -ExtLinksBot +Extractor +ExtractorPro EyeNetIE -facebookscraper -FDM -FemtosearchBot -Firefox/7.0 -FlashGet +FHscan Foobot -FyberSpider Fyrebot GermCrawler -Getintent GetRight Gigablast G-i-g-a-b-o-t @@ -77,174 +75,176 @@ Go-Ahead-Got-It GoZilla Go!Zilla Grabber -Grafula GrapeFX -GridBot -GT::WWW +GrapeshotCrawler Haansoft HEADMasterSEO +heritrix Heritrix Hloader -HTTP::Lite +HMView +HTMLparser HTTrack +Humanlinks +HybridBot +Iblog IDBot -Id-search -InterGET -InternetSeer +Image Fetch +Image Sucker +InfoNaviRobot +InfoTekies +instabid internetVista monitor -Iria -isitwp.com -JamesBOT Jbrofuzz -Jetty -Joomla -Jorgee -Jyxobot -Kozmosbot -Lanshanbot -Larbin -LeechFTP -Libwhisker -LieBaoFast +JetCar +JikeSpider +JOC Web Spider +JustView +LeechGet +LexiBot +Lftp Lightspeedsystems -Linkdexbot -LinkextractorPro LinkpadBot -LinkWalker -LinqiaScrapeBot -Lipperhey Spider -Lmspider +LinqiaRSSBot +Lipperhey +Litemage_walker LNSpiderguy lwp-request -lwp-trivial -Magnet -Mail.RU_Bot +LWP::Simple Majestic12 -Majestic SEO -Masscan -Mass Downloader -Meanpathbot +MarkWatch +Mb2345Browser +meanpathbot MeanPath Bot +Mediatoolkitbot +mediawords MegaIndex.ru -MicroMessenger +Metauri +MFC_Tear_Sample +Microsoft Data Access +Microsoft URL Control MIDown tool -MIIxpc Mister PiX +Mojolicious Mr.4x3 -MSIECrawler -muhstik-scan -Musobot -Name Intelligence NearSite -Needle -Net Vampire +Nessus +netEstate NE Crawler +Nettrack Netvibes -Nibbler -NICErsPRO Niki-bot -Nimbostratus +NimbleCrawler +Ninja Nmap NPbot -oBot +Octopus Offline Navigator -PageAnalyzer +OnCrawl +Openfind +OrangeBot +OrangeSpider Page Analyzer -PageScorer Panscient -Pavuk -pcBrowser +PECL::HTTP PeoplePal +PHPCrawl Picscout -PictureFinder +Pimonster +Pi-Monster Pixray -PleaseCrawl -plumanalytics -Pockey +POE-Component-Client-HTTP polaris version -ProPowerBot -ProWebWalker -Psbot -Pump -PyCurl -RankActive -RankActiveLinkBot +Probethenet +PxBroker +QueryN Metasearch +Quick-Crawler RankFlex -Rankivabot +RankingBot2 +RealDownload Reaper -RebelMouse ReGet +Ripper RocketCrawler -s1z.ru +Rogerbot SalesIntelligent +ScanAlert scan.lol -Scrapy +Screaming +Searchestate +SearchmetricsBot Semrush -SemrushBot SEOkicks -SEOkicks-Robot -SEOlyticsCrawler Seomoz SEOprofiler -seoscanners -sexsearcher -Siphon +SEOstats +serpstatbot +Shodan SISTRIX -Siteimprove -Sitevigil -SMTBot -Snoopy -SocialRankIOBot -Sosospider +SiteExplorer +SiteLockSpider +SiteSnagger +SiteSucker +SmartDownload +Snapbot Sottopop -Spammen +SpaceBison SpankBot -Spanner +sp_auditbot SputnikBot -Steeler +Sqlmap +Sqlworm +Sucuri SuperHTTP +Surfbot +SurveyBot Suzuran -Swiftbot -sysscan +Szukacz +T0PHackTeam T8Abot -The Intraformant +Telesoft TheNomad +Thumbor +Titan +Toata Toweyabot Tracemyfile -Trendictionbot -True_Robot +Trendiction Turingos -Twice -Typhoeus -UnisterBot +TurnitinBot Upflow -URLy Warning -URLy.Warning -Vagabondo -VB Project -VCI -VoidEYE +Vacuum +VidibleScraper +Voltron WallpapersHD +WBSearchBot Webalta -WebCollage +WebBandit Web Collage -WEBDAV +WebCopier +WebEnhancer Web Enhancer +Web Fetch WebFuck WebGo IS -WebLeacher +WebImageCollector webmeup-crawler -WebSauger WebsiteExtractor +WebsiteQuester +Website Quester WebStripper Web Sucker WebWhacker +WebZIP +WeSEE Whack -Whacker +Whatweb Widow -WinHTTrack -Wprecon -WWW-Mechanize -x09Mozilla -x22Mozilla -zauba.io -ZumBot +Wotbox +WPScan +WWWOFFLE +Xaldon_WebSpider +Xenu +xpymep1.exe +YoudaoBot +Zermelo +Zitebot diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 0a510071d..87f25734a 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ +360Spider 80legs -Abonti Aboundex -Acunetix -ADmantX -AhrefsBot -AiHitBot +Aboundexbot +AfD-Verbotsverfahren Aipbot -Alexibot -Anarchie Ankit -Apexoo -archive.org_bot +Anthill +arquivo.pt arquivo-web-crawler -Attach -AwarioRssBot -BDCbot +Aspiegel +ASPSeek +BacklinkCrawler +BackWeb BDFetch -Bitacle -Blow +BlackWidow +BLEXBot BlowFish +Boardreader Bolt -Brandwatch -Buddy -BuiltBotTough -Calculon -CATExplorador +BuiltWith +Bullseye +BunnySlippers +BuzzSumo +CCBot +check1.exe CheeseBot -CherryPicker CheTeam -Claritybot -Cliqzbot -Cogentbot -com.plumanalytics -Copier -Craftbot -CrazyWebCrawler +ChinaClaw +Chlooe +Cloud mapping +coccocbot-web +CODE87 +CopyRightCheck +Copyscape +Cosmos +crawler.feedback +Curious Custo CyotekWebCopy +DataCha0s +DBLBot demandbase-bot -Demon +Deusu Digincore -DIIbot -Disco +Dirbuster Discobot +DittoSpyder DomainAppender -Domains Project -DomainStatsBot -Dragonfly +DomainCrawler +domainsproject.org +Download Wonder Drip DSearch -DTS Agent Ebingbong -ECCP/1.0 -EMail Wolf -EroCrawler +eCatch +EirGrabber +evc-batch Evil Exabot +Express WebPictures ExtLinksBot -Extractor +Extreme Picture Finder Ezooms -FemtosearchBot -Fimap +facebookscraper +FlashGet Flunky Foobot +Freeuploader FrontPage +FyberSpider +Genieo GermCrawler Gigablast -G-i-g-a-b-o-t Go-Ahead-Got-It -Gotit -GoZilla -Go!Zilla -GT::WWW -heritrix -HMView -HTTP::Lite -HTTrack -HybridBot -Id-search -Image Fetch +Grabber +GrabNet +Grafula +GrapeFX +GrapeshotCrawler +GridBot +HaosouSpider +HEADMasterSEO +HTMLparser +IDBot +IDBTE4M +Image Sucker IndeedBot Indy Library -instabid +InfoNaviRobot +InfoTekies Intelliseek -Internet Ninja -InternetSeer -Iskanie -IstellaBot -JetCar +internetVista monitor +ips-agent +JennyBot JikeSpider JOC Web Spider -Joomla Jorgee -JustView Kenjin Spider -Kinza +Keyword Density LexiBot -Libwhisker -Lightspeedsystems -Likse -LinkextractorPro -LinkScan -LinqiaRSSBot -LinqiaScrapeBot -Lipperhey +Lftp +LibWeb +LinkpadBot +LinqiaMetadataDownloaderBot +Lipperhey Spider Litemage_walker +lwp-request +LWP::Simple lwp-trivial -MarkWatch +Magnet +magpie-crawler +Mail.RU_Bot +Majestic12 +Majestic-SEO +MarkMonitor +Mass Downloader Mata Hari -meanpathbot -Meanpathbot +MauiBot MeanPath Bot -MegaIndex.ru Metauri -MIDown tool -MJ12bot -Morfeus Fucking Scanner -MSFrontPage -Msrabot -Musobot -Name Intelligence +MicroMessenger +Microsoft URL Control +Mister PiX +Mojeek +Mozlila +MQQBrowser +Mr.4x3 +MSIECrawler Navroad -Needle -Nessus +NearSite NetAnts netEstate NE Crawler +NetLyzer +NetMechanic NetSpider Nettrack -Net Vampire +Netvibes NetZIP NextGenSearchBot Niki-bot +Nikto +NimbleCrawler NPbot -Offline Navigator -Openfind -OpenVAS +Nutch +Octopus +OpenLinkProfiler OPPO A33 OutfoxBot -PageScorer -Pandalytics -Papa Foto -PECL::HTTP +PageAnalyzer +PageGrabber +Panscient PHPCrawl PictureFinder -Pixray +Pimonster Pockey POE-Component-Client-HTTP -Pump -PyCurl +polaris version +ProPowerBot QueryN Metasearch -RankFlex -RankingBot -RankingBot2 -Rankivabot -RankurBot -RebelMouse -RedesScrapy +RankActive +Recorder RepoMonkey +s1z.ru SalesIntelligent -ScanAlert +satoristudio.net Scanbot -scan.lol -ScoutJet Searchestate SearchmetricsBot -SEOkicks +Semrush +SemrushBot +SEOlyticsCrawler Seomoz -SEOprofiler -SeoSiteCheckup +seoscanners SEOstats -Siphon -SISTRIX Sitebeam -SiteCheckerBotCrawler sitechecker.pro -SiteExplorer -Siteimprove SiteLockSpider -SMTBot -Snake +SmartDownload Snapbot +Sociscraper Sogou web spider Sottopop +SpankBot Spanner -Spbot +sp_auditbot +SputnikBot spyfu Sqworm -Steeler -Szukacz -T0PHackTeam -Teleport +Stripper +Sucuri +SuperBot +SurveyBot +Swiftbot +T8Abot +tAkeOut TeleportPro +Telesoft Telesphoreo -Telesphorep TightTwatBot -Titan -Toweyabot -Tracemyfile -trendiction.com -trendiction.de +Toata True_Robot +Turingos +Turnitin +TurnitinBot TwengaBot +Twice +Typhoeus +Upflow URLy Warning -Vacuum +URLy.Warning Vagabondo VB Project -VCI -VeriCiteCrawler Virusdie Voil -Voltron Wallpapers/3.0 WallpapersHD WASALive-Bot WebAuto -Web Auto -WebBandit -Web Collage -Web Enhancer +WebCollage +WebCopier +WEBDAV +Web Fetch WebFuck -webmeup-crawler +Web Fuck +WebGo IS +WebmasterWorldForumBot Web Pix -WebSauger +WebReaper +Webshag WebsiteExtractor Webster -WebSucker -Web Sucker WebWhacker +WebZIP WeSEE -Whack Whacker -Whatweb Who.is Bot -WinHTTrack WISENutbot Wonderbot -Woobot -Wotbox -Wprecon +WPScan WWW-Collector-E WWW::Mechanize -Xaldon WebSpider -Xenu +x09Mozilla +Xaldon_WebSpider YoudaoBot +Zade Zauba -zauba.io Zeus -zgrab +Zitebot ZmEu diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index 0edb86bae..9816e477f 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -5000-cotydzien.com -bonky.biz -compliance-checker.info -compliance-mark.top -compliance-walter.top -drunkmoms.net -prokotov.com -searchtooknow-a.akamaihd.net -skachat-besplatno-obrazcy.ru -timecrimea.ru +business-suggested.tk +edelstahlschornstein-123.de +histock.info +idealtits.net +luxmagazine.cf +machicon-akihabara.info +promgirldresses.xyz +shop.acim.org +writersgroup580.web.fc2.com +xxxtube69.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index e41631e58..23cbe115d 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -10bet.com +0912701309f8ce.com 11235813.webzdarma.cz -1-free-share-buttons.com -1xbet4you.com -20pascals.nl -2girls1cup-free.com -5000-cotydzien.com -51.la -66cpwgln.space -777-club.ru -7xc4n.com -aa81bf391151884adfa3dd677e41f94be1.com -aac52006ec82a24e08b665f4db2b5013f7.com -abbanreddy.com -academiacsmendoza.org -acads.net -adclickservice.com -adelly.bg -adservme.com -affiliate-fr.com -affrh2015.com -a.frcls.fr -airmaxshoes-2016.com -alert-fjg.xyz -aliexpress.com -allnews24.in -allornamenti.com -alphacarolinas.com -amanda-porn.ga -amateurgalls.com -ameblo.top -amung.us -anabolics.shop -ananumous.ru -animal36.com -apelsinnik.website -apiluckyleapnet-a.akamaihd.net -apioutoboxnet-a.akamaihd.net -apiwebwebgetcom-a.akamaihd.net -apps-analytics.net -aramaicmedia.org -arenda-avtoprokat-krasnodar.ru +123kuma.com +123locker.com +1webmaster.ml +24x7-server-support.site +300richmond.co.nz +4istoshop.com +7a2rnuey1tw9ar.ru +83net.jp +8558l.jobs.net +999webdesign.com +abcdefh.xyz +addblueoff.com.ua +adman.gr +admeasures.com +adultwebhosting.info +alert-jdh.xyz +alfa-medosmotr.ru +alive-ua.com +all-number.com +allsilver925.co.il +allwomen.info +ample-awards-today.us +android-systems.ru +animalia-life.club +apartamentwroclaw.eu +apartment.ru +apisecretsaucebi-a.akamaihd.net +apishops.ru +appartement-stumm.at +apper.de +approvedlocal.co.za +arate.ru +arcadepages.com arkartex.ru -asia-forum.ru -asscenihotosidea.blogspot.com -auspolice.com -auto.rusvile.lt -aviav.org -avtolombard-krasnodar.ru -ayodhya.co -azartniy-bonus.com -azinoofficial777.ru -balitouroffice.com -balkanfarma.ru -bambi.ck.ua -bangla-chat-uk-paralud.ga -bankcrediti.ru -bannerpower.com -belinvestdom.by -berdasovivan.ru -bestadbid.com -best-deals-products.com -bestdraws.com -bestfortraders.com -best-mam.ru -bestmarriages.com -bestofupload.info -bigames.online -bingoporno.com -brainboostingsupplements.org -brandehk.dk -brandov.ru -brk-rti.ru -buckspar.gq -buketeg.xyz -bukleteg.xyz -bum.com.ru -bupropion-sr-150-mg.us -buyk.host.sk -buyviagraa.blogspot.com -buzz-porno.info +arraty.altervista.org +artavenuegardenstudios.com +asmxsatadriverin.aircus.com +autodo.info +avkzarabotok.com +azartclub.org +azazu.ru +b2b-lounge.com +bablonow.ru +bamo.xsl.pt +bang-hotties.com +bankofthewext.com +beauty-b0x.pl +belinka.com.ua +benchmarkcommunications.co.uk +best-seo-software.xyz +bestwebsiteawards.com +bestwrinklecreamnow.com +beztuberkuleza.ru +bigpictures.club +biketank.ga +bitcoinremote.com +bloglag.com +bmsco.co +bmw-ark.ru +boltalko.xyz +bonky.biz +bonvillan.com +bookhome.info +brainsandeggs.blogspot.com +brewdom.ru +brillianty.info +buchananshardware.com +buron.pw +buyessaynow.biz +buytizanidineonline.blogspot.com +buzzsumo.com bvlgariwallet.xyz +callawaygolfoutlet.online +callmd5map.com camdenmemorials.com -candelluxsklep.pl -cardiosport.com.ua -caulderwoodkennels.com -cdncash.net -centre-indigo.org.ua -centrumcoachingu.com -cfjump.com -cfsrating.sonicwall.com -cheapmoncler.xyz -cheatcode-lita12.rhcloud.com -choice-credit.ru +canvas.gloverid.site +cartierbraceletsreplica.pw +cartierbracelet.xyz +cartierlove2u.xyz +cartiertrinity.xyz +cbb1smartlist12.click +cdnanalytics.xyz +chatroulette.life +cheapjerseysap.com +check-host.net +chinese-amezon.com +chiptuninger.com +chocolateslim-original.com +chorus.terakeet.com christianlouboutinoutlet.win -cialis-samples.com -citysecurity.nu -cjs.com.ru -communisave.co.za -compliance-alex.xyz -continent-e.tv +cialgenisrx.com +clickaider.com +compliance-alex.top +compliance-andrew.xyz +compliance-checker-7.info +compliance-don.top +compliance-viktor.top +concordexoticrentals.com +cookie-law-enforcement-aa.xyz +coolingoods.com coolwallpapers-hd.com +coop-gamers.ru coprofam.org -copypaste.traderzplanet.in -cottageofgrace.com -counciltally.com -covadhosting.biz -cs-passion.pl -custom-product-labels.com -dailyrank.net -darkbooks.org -dayibiao.com -day-news.info -dekoration.us -delfin-aqua.com.ua -design-ideas.info -dev78.dioniqlabb.se -dgfitness.co -divci-hry.info -djonwatch.ru +cowblog.fr +crackzplanet.com +craftburg.ru +creatives.adbetclickin.pink +crynet.cc +cryptoswap.biz +daneshetabiat.com +darodar.com +delta-line.men +dengi-pod-zalog-nedvizhimosti.ru +digital-video-processing.com dnepr-avtospar.com.ua -dogsrun.net -dolohen.com -domaineaneblanc.com -domznaniy.ru -dotmass.top -doxysexy.com -drillsaw.ru -droidlook.net -drunkmoms.net -dynainbox.com -easytuningshop.ru +domcran.net +downloader12.ru +download-of-the-warez.blogspot.com +downtuptv.gq +downvids.net +drugstoreforyou.com +drupa.com +dstroy.su +dumpsmania24.com +dyshagi.ru +e-avon.ru ebooknovel.club ecxtracking.com -ekobata.ru -electro-prom.com -elektrozigaretten1.tumblr.com -eralph.tk -erot.co -errorfixing.space -escort-russian.com -eu-cookie-law.blogspot.com -everypony.ru -eywords-monitoring-your-success.com -ez8motelseaworldsandiego.com -ezigarettenshop1.yolasite.com -f1nder.org -filesdatabase.net -findacheaplawyers.com +edelstahlschornstein-123.de +editmedios.com +ed-shop01.ru +ekspertmed.com +elidelcream.weebly.com +elmifarhangi.com +enhand.se +enskedesquashclub.se +eredijovon.com +eshop4u.jp +europeanwatches.ru +evehemming.blogspot.com.au +excaliburfilms.com +exlarseva.webblog.es +exmasters.com +eyelike.com.ua +ezigarettekaufen2.mpbloggar.se +fanrto.com +fashion.stellaconstance.co +fellowshipoftheminds.com +fickenprivat.info +filefilter.weebly.com +fillmewithhappiness.com +financetip.eu +flash4fun.com +flashahead.info flashbannernow.com -flowersbazar.com -fm-upgrade.ru +floridamhca.org f-online.de -footballfarrago.com -forminecrafters.ru -freewlan.info -fx-brokers-review.com +forex.osobye.ru +fotosfotos.eu +foxtechfpv.com +freakycheats.com +frequiry.com +fres-news.com +fr.netlog.com +frustrated-favorable.gq +fun2cell.net +fun-mobi.pl +funtoonez.com fxtips.ru -galaxyflowers.ru -gallictures.com -gdcentre.ru -gemara.com -gepezz.info -germes-trans.com +gallery.rennlist.com +gateway.zscalertwo.net +gcup.ru get-free-social-traffic.com -getresponse.com -girlspicsa.com -goldpanningtools.com -goo.ne.jp -gorabagrata.ru -greatzip.com -gsasearchenginerankerhelp.com -gsou.cf -gtopstats.com +getlaid-xxxhookupdirect.com +gfaq.ru +gifspics.com +go2album.com +gonextmedia.com +goodhumor24.com +goodnightjournal.com +gorodservis.ru +gratuitbaise.com +gsasearchengineranker.pw +guardlink.org guod.me -hacktougroup.ru -hdpixent.com -hdxnxxtube.mobi -heliko.no +gwagka.com +gz2.bbsoldes.fr +handsandlegs.ru +hatdc.org +hawaiisurf.com +hd720kino.ru +hdpixion.com +headpharmacy.com +hermesbracelets.xyz +herokuapp.com hifidesign.ru -histock.info -hit-kino.ru -hiwibyh.bugs3.com -hkladys.com -hmywwogw.bloger.index.hr -hotdl.in -hoterika.com -hotshoppymac.com -howtowhitenteethfast.xyz -hoztorg-opt.ru +himazin.info +hkdiiohi.skyrock.com +hmmm.cz +homedecoguide.info +homedo.fabpage.com +hop.clickbank.net +hornymatches.com +hostnow.men +hotel-mkad.ru +hotgirlhdwallpaper.com +hotloans.ru htmlcorner.com -hundejo.com -hydropump.su -ibmdatamanagement.co -ido3.com -ifmo.ru -ihc2015.info -import-sales.com -incanto.in.ua -infoupdate.org -insta-add.pro -investingclub.ru -iplusbit.blogspot.co.za -irkutsk.online-podarki.com -itag.pw -itservicesthatworkforyou.com -iwantedmoney.com -iweblist.info -ix20.ru -jasonpartington.com -javidol.site -jennyfire.ru +hustoon.over-blog.com +ibb.com.ua +iccornacircri.cf +igrovyeavtomaty777.ru +ilo134uloh.com +imagine-ex.co +imgarit.pw +infostatsvc.com +inmate-locator.us +innodgfdriverhm.aircus.com +iqbazar.ru +isitwp.com +iusstf.org +jacago.com +janerikholst.se +javstock.com +jaxcube.info +jjbabskoe.ru +jobmarket.com.ua +jpcycles.com +juliadiets.com +junglenet-a.akamaihd.net justbcause.com -justdating.online -jwss.cc -kazrent.com -kinostorm.net -komputers-best.ru -konpax.com -krasnodar.zrus.org -kreativperlen.ch -krynica.info -kvartir-remont.biz -kwzf.net -labafydjxa.su -ladsblue.com -ladsup.com -laexotic.com -lafriore.ru -lalalove.ru -lanasshop.ru -lancheck.net -laptoper.net -larose.jb2c.me -layola.biz.tc -linkarena.com -linkredirect.biz -linkrr.com -livefixer.com -liver-chintai.org -live-sexchat.ru -loadingpages.me -lomza.info -ltvperf.com -lyngdalhudterapi.no -macrotek.ru -magazin-pics.ru -make-money-online.com -malls.com -managerpak204.weebly.com -maridan.com.ua -marketland.ml -marmotstore.online -matchpal-a.akamaihd.net -max-p.men -mebel-vstroika.ru -meble-bogart.info -medtherapy.ru -megaapteka.ru -megavolt.net.ua -mirmedinfo.ru -modabutik.ru -morlat.altervista.org -most.gov.iq -mrbitsandbytes.com +kamalsinha.com +kam-dom.ru +kamorel.com +kartiniresto.com +karusel-market.ru +kasino-money.pw +katadhin.co +kazan.zrus.org +keywordsuggest.org +keywordsuggests.com +kidd.reunionwatch.com +knigonosha.net +kolotiloff.ru +komp-pomosch.ru +konkursowo-24.pl +kosynka-games.ru +kredit-pod-zalog-krasnodar.ru +ku6.com +kvartira-sutochno.com +larutti.ru +laudit.ru +lechenie-gemorroya.com +lflash.ru +lightinghomes.net +lion.snapmap.us +liupis.com +llastbuy.ru +lol-smurfs.com +lonfon.xyz +lsp-awak-perikanan.com +machicon-akihabara.info +mackeeper-center.club +mac-shield.com +madot.onlinewebshop.net +mahnwachen-helfen.info +manifestyourmillion.com +mebel-alait.ru +mebel-ekb.com +medicine-4u.org +medkletki.ru +metalonly.info +mex-annushka.ru +militarysale.pro +mister-shop.com +mmofreegames.online +mossmesi.com +mountainstream.ms +mp3films.ru multgo.ru -muz-baza.net +mvpicton.co.uk mycouponizemac.com -mydoctorok.ru -myplaycity.com -nakozhe.com -narco24.me -narkologiya-peterburg.ru -narutonaruto.ru -nastydollars.com -nate.com -news-speaker.com -nfljerseysforsalewholesaler.com -nhl17coins.exblog.jp -nikitabuch.com -nmrk.ru -normalegal.ru +mymercy.info +nac-bearings.ru +news-readers.ru +nextlnk12.com +nikhilbahl.com +niroo.info +northfacestore.online +norththeface.store +novosibirsk.xrus.org +npoet.ru nucia.biz.ly -nuup.info -obuv-kupit.ru -ochistka-stokov.ru -oda.as -oecnhs.info -ok.ru -onblastblog.online -oneshotdate.com -onlainbesplatno.ru -online247.ml -onlineporno.site -online-templatestore.com -onlywoman.org -ons-add.men -openstat.com -optom-deshevo.ru -osnova3.ru -ostrovtaxi.ru -ourtherapy.ru -oz-offers.com -palvira.com.ua -pammik.ru -partner-print.men -partner-stop.men -partybunny.ru -pattersonsweb.com -pcads.ru -pdamods.ru -pdns.cz -pelfind.me -perfectpracticeweb.com -picture-group.com -pictures-and-images.net -piluli.info -pio.polytopesexempt.com -platinumdeals.gr -pochemychka.net -podshipniki-ntn.ru -poems.com.ua -poiskzakona.ru -pojdelo.weebly.com -pokemongooo.ml -pokemon-go-play.online -polybuild.ru -porndroids.com -pornmania.pl -pornoblood.com -porno-chaman.info -pornodojd.ru -pornofeuer.com -pornolook.net -porno-play.net -pornosee.info -pornosexrolik.com +obnal.org +okmedia.sk +olympescort.com +onlyforemont.ru +onlyporno.ru +oralsexfilme.net +orel-reshka.net +ororodnik.goodbb.ru +ostroike.org +page2rss.com +papasdelivery.ru +par-fallen.ga +partnerads.men +partner-trustworthy.men +pateaswing.com +pearlisland.ru +perso.wanadoo.es +photorepair.ru +php-market.ru +picsfair.com +pinwallpaper.xyz +piratecams.com +pixelrz.com +pkr1hand.com +playlott.com +plohaya-kreditnaya-istoriya.ru +pobeiranie.pl +popup-hgd.xyz +popup-jdh.xyz +porndl.org pornoslive.net -pornpost.in -preg.marketingvici.com -priceg.com -productarium.com -prodvigator.ua -proekt-mos.ru -promodj.com -purplestats.com -putevka24.ru -qpypcx.com -razleton.com -redirectme.net -redirect.trafficreceiver.club -reeyanaturopathy.com -reklama1.ru -remont-mobile-phones.ru -remote-dba.de -reward-survey.net -rfid-locker.co -rn-to-bsn.com -rockprogblog.com -rospromtest.ru -rusexy.xyz -rus-pornuha.com -rusprostitute.com -rus-teh.narod.ru -sabizonline.com -samara.rosfirm.ru -saratov.xrus.org -sax-sex.com -sayyoethe.blogspot.co.za -sbtdesign.co.uk -scalerite.co.za -scanner-elena.top -scmor.ilxc.cc +pornstartits.xblog.in +potoideas.us +pozdravleniya-c.ru +pr0fit-b0x.com +pricheski-video.com +priora-2.com +privacylocationforloc.com +privat-girl.net +privatov-zapisi.ru +prohoster.info +pskcijdc.bloger.index.hr +puzo2arbuza.ru +qor360.com +rada.ru +rankingchart.de +ranksonic.info +razorweb-a.akamaihd.net +realresultslist.com +real-time-analytics.com +redhotfreebies.co.uk +remont-komputerov-notebook.ru +rentehno.ru +repeatlogo.co.uk +rewardpoll.com +rezeptiblud.ru +robertefuller.com +roleforum.ru +rukino.org +ryetaw.com +sadaholding.com +santechnik.jimdo.com +sardinie.us +saugatuck.com +savingsslider-a.akamaihd.net +sbetodiodnye-lampy.ru +scanner-ivan.top +scanner-john.top +scanner-julia.top +scanner-olga.top +screentoolkit.com sdelai-prosto.ru -search-goo.com -sendearnings.com -seocheckupx.com -sexblog.pw -sexphoto.site -sexvideo-sex.com -sexvporno.ru -sexystrippe.info -shoesonlinebuy.xyz -shop.xz618.com +sdi-pme.com +searchlock.com +searchmywindow-a.akamaihd.net +secretscook.ru +seeingmeerkat.com +seemoreresultshu-a.akamaihd.net +semaltmedia.com +seojokes.net +seomarketings.online +serialsway.ucoz.ru +serving.adbetclickin.pink +sexpornotales.com +sexpornotales.net +sexspornotub.com +sextracker.be +sharebutton.net +sharebutton.org +share-buttons-for-free.com +sharpchallenge.com +shiksabd.com +shmetall.com.ua +shopcheermakeup.info +sicfor.bcu.cc +sierraapps.com +siha.de sindragosa.comxa.com -sitechecker.pro -siteripz.net -skyway24.ru -smailik.org -social-buttons.com -social-button.xyz -socialbutton.xyz -socialsignals24.com -soc-proof.su -sodexo.com -sovinsteel.ru -spb.afora.ru -spb-plitka.ru +sisiynas.ru +sitopreferito.it +skypasss.com +slkrm.ru +slujbauborki.ru +socialmonkee.com +soundfrost.org spb.ru -sprttrack.com -spywareit.com -stal-rulon.ru -steelmaster.lv -sterva.cc -stop-gepatit.te.ua -stpicks.com +special-porn.com +spl63.fr +spy-sts.com +srgwebmail.nl +stard.shop +statoutlook.info +staynplay.net +steebook.com +stockquotes.wooeb.com +stoki.ru +strana-solnca.ru +stroimajor.ru stroiminsk.com -strongsignal-a.akamaihd.net -suture.co -svnuppsalaorebro.se -swsociety.se -symphonyintegratedhealthcare.com -szucs.ru -taihouse.ru -tattooreligion.ru -technika-remont.ru -tedxrj.com -telefonsex-ohne0900.net -thepantonpractice.co.uk -tiens2010.ru -titan-cloud.life -tizanidineandhydrocodone.blogspot.com -tizanidinedruginteractions.blogspot.com -tizanidineduringpregnancy.blogspot.com -tizanidinesideeffectsweightloss.blogspot.com -tkanorganizma.ru -topcar-krasnodar.ru -topsy.com -torrent-to-magnet.com -tracksz.co -trailer.cinemaflix.website -trustaffs.com -tyumen.xrus.org -udsgame.online -uhdtv.website -ukrobstep.com -ultimatesetnewfreeallsoftupgradesystems.pw -um-razum.ru -unrealcommander.org -url2image.com -usswrite.com -ustion.ru -uvozdeckych.info -valoresito.com -valueclick.com -vchulkah.net -vclicks.net -vduplo.ru -vesnatehno.ru -vgoloveboli.net -via-gra.webstarts.com +stroy-portal22.ru +styro.ru +sugarlyflex.pw +susanholtphotography.com +svarbit.com +svetlotorg.ru +symbaloo.com +tacbibirfa.tk +taqplayer.info +taraz.xkaz.org +td-l-market.ru +techart24.com +tedy.su +telefonsexsofort.tk +theautoprofit.ml +thenetinfo.com +thetattoohut.com +the-trader.net +thewebsitetemplate.info +thfox.com +timecrimea.ru +tizanidineandcipro.blogspot.com +tizanidineinteractions.blogspot.com +tizanidinestreetprice.blogspot.com +trackzapper.com +traderzplanet.in +trafficgenius.xyz +trahic.ru +tripper.de +trustedhealthtips.com +tuckpointingmasonrysystems.com +tvand.ru +twincitiescarservice.com +txxx.com +tzritel.tk +ufa.xrus.org +uggbootsoutletsale.us +uhodzalijami.ru +unitexindia.com +uptime-delta.net +uptime-gamma.net +userequip.com +ventelnos.com viagroid.ru -video-camer.com -videochat.bz -videosbox.ru -video-woman.com -vltai.com -vseigru.one -vtc.pw -w7s.ru -wait3sec.org -walkme.com +video-hollywood.ru +video-production.com +vietimgy.pw +vkontarkte.com +volgograd.xrus.org +voloslove.ru +vpnhowto.info +vroze.com +vtcdns.com +vvon.co.uk +warningwar.ru +watsonrealtycorp.com webenlace.com.ar -websiteaccountant.de -website-analyzer.info -wemarketing.se -westvilletowingservices.co.za +webshoppermac.com +weburok.com +westermarkanjou.se wheelchairliftsarea.com -wieseversa.no -wpsecurity.website -wpthemedetector.co.uk -wsgames.ru -xchangetrak.com -xjlottery.com -xmronta.com -xn----7sbaaabaei0cc8aj5bj0bncejx.xn--p1ai -xn----7sbbagbq7bd5aheftfllo4m.xn--p1ai -xn--80aaaks3bbhabgbigamdr2h.xn--p1ai -xn--80aikhbrhr.net -xn--80ak6aa92e.com -xn--hxazdsfy.blogspot.com -xn----itbkqkfiq.xn--p1ai -yellocloud.be -yopoint.in -yun56.co -yuweng.info -zaloro.com -zdesoboi.com -zeg-distribution.com -zeroredirect9.com -zeroredirect.com -zhacker.net -zhuravlev.info -zixizop.net.ru -zlatnajesen.com -znakomstva-piter78.ru -zoogdinsney.com -zs2vm.top -zskdla.site -zynax.ua +while.cheapwebsitehoster.com +wholesalejerseyscheapjerseys.us.com +whos.amung.us +whosonmyserver.com +wleuaprpxuvr.ga +woman-orgasm.ru +womens-journal.net +wordpress-crew.net +xedserver.com +xn--24-glceagatoq7c2a6ioc.xn--p1ai +xn--80aafb2a.xn--p1ai +xn--80ahvj9e.xn--p1ai +x-rates.ru +xxxnatelefon.ru +yebocasino.co.za +yougotanewdomain.com +youjizz.com +your-bearings.com +yourserverisdown.com +yuarra.pluto.ro +zambini.ru +zaobao.com.sg +zerocash.msk.ru +zeroredirect7.com +znakomstva-moskva77.ru diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index 9173937cc..602f33900 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -0c47f8422d3f.com -0lovespells0.blogspot.com -178evakuator178.ru -1pamm.ru -24-ak.ru -51.la -55wmz.ru -8si.ru -adminshop.com -adsfresh.men -agahinameh.com -albuteroli.com -amateurlivechat.org -animebox.com.ua -a.pr-cy.ru -arcadeplayhouse.com -aussie-prizes.com -autoua.top -azartmsl.com -baixar-musicas-gratis.com -berrymall.ru -bestbrainprod.win -bestempresas.es -bestkfiledxd.cf -best-mam.ru -betonka.pro -bigcareer.info -blavia.00author.com -bmw.afora.ru -bristolhostel.com -budilneg.xyz -buzzurl.jp -bytimedance.ru -call-of-duty.info -cdnnetwok.xyz -chatroulette.world -chelyabinsk.xrus.org -cialis-samples.com -cialiswithoutadoctor.net -cometorussia.net +00webcams.com +03p.info +4kplayer.pl +83net.jp +aad1f4acb0a373420d9b0c4202d38d94fa.com +acc.eu.org +adtiger.tk +advokateg.xyz +alevinefesleri.com +alfa-img.com +ananumous.ru +andpolice.com +apccargo.com +apilinkswiftco-a.akamaihd.net +applyneedy.xyz +artdestshop.eu +asdfg.pro +asrv-a.akamaihd.net +asseenontvonline.ru +atout-energie-69.com +autobrennero.it +aviav.co +azartmix.com +bamo.xsl.pt +bang-hotties.com +bannerads.de +beachpics.com +beclean-nn.ru +beremenyashka.com +birzha-truda.eu +blacktwhite.com +bouda.kvalitne.cz +brateg.xyz +break-the-chains.com +browsepulse-a.akamaihd.net +brus.city +btc4u.ru +businescoop.men +camdolls.com +camsex.xxx-cam.webcam +carmuffler.net +cayado.snn.gr +cb.iphantom.com +cctva.tv +ceotrk.com +charmstroy.info +chatroulette.si +chidporn.com +cibpenonptib.flu.cc +ciceros.co +classicakuhni.ru +clickintext.net +clicksotrk.com compliance-fred.xyz -dawlenie.com -demian.kz -destinationrealestate.com -dipstar.org -directrev.com +computernetworksonline.com +com-secure.download +constantaservice.net +cornomase.win +cpajunkies.com +crystalslot.com +customerpromos-a.akamaihd.net +czeshop.info +damasarenai.info +detective01.ru +detsky-nabytek.info +dimkino.ru +djstools.com +dktr.ru +docsportal.net domaincrawler.com -earthmagic.info -e-commerce-seo1.com -education-cz.ru -ege-essay.ru -elektrischezigarette2.onsugar.com -elektronischezi.livejournal.com -elmifarhangi.com -elstal.com.pl -eropho.net -eshop4u.jp -esnm.ru -euronis-free.com -evehemming.blogspot.com.au -exonline.info -eywords-monitoring-your-success.com -ezaz.info -ezigaretteshop.webs.com -f012.de -facebook-mobile.xyz -fakehandbags.xyz -fdzone.org -finuse.com -forex-indextop20.ru -forums.toucharcade.com -frbizlist.com +dood.live +doublepimp.com +duplicashapp.com +eandsgallery.com +efnor-ac.com +egovaleo.it +ekn-art.se +elektrischezigarettekaufen2.cowblog.fr +elidelcream.weebly.com +englishgamer.com +eredijovon.com +eurosamodelki.ru +eyessurgery.ru +f00kclan.de +failingmarriege.blogspot.com +falcoware.com +famix.xyz +flashbannernow.com +f-online.de +formseo.com +forpostlock.ru free-fbook-traffic.com -freejabs.com -freetangodownload.com -frequiry.com -frvo.alptandem.ru -fuckmill.com -gambarkatabaru.com -geilehausfrauen.net -getpopunder.com -girlsatgames.ru -glass-msk.ru -gorabagrata.ru -growmyfunds.ca -hdapp1008-a.akamaihd.net -heatpower.ru -hetmanship.xyz -homemature.net -ie.57883.net -iflycapetown.co.za -import-sales.com -indiasourcemart.in -investmac.com -iplogger.org -iptool.xyz -iqupdatetmz.win -irkutsk.online-podarki.com -isistaylorporn.info -islamtoday.co.za -istripper.com -it-max.com.ua -japfm.com -jerseyschinabizwholesale.us -jestr.org -justdating.online -khafre.us +free-share-buttons.blogspot.com +frighteningremain.cf +fxgallery.com +gamblingnerd.com +geckoandfly.com +goldpanningtools.com +googglet.com +green-tea.tv +gribkovye-zabolevaniya.com +hacktougroup.ru +hdxnxxtube.mobi +healmytrauma.info +highland-homes.com +high-speed1.net +histock.info +hol.es +hosting-tracker.com +huhn.altervista.org +ibmdatamanagement.co +ihc2015.info +imfamous.info +insta-add.pro +interferencer.ru +i-service.kz +ix20.ru +janavibekken.no +janettabridal.com +jose.mulinohouse.co +kadashihotel.com +kalb.ru +kazinogames.lv +kicknights.gq kinobest.pl -kino-fun.ru -kinozapas.org -kirov.zrus.org -kiwi237au.tk -kletkimehan.ru -knowyournextmove.com -kongruan.com -kvartira-sutochno.com -lezbiyanki.net -lida-ru.com -linerdrilling.com -lostaruban.com -magicdiet.gq +kmd-pto.ru +konkursowo-24.pl +koronirealestate.gr +landinez.co +leboard.ru +liveinternet.ru +lizace.com +locksmith.jp +lolnex.us +lomb.co +losangeles-ads.com +macrotek.ru +magicplayer-s.acestream.net mail.allnews24.in -marketland.ml -medanestesia.ru -medicine-4u.org -medicovi.com -medkletki.ru -meetingrainstorm.bid -mindbox.co.za -missis.top -mompussy.net -moscow-clining.ru -musicstock.me -mygameplus.ru -myyour.eu -naj-filmy24.pl -naturalbreakthroughsresearch.com -naturtreenspicerx.pw -nootrino.com -novgorod.xrus.org -nyfinance.ml -ochistka-stokov.ru -offf.info -office-windows.ru -offtime.ru -okmusic.jp +mainlinehobby.net +meendo-free-traffic.ga +microstatic.pl +mncrftpcs.com +mojpregled.com +more-letom.ru +musezone.ru +musicdaddy.net +musicktab.com +mwtpludn.review +nate.com +nero-us.com +nextlnk12.com +nfljerseysforsalewholesaler.com +nordvpn.com +nottyu.xyz +of-ireland.info +oil-td.ru +okuos.com +online247.ml online-hit.info -oops-cinema.ru -osagonline.ru -ownshop.win -pandarastore.top -parvezmia.xyz -pearlisland.ru -perederni.net -perfectpracticeweb.com -picturesify.com -pirateday.ru -piter.xrus.org -pizdopletka.club -pl.aasoldes.fr -plaff-go.ru -polcin.de -popads.net -porndroids.com -pornobrazzers.biz -porno-dojki.net -pornoelita.info -princevc.com -promalp-universal.ru -promodj.com -proposal-engine.com -pssucai.info -quick-offer.com +o-o-6-o-o.ru +palocco.it +parajumpersstore.online +percin.biz.ly +play-movie.pl +poiskzakona.ru +polimga.pw +pts163.ru +push-ad.com +qexyfu.bugs3.com +quelle.ru qwarckoine.com -randalljhoward.com -razleton.com -redbottomheels.xyz -remont-ustanovka-tehniki.ru -rent2spb.ru -resant.ru -robot-forex.biz -rogervivierforsale.com -rusexy.xyz -russian-postindex.ru -saneyes.com -sfj-ror.no -shikiso.info -sierraapps.com -sinel.info -site-auditor.online -siteexpress.co.il -skladvaz.ru -small-games.biz -smartbalanceworld.com -socialbutton.xyz -soda.media -sports-supplements.us -sprttrack.com -stairliftsarea.com -standardchartered-forex.com -start.myplaycity.com -stretchmate.net -superstats.com -taboola.com -template-kid.com -templates.franklinfire.co -testingads.pro -thenetinfo.com -tizanidine4mgstreetprice.blogspot.com -tizanidinehcl4mgisitanarcotic.blogspot.com -tizanidineinteractions.blogspot.com -tizanidineorflexeril.blogspot.com -tizanidinesideeffectsweightloss.blogspot.com -tmtrck.com +reeyanaturopathy.com +refudiatethissarah.info +reimageplus.com +reklamuss.ru +ric.info +robertefuller.com +rollercoin.com +rozalli.com +santasgift.ml +sarf3omlat.com +saveindex.xyz +scanner-brian.top +scanner-don.top +seminarygeorgia59.ga +seoanalyses.com +seo-tools-optimizing.com +sexs-foto.top +signx.info +sms2x2.ru +snyatie-lomki-v-stacionare.ru +social-s-iii.xyz +souvenirua.com +space2019.top +sribno.net +staff.prairiesouth.ca +storehouse.ua +stromerrealty.com +studworks.org +sugarkun.com +supervesti.ru +sureone.pro +suture.co +svolze.com +thegalerie.eu +thetoiletpaper.com +tizanidine4mgstreetvalue.blogspot.com tn811.us -tnaionline.org -totu.us -trahic.ru -trk-4.net -tvoystartup.ru -vereo.eu -viagra.pp.ua -vysigy.su -warezkeeper.com -warning.or.kr -webadvance.club -webpromotion.ae -webtherapy.ru -worst-sites.online -writersgroup580.web.fc2.com -wygraj-teraz.com -xerox-douglas.cf -xlovecam.com -xmlinde.com -xn--24-glceagatoq7c2a6ioc.xn--p1ai -xn--80aagddcgkbcqbad7amllnejg6dya.xn--p1ai -xwatt.ru -yatrk.xyz -ynymnwbm.bloger.index.hr -yopoint.in -youbrainboost.asia -youdao.com -yousense.info -yubikk.info -yugo-star.ru -za-music.mymobiplanet.com -zdesformula.ru -zolotoy-lis.ru -zoodrawings.com -zryydi.com +todohr.com +topanasex.com +topdownloads.ru +torrents-tracker.com +track112.site +tramadolandtizanidine.blogspot.com +tula.howotorg.ru +uptime-as.net +v720hd.ru +vbikse.com +venerologiya.com +via-energy-cumpara.com +vkmusics.ru +vladimir.zrus.org +vodaodessa.com +vozbujdenie.com +vriel.batcave.net +wareseeker.com +waterpurifier.club +website-analyzer.info +wedding-salon.net +westen-z.life +whosonmyserver.com +word-vorlagen.xyz +worldis.me +wpthemedetector.co.uk +wsgames.ru +xn----7sbahjd3btneuw1joc.xn--p1ai +xn--b1agm2d.net +xn----ctbbcjd3dbsehgi.xn--p1ai +xz618.com +yellowstonevisitortours.com +yourporngay.com +youthreaders.com +ytmnd.com +zaim-pod-zalog-krasnodar.ru +zawyna.ua +zebradudka.com +zeroredirect12.com +z-master.ru +zskdla.site diff --git a/.dev-tools/test_units/whitelist-ips.conf b/.dev-tools/test_units/whitelist-ips.conf index bb50a4a64..cc2baf875 100644 --- a/.dev-tools/test_units/whitelist-ips.conf +++ b/.dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.173.211 0; +35.193.184.18 0; 127.0.0.1 0; diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index 94aab5108..5a952a956 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index b7772c88b..02b9b7e9b 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index a0d391f7c..1f12a1caa 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2020.12.2186 +#### Version: V4.2020.12.2187 #### Bad Referrers Blocked: [6842](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [581](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [583](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 9ac5b37c7..6837d45af 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2020.12.2186 +#### Version: V4.2020.12.2187 #### Bad Referrers Blocked: [6842](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [581](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [583](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index 721430292..73050691c 100755 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2020.12.2186 +#### Version: V4.2020.12.2187 #### Bad Referrers Blocked: [6842](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [581](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [583](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index 0184ffdfe..01a96f5e0 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,5 +1,3 @@ -IDBTE4M -CODE87 360Spider 404checker 404enemy @@ -81,6 +79,7 @@ Claritybot Cliqzbot Cloud\ mapping coccocbot-web +CODE87 Cogentbot cognitiveseo Collector @@ -197,6 +196,7 @@ Humanlinks HybridBot Iblog IDBot +IDBTE4M Id-search IlseBot Image\ Fetch diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 1e05e1a07..8bf25d310 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2020.12.2186 +#### Version: V4.2020.12.2187 #### Bad Referrers Blocked: [6842](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [581](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [583](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 882f8d061..7981d4484 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.12.2186 -### Updated: Fri Dec 4 09:59:12 SAST 2020 +### Version: V4.2020.12.2187 +### Updated: Fri Dec 4 11:03:33 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index a5914add1..4f32983e7 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.12.2186 -### Updated: Fri Dec 4 09:59:13 SAST 2020 +### Version: V3.2020.12.2187 +### Updated: Fri Dec 4 11:03:34 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 882f8d061..7981d4484 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.12.2186 -### Updated: Fri Dec 4 09:59:12 SAST 2020 +### Version: V4.2020.12.2187 +### Updated: Fri Dec 4 11:03:33 SAST 2020 ### Bad Referrer Count: 6842 -### Bad Bot Count: 581 +### Bad Bot Count: 583 ################################################### ### VERSION INFORMATION ## @@ -228,6 +228,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Cliqzbot(?:\b)" 3; "~*(?:\b)Cloud\ mapping(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)CODE87(?:\b)" 3; "~*(?:\b)Cogentbot(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; "~*(?:\b)Collector(?:\b)" 3; @@ -344,6 +345,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)HybridBot(?:\b)" 3; "~*(?:\b)Iblog(?:\b)" 3; "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)IDBTE4M(?:\b)" 3; "~*(?:\b)Id-search(?:\b)" 3; "~*(?:\b)IlseBot(?:\b)" 3; "~*(?:\b)Image\ Fetch(?:\b)" 3; diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 66215bd04..770e6920c 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2020.12.2186 -### Updated: Fri Dec 4 09:59:14 SAST 2020 -### Bad Bot Count: 581 +### Version: V4.2020.12.2187 +### Updated: Fri Dec 4 11:03:35 SAST 2020 +### Bad Bot Count: 583 ################################################### ### Version Information ## @@ -177,6 +177,8 @@ User-agent: Cloud mapping Disallow:/ User-agent: coccocbot-web Disallow:/ +User-agent: CODE87 +Disallow:/ User-agent: Cogentbot Disallow:/ User-agent: cognitiveseo @@ -409,6 +411,8 @@ User-agent: Iblog Disallow:/ User-agent: IDBot Disallow:/ +User-agent: IDBTE4M +Disallow:/ User-agent: Id-search Disallow:/ User-agent: IlseBot