diff --git a/.dev-tools/_robots_input/robots-input.txt b/.dev-tools/_robots_input/robots-input.txt index 712bb9dfe..91666af48 100644 --- a/.dev-tools/_robots_input/robots-input.txt +++ b/.dev-tools/_robots_input/robots-input.txt @@ -115,6 +115,8 @@ DnyzBot DomainAppender DomainCrawler DomainSigmaCrawler +Domains Project +domainsproject.org DomainStatsBot Dotbot Download Wonder diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 6ba43101a..d73ca912e 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.04.2054 -### Updated: Sat Apr 25 22:39:31 SAST 2020 +### Version: V3.2020.04.2055 +### Updated: Sun Apr 26 10:24:15 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 4a51004bf..38f5c32c1 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.231.58.0 1; +34.73.34.132 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index e288dd266..c7afa9b6e 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.231.58.0 0; +34.73.34.132 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 6ba43101a..d73ca912e 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.04.2054 -### Updated: Sat Apr 25 22:39:31 SAST 2020 +### Version: V3.2020.04.2055 +### Updated: Sun Apr 26 10:24:15 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 4a51004bf..38f5c32c1 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.231.58.0 1; +34.73.34.132 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index e288dd266..c7afa9b6e 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.231.58.0 0; +34.73.34.132 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 6ba43101a..d73ca912e 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.04.2054 -### Updated: Sat Apr 25 22:39:31 SAST 2020 +### Version: V3.2020.04.2055 +### Updated: Sun Apr 26 10:24:15 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 4a51004bf..38f5c32c1 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.231.58.0 1; +34.73.34.132 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf index c4c74fa6e..9fd85da5f 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -176,6 +176,8 @@ "~*(?:\b)DomainAppender(?:\b|)" 0; "~*(?:\b)DomainCrawler(?:\b|)" 0; "~*(?:\b)DomainSigmaCrawler(?:\b|)" 0; + "~*(?:\b)Domains\ Project(?:\b|)" 0; + "~*(?:\b)domainsproject.org(?:\b|)" 0; "~*(?:\b)DomainStatsBot(?:\b|)" 0; "~*(?:\b)Dotbot(?:\b|)" 0; "~*(?:\b)Download\ Wonder(?:\b|)" 0; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index e288dd266..c7afa9b6e 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.231.58.0 0; +34.73.34.132 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 6ba43101a..d73ca912e 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.04.2054 -### Updated: Sat Apr 25 22:39:31 SAST 2020 +### Version: V3.2020.04.2055 +### Updated: Sun Apr 26 10:24:15 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 6ba43101a..d73ca912e 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.04.2054 -### Updated: Sat Apr 25 22:39:31 SAST 2020 +### Version: V3.2020.04.2055 +### Updated: Sun Apr 26 10:24:15 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 9097ea73b..4af9c93f5 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.04.2054 -### Updated: Sat Apr 25 22:39:30 SAST 2020 +### Version: V4.2020.04.2055 +### Updated: Sun Apr 26 10:24:14 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 7f4127dd4..51878438b 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.73.66.97 0; +35.196.82.30 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 9097ea73b..4af9c93f5 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.04.2054 -### Updated: Sat Apr 25 22:39:30 SAST 2020 +### Version: V4.2020.04.2055 +### Updated: Sun Apr 26 10:24:14 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 9097ea73b..4af9c93f5 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.04.2054 -### Updated: Sat Apr 25 22:39:30 SAST 2020 +### Version: V4.2020.04.2055 +### Updated: Sun Apr 26 10:24:14 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 9097ea73b..4af9c93f5 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.04.2054 -### Updated: Sat Apr 25 22:39:30 SAST 2020 +### Version: V4.2020.04.2055 +### Updated: Sun Apr 26 10:24:14 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index c4c74fa6e..9fd85da5f 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -176,6 +176,8 @@ "~*(?:\b)DomainAppender(?:\b|)" 0; "~*(?:\b)DomainCrawler(?:\b|)" 0; "~*(?:\b)DomainSigmaCrawler(?:\b|)" 0; + "~*(?:\b)Domains\ Project(?:\b|)" 0; + "~*(?:\b)domainsproject.org(?:\b|)" 0; "~*(?:\b)DomainStatsBot(?:\b|)" 0; "~*(?:\b)Dotbot(?:\b|)" 0; "~*(?:\b)Download\ Wonder(?:\b|)" 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 9097ea73b..4af9c93f5 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.04.2054 -### Updated: Sat Apr 25 22:39:30 SAST 2020 +### Version: V4.2020.04.2055 +### Updated: Sun Apr 26 10:24:14 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 4105b2ced..48ec7d3f6 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.231.58.0 0; -34.73.66.97 0; +34.74.91.53 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 54ca771bd..b1f6b2122 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.04.2054 -### Updated: Sat Apr 25 22:39:30 SAST 2020 +### Version: V4.2020.04.2055 +### Updated: Sun Apr 26 10:24:14 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index dcd038a5e..5f04cdae8 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.231.58.0 0; -35.227.58.83 0; +34.73.34.132 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index c838c728b..53cc2cc32 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.04.2054 -### Updated: Sat Apr 25 22:39:30 SAST 2020 +### Version: V4.2020.04.2055 +### Updated: Sun Apr 26 10:24:14 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index e166fc4b3..c2491f177 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.74.91.53 0; +35.196.82.30 0; 35.231.58.0 0; -34.73.34.132 0; +35.227.58.83 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 9097ea73b..4af9c93f5 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.04.2054 -### Updated: Sat Apr 25 22:39:30 SAST 2020 +### Version: V4.2020.04.2055 +### Updated: Sun Apr 26 10:24:14 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 8fdda0a49..633442145 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.227.58.83 0; 34.74.91.53 0; -34.73.66.97 0; +35.196.82.30 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 9097ea73b..4af9c93f5 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.04.2054 -### Updated: Sat Apr 25 22:39:30 SAST 2020 +### Version: V4.2020.04.2055 +### Updated: Sun Apr 26 10:24:14 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 4a51004bf..38f5c32c1 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.231.58.0 1; +34.73.34.132 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/blacklist-user-agents.conf b/.dev-tools/test_units/blacklist-user-agents.conf index c4c74fa6e..9fd85da5f 100644 --- a/.dev-tools/test_units/blacklist-user-agents.conf +++ b/.dev-tools/test_units/blacklist-user-agents.conf @@ -176,6 +176,8 @@ "~*(?:\b)DomainAppender(?:\b|)" 0; "~*(?:\b)DomainCrawler(?:\b|)" 0; "~*(?:\b)DomainSigmaCrawler(?:\b|)" 0; + "~*(?:\b)Domains\ Project(?:\b|)" 0; + "~*(?:\b)domainsproject.org(?:\b|)" 0; "~*(?:\b)DomainStatsBot(?:\b|)" 0; "~*(?:\b)Dotbot(?:\b|)" 0; "~*(?:\b)Download\ Wonder(?:\b|)" 0; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index d3251554f..14d0ea504 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Asterias -BackStreet -Internet Ninja -Keyword Density -LeechFTP -Microsoft Data Access -Octopus -OnCrawl -PHPCrawl -VCI +Cosmos +Getintent +NetAnts +Openvas +Shodan +Spanner +Sqworm +TeleportPro +trendiction.de +Vacuum diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index f2f5c321d..b9cbeb792 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -404checker -Abonti -Aboundexbot -ADmantX +360Spider +404enemy AhrefsBot AIBOT +AiHitBot +Alligator +AllSubmitter Anarchie Apexoo -Aspiegel ASPSeek Asterias -Attach +autoemailspider AwarioRssBot BackDoorBot -BackStreet -BackWeb -Barkrowler -BDCbot +Backlink-Ceck +Badass +Battleztar Bazinga BDFetch Bigfoot +Bitacle Blackboard -Blow +Black Hole BlowFish Boardreader +BotALot Brandprotect -BuiltBotTough +Buddy BuiltWith -BunnySlippers -CCBot -Cegbfeieh -CherryPicker -CheTeam -Claritybot +Bullseye +CATExplorador +CazoodleBot Cloud mapping -coccocbot-web -cognitiveseo +Cogentbot com.plumanalytics -CopyRightCheck +Copyscape Cosmos Craftbot -crawler.feedback crawl.sogou.com -CrazyWebCrawler Crescent -CrunchBot +Curious Custo DatabaseDriverMysqli DBLBot -Deusu +demandbase-bot Devil -Discoverybot -DomainAppender -DomainSigmaCrawler +Digincore +DIIbot +Discobot +DittoSpyder +Domains Project DomainStatsBot +Dotbot Download Wonder -Dragonfly -Drip +EasyDL +eCatch ECCP/1.0 Ecxi EirGrabber -EMail Siphon EMail Wolf -Express WebPictures +Evil +ExtLinksBot Extractor -EyeNetIE -facebookscraper +ExtractorPro +Extreme Picture Finder +Ezooms FDM +FHscan +Fimap +Firefox/7.0 FlashGet -FrontPage -FyberSpider +Freeuploader +Fyrebot GalaxyBot -GermCrawler -GetWeb G-i-g-a-b-o-t Gotit GoZilla -Go!Zilla -Grafula -GrapeFX -GT::WWW +GrabNet +GrapeshotCrawler HaosouSpider -Hloader -HTTP::Lite +Heritrix +Humanlinks HybridBot -Id-search -Image Fetch -Image Sucker +IDBot +IndeedBot Indy Library -instabid Intelliseek -Internet Ninja -ips-agent -Iria -Iskanie -JamesBOT -Jbrofuzz +InternetSeer +IRLbot JennyBot JetCar +JOC Web Spider JustView -Jyxobot Keyword Density -Kozmosbot -LeechGet -Likse +LieBaoFast LinkextractorPro -LinkpadBot LinkScan -LinkWalker -LinqiaMetadataDownloaderBot +LinksManager LinqiaRSSBot -LinqiaScrapeBot -Litemage_walker -Ltx71 -lwp-request +LWP::Simple Mag-Net +Mail.RU_Bot Majestic SEO -MarkMonitor +Majestic-SEO MarkWatch -Mata Hari +Mass Downloader +MauiBot Mb2345Browser -Mediatoolkitbot +meanpathbot +Meanpathbot +MeanPath Bot mediawords MegaIndex.ru Metauri -MicroMessenger -Microsoft URL Control -Morfeus Fucking Scanner +MFC_Tear_Sample +Microsoft Data Access +Mister PiX +MJ12bot +Mojeek +Mojolicious Mr.4x3 -MSIECrawler Msrabot +muhstik-scan Musobot Name Intelligence -Nameprotect +NearSite Needle +Nessus +NetAnts Netcraft -NetSpider +netEstate NE Crawler +NetLyzer +NetMechanic +Net Vampire NetZIP +Nibbler +NICErsPRO +Niki-bot NimbleCrawler -Nimbostratus -Ninja Nmap -Nutch +oBot Offline Explorer +Offline Navigator +Openfind OpenLinkProfiler -Openvas +OPPO A33 OrangeBot -OrangeSpider -OutclicksBot -PageAnalyzer +Page Analyzer +PageGrabber page scorer PageScorer -Pandalytics -Papa Foto -PHPCrawl -PictureFinder +Pavuk +Picsearch Pimonster -plumanalytics +Pi-Monster Pockey -POE-Component-Client-HTTP -polaris version +ProPowerBot ProWebWalker -Psbot -PxBroker -QueryN Metasearch +Pump +RankActive RankActiveLinkBot -RankFlex RankingBot -Rankivabot -RankurBot Reaper +Recorder +ReGet +Ripper RocketCrawler RSSingBot -s1z.ru SalesIntelligent -SBIder -ScanAlert Scanbot -scan.lol -ScoutJet -Screaming -Searchestate +Scrapy SEOkicks-Robot SEOlyticsCrawler -SEOprofiler -seoscanners -Shodan -Siphon -Siteimprove +serpstatbot SiteLockSpider SiteSnagger -SiteSucker +Site Sucker Sitevigil SlySearch -Snake Snapbot +Snoopy +Sogou web spider Sosospider -Spammen -Sqlmap +SpankBot +Spbot +Spinn3r +SputnikBot Sqlworm Sqworm Steeler -Stripper Sucker Sucuri +SuperBot SurveyBot +Suzuran Swiftbot -T8Abot +Szukacz +tAkeOut +TeleportPro +Telesoft Telesphoreo -TightTwatBot -Titan +Telesphorep +The Intraformant +Thumbor Toata -Trendiction -trendiction.de +Toweyabot +Tracemyfile +Trendictionbot True_Robot +Turingos Turnitin TurnitinBot -URLy Warning +Twice +UnisterBot +Upflow VeriCiteCrawler +Virusdie Voil -Voltron +Wallpapers/3.0 +WallpapersHD WASALive-Bot -WebCollage -WebEnhancer -Web Enhancer +WBSearchBot +Webalta +WebAuto +WebBandit +WEBDAV WebFuck -WebLeacher -WebmasterWorldForumBot -webmeup-crawler +WebGo IS WebPix -WebReaper +Web Pix WebSauger -Web Sauger -Webster +WebsiteExtractor +WebsiteQuester +WebStripper Web Sucker WebWhacker WebZIP -WeSEE -WiseGuys Robot -Wonderbot +Whack +Whatweb +WISENutbot +Woobot +Wotbox Wprecon -WPScan -WWW-Mechanize +WWW::Mechanize Xaldon_WebSpider -YoudaoBot +xpymep1.exe Zade -zauba.io -Zermelo Zeus -ZmEu +Zitebot ZumBot +ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 3fc1bf531..7c088237e 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -404enemy -80legs -Aboundex -Aboundexbot -AIBOT -Aipbot +360Spider +404checker +ADmantX +AiHitBot Alexibot +Alligator +AllSubmitter AlphaBot -Ankit +Anarchie arquivo.pt -Aspiegel +arquivo-web-crawler autoemailspider -BacklinkCrawler +BackDoorBot +Backlink-Ceck +backlink-check BackStreet +BackWeb Badass -Bandit -Barkrowler +BatchFTP BBBike -BetaBot -Bigfoot -Blackboard +BDCbot +BDFetch Black Hole -BlackWidow -BLEXBot -Boardreader +BlowFish BotALot Brandprotect -Buddy -BuiltWith -BunnySlippers -Cegbfeieh +Brandwatch +BuiltBotTough +BuzzSumo +CazoodleBot CherryPicker +CheTeam Chlooe Claritybot -Cliqzbot -Collector +coccocbot-web com.plumanalytics -Copyscape +CopyRightCheck +crawler4j crawler.feedback -crawl.sogou.com -CSHttp +Crescent +CrunchBot +Custo DatabaseDriverMysqli -DataCha0s demandbase-bot -Demon Deusu Devil DigitalPebble -DIIbot -Dirbuster Discobot DittoSpyder DnyzBot -DomainCrawler +domainsproject.org Dotbot Download Wonder +Drip EasyDL eCatch ECCP/1.0 -EMail Siphon +Ecxi +EMail Wolf EroCrawler -evc-batch -ExtLinksBot -Extractor -ExtractorPro -Extreme Picture Finder -facebookscraper -FDM +Express WebPictures +EyeNetIE +Ezooms +FHscan Fimap +Firefox/7.0 FlashGet -FrontPage -Fyrebot -GalaxyBot -GetRight +Flunky GetWeb -Gigablast -Gigabot -Go-Ahead-Got-It +G-i-g-a-b-o-t Gotit -Go!Zilla +GoZilla GrapeFX -GrapeshotCrawler -GridBot -Havij -HEADMasterSEO -Heritrix Hloader -HMView -HTTP::Lite +HTMLparser HTTrack -IndeedBot +Humanlinks +Iblog +IlseBot +Image Sucker Indy Library +InfoNaviRobot InfoTekies +instabid Intelliseek -InterGET -Internet Ninja -internetVista monitor +InternetSeer +IRLbot Iskanie JamesBOT +Jbrofuzz JetCar -Jetty -Jyxobot +Jorgee +JustView +Kinza +Lanshanbot +LeechFTP LexiBot -LinkScan -LinksManager +LibWeb +Linkdexbot +LinqiaMetadataDownloaderBot +LinqiaRSSBot +Litemage_walker Lmspider Ltx71 LWP::Simple +lwp-trivial +Mag-Net Mail.RU_Bot -Majestic12 -MarkMonitor +Majestic SEO +Majestic-SEO MarkWatch -Mass Downloader Mata Hari -MauiBot -Mb2345Browser -meanpathbot -Meanpathbot Mediatoolkitbot MegaIndex.ru +Metauri +MFC_Tear_Sample +MicroMessenger +Microsoft URL Control MIDown tool Mister PiX -Mojeek -Morfeus Fucking Scanner +MJ12bot +MQQBrowser +Mr.4x3 MSIECrawler -Navroad -Needle -Nessus NetAnts +netEstate NE Crawler +NetMechanic NetSpider -NetZIP +Nettrack +Net Vampire NextGenSearchBot +Nibbler NICErsPRO -Niki-bot -NimbleCrawler -Nimbostratus -NPbot -Offline Explorer +Nikto +Ninja +Nmap +Nutch +Octopus OnCrawl -Openfind -OpenLinkProfiler Openvas -OPPO A33 -OrangeSpider +OpenVAS OutclicksBot OutfoxBot PageAnalyzer -Page Analyzer PageGrabber +page scorer PageScorer -Papa Foto -PECL::HTTP +Panscient PeoplePal -Picsearch -Pimonster +PHPCrawl +Picscout PleaseCrawl -Pockey -polaris version -PyCurl -RankActive +Probethenet +ProPowerBot +ProWebWalker +Psbot +Pump +PxBroker +QueryN Metasearch RankActiveLinkBot -Rankivabot -RankurBot +RankFlex +RealDownload +RedesScrapy ReGet -RepoMonkey -Rogerbot -SBIder +Ripper +RocketCrawler +ScanAlert +scan.lol Scrapy Screaming -Searchestate -SEOlyticsCrawler -Seomoz +ScreenerBot +SearchmetricsBot +SEOkicks SEOprofiler -seoscanners SeoSiteCheckup -SEOstats Shodan -Siteimprove -SiteSucker +SISTRIX +SiteExplorer +SiteSnagger Sitevigil -SmartDownload -Snake +Snapbot Snoopy SocialRankIOBot -sogouspider -Spammen -sp_auditbot +Sottopop +SpankBot +Spanner +Spbot Spinn3r spyfu Sqlmap -Sqworm -Steeler Stripper +Sucker +Sucuri +SuperBot +SuperHTTP Surfbot -SurveyBot Suzuran Swiftbot +T8Abot tAkeOut -Telesoft +TeleportPro Telesphoreo The Intraformant -Titan +TheNomad Toata Tracemyfile Trendiction -Trendictionbot +trendiction.com trendiction.de -True_Robot -Turingos Turnitin -TurnitinBot +TwengaBot +Twice Upflow Vacuum -VCI -Virusdie -VoidEYE -Voil -WallpapersHD -WBSearchBot -WebAuto -WebCollage +Voltron +WASALive-Bot +Webalta +Web Collage +WEBDAV Web Enhancer +WebFetch Web Fuck +WebGo IS +WebImageCollector WebLeacher -WebmasterWorldForumBot -webmeup-crawler -WebSauger -Webshag +Web Pix +Web Sauger WebsiteQuester -Website Quester +WebStripper +WebSucker Web Sucker -WebZIP WeSEE Whack -Whacker Who.is Bot -Widow -Woobot -WPScan +WinHTTrack +WiseGuys Robot +WISENutbot +Wotbox +WWW-Collector-E WWW-Mechanize -WWW::Mechanize WWWOFFLE x22Mozilla -xpymep1.exe -YoudaoBot -Zauba +Xaldon_WebSpider +Xenu zauba.io -Zermelo +Zeus zgrab +Zitebot +ZmEu ZumBot diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index dd44f950e..b4e340618 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -101flag.ru -buzzsumo.com -cubs-jerseys.us -ez8motelseaworldsandiego.com -gaming-journal.com -goodwinmetals.co -park.above.com -scanner-viktor.top -speeddream.xyz -wosik-dach.service-for-web.de +den-noch24.ru +ertelecom.ru +exoclick.com +fillmewithhappiness.com +headpharmacy.com +kinomaniatv.pl +michaelkorssaleoutletonline.net +shtaketniki.kz +stroicol.net +teslathemes.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index f183b3e13..8880e1e49 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -00go.com -1001watch.com.ua -101billion.com -123kuma.com -12bet.com -1688.com -1millionusd.xyz -24videos.tv -256bit.by -3xforum.ro -4replicawatch.net -4webmasters.com -51.la -7search.com -7wind.ru -8558l.jobs.net -991mostfm.co.id -aa14ab57a3339c4064bd9ae6fad7495b5f.com -absolute-s.ru -acgs.tk -activepr.ru -adelly.bg -adexprts.com -adminshop.com -adrenalin-stk.ru -adtech.fr -advancedmassagebysara.com -advertex.info +000free.us +10bet.com +123cha.com +5u.com +72-news.com +7xc4n.com +7yue.org +aanapa.ru +abbanreddy.com +actices.com +addbin.men +addray.pro +adloads.com +adservme.com +adultgalls.com +adverster.com +advertisingtag.net +affordablewebsitesandmobileapps.com +agreda.pluto.ro +ahrefs.com aihelen.net -alekseevec.ru -alfa-img.com -algarveglobal.com -am15.net -amanda-porn.ga -amehdaily.com -animalrank.com -anime.dougasouko.com -antons-transporte.de -apiholdingmypage-a.akamaihd.net -apimegabrowsebiz-a.akamaihd.net -apiqualitinknet-a.akamaihd.net -apishops.ru -apple.com-cleaner.systems -arate.ru -architecturebest.com -arewater.com +airlandsea.info +alert-fdm.xyz +algerianembassy.co.in +ampower.me +ananumous.ru +anapa-inns.ru +anticrawler.org +apccargo.com +apibetweenlinesn-a.akamaihd.net +apibrowsesmartne-a.akamaihd.net +app5.letmacworkfaster.world +arcadeplayhouse.com +arkkivoltti.net +arquapetrarca.info artefakct.com artel116.ru -artparquet.ru -arvut.org -asseenontvonline.ru -autoloans.com -automatic-seo.com -aviapanda.ru -avon-novosib.ru -ayeartoforget.com -baditri.com -baikaleminer.com -balkanfarma.ru -banan.tv -basedecor.ru -beachtoday.ru +astrochicks.com +autobudpostach.club +auto.rusvile.lt +ayerbo.xhost.ro +azbukadiets.ru +azte.ch +bablonow.ru +babs.com.ua +barbourjackets.online +bbsoldes.fr +beachpics.com beauby.ru +beauty-clinic.ru beauty-things.com -beetpics.pw -bestbuy.ca -bestchoice.cf -best-coupon-offer.com -bestfortraders.com -bestplacetobuyeriacta.jw.lt +beclean-nn.ru +beget.tech +bestkfiledxd.cf +best-seo-offer.com +betterscooter.com +betwinservice.com biketank.ga -bildsuche.ru -bingo8888.com -bio-optomarket.ru -bloggedporn.com -blogig.org -blogqpot.com -blog.remote-computer.de -blpmovies.com -bluesman.nu +blobar.org +blog-hits.com +blogporn.in +bmwhighperformers.com +bobinoz.com boleznikogi.com -bongacams.com -boom-celebs.com -brainsandeggs.blogspot.com -brandehk.dk -bristolhotel.com.ua -budilneg.xyz -buyantiviralwp.com -buydissertation.net -buyessaynow.biz -buypillsorderonline.com -buzzsumo.com -bvlgariring.xyz -cakesplus.com.au -callawaygolfoutlet.online -camsex.xxx-cam.webcam -cand.jp -candlewooddentalcentre.com.au -carloans.com -cayado.snn.gr +bookhome.info +braintobe.win +brakehawk.com +brianhenry.co +bristolhotel.com +browserprotecter.com +budpost.com.ua +bugof.gq +buzzzg.men +bvlgariwallet.xyz +bycontext.com +c1.onedmp.com +canrole.com +cartierbracelet.xyz +cartierreplica.win +cartiertrinity.xyz +casablancamanor.co.za +castingbank.ru cbb1smartlist12.click -cdn.walkme.com -celejihad.info -cenokos.ru -centre-indigo.org.ua -centrumcoachingu.com -cheapbarbour.online -cheapmoncler.pw -checktext.org -chelnytruck.ru -cherrypointplace.ca -christianlouboutinreplica.win -cineacademy.ru -clarithromycin500mg.com -clean-start.net -clickcash.com -clubfashionista.com -cognitiveseo.com -collegeessay19.blogspot.ru -comissionka.net +cbcseward.com +celebsopics.com +centraletermice.us +chatroulette.online +cheatcode-lita12.rhcloud.com +checkhit.com +chocolateslim-u-srbiji.com +christianlouboutinshoes.xyz +classiquebijoux.ru +clickso.com +club-musics.ru +colehaanoutlet.store +com-onlinesupport.host compliance-brian.top -compliance-checker-7.info -compliance-don.top -compliance-julianna.top -connectingsingles.com -converse.ddsoldes.fr -cookie-law-enforcement-ee.xyz -coolgamechannel.com -coolgramgoods.com -coprofam.org -courtshipgift.com -creams.makeforum.eu -creativehutindia.com -data-mining.tk -datsun-do.com -daydream-studio.ru -dcj-nn.ru -deanmoore.ie -dev33.dioniqlabb.se -dev-seo.blog -diarioaconcagua.com -digitalfaq.com -digital-sale.su -dignitasdata.se -dikx.gdn -dimfour.com -disruptingdinnerparties.com -dostavimvdom.ru -dosug-lux.ru -downloaddy.net -drillsaw.ru -drugs-no-rx.info -drunkenstepfather.com -dvervmoskvu.ru -e2click.com -election.interferencer.ru -electrik-avenue.com -elvenar.com -enews.tech -eropho.net -estibot.com -exchangeit.gq -exchanges-bet.com -excitacao.com -extractorandburner.com -ezigarettekaufen2.blox.pl -ezigarettenshop1.yolasite.com -face.hostingx.eu -factorynightclub.com -falcoware.com -fanpagerobot.com -faptitans.com -farmprofi.net -fdzone.org -fermersovet.ru -fillmewithhappiness.com -fitfloponline.store -fitnesspiks.com -fiuxy.com -fix-website-errors.com -floating-share-buttons.com -florida-tourism.net -forensicpsychiatry.ru -freza-sverlo.ru -front.to -fugarif.ga -fungamelands.com -galaxyflowers.ru -gallerylisting.com -gallictures.com -gamevalue7.weebly.com -gaz-voshod.ru -geoads.com +compliance-ivan.top +concealthyself.com +constantaservice.net +coolbar.pro +cosmetic.donna7753191.ru +covadhosting.biz +custodieva.ru +cyse.tk +dafatiri.com +dbmkfhqk.bloger.index.hr +demian.kz +deplim.com +dev.citetick.com +dnsrsearch.com +domain.webkeyit.com +donvito.unas.cz +dood.live +downloadeer.net +downtuptv.gq +drupa.com +dscaas.website +duplicashapp.com +eas-seo.com +e-biznes.info +ekn-art.se +e-kwiaciarz.pl +elektir.ru +elitesportsadvisor.com +en.home-task.com +eralph.tk +erotische-geschichten-xxl.com +eshop4u.jp +evehemming.blogspot.com.au +excaliburfilms.com +exmasters.com +exoclick.com +expdom.com +expertblog.info +ezigarettekaufen1.hpage.com +ezrvrentals.com +f00kclan.de +familyphysician.ru +famix.xyz +fashion.stellaconstance.co +fealq.com +fearcrow.com +fetlifeblog.com +fettefrauen.net +financehint.eu +findclan.org +fireads.men +firma-legion.ru +fodelsedagspresenter.nu +forex21.ru +forexgb.ru +forex-indextop20.ru +forex.osobye.ru +forodvd.com +forpostlock.ru +fortevidyoze.net +forums.toucharcade.com +frameimage.org +freegamesplay.online +freemaintenancesysforpcandmac.top +freenom.link +freesitetest.com +freeuploader.ml +fsalas.com +gay-file.com +gayxperience.com +gemara.com getdot.ru -getoutofdebtfree.org +get-free-social-traffic.com getresponse.com -gg.zzyjxs.com -gilbertbanda.net -given2.com -gktt.ru -glass-msk.ru -glopages.ru -gne8.com -goldenggames.com -googst2.ru -greatgrace.ru -greatzip.com -greenidesign.co -gribkovye-zabolevaniya.com -gribokstop.com -gsbs.com.ua -gstatey.net -gtopstats.com -gymi.name -hannasolution.ru -hchha.com -hdimagegallery.net -hdseriale.pl -healbio.ru -healmytrauma.info -health-medical-portal.info -hermesbracelets.xyz +get-seo-domain.com +gfaq.ru +gigixo.com +glasof.es +goforexvps.com +gojiberriess.apishops.ru +googlefeud.com +googlemare.com +gotcher.us +gsmtlf.ru +guod.me +habermetre.com +hamacapty.com +hard-porn.mobi +healing-dysplasia.ru +healthcarestore.info +hermesbelts.xyz +hermesbirkinhandbagoutlets.com +hermesreplica.pw +hikesearch.net +hit-kino.ru +hit-men.men +hjaoopoa.top +hmmm.cz +hol.es home-task.com -honyaku.yahoofs.jp -horoshieokna.com -hoste.octopis.com -hotelcrocenzi.sm -hoterika.com -hotxnights.info -house.sieraddns.com -hyhj.info +hspline.com +huimin764128.com +hulfingtonpost.com +hybrid.ru +hydropump.su +hyipmanager.in iccornacircri.cf -idegenvezeto.eu -ideoworld.org -id-forex.com -ido3.com -ihc2015.info -imicrovision.com -immobiliaremassaro.com +ie.57883.net +iflycapetown.co.za +igrovyeavtomaty777.ru impisr.edunsk.ru -infoupdate.org -interior-stickers.ru -internet-apteka.ru -io9.com -ip-guide.com -jackwolfskinoutlet.online -japan-bearings.ru -jerseysbizwholesalecheap.com -jimmychoosale.online -karachev-city.ru -kazan.xrus.org -kchaxton.com -kimcurlrvsms.com -kinky-fetishes.com +indiakino.net +inmate-locator.us +internetartfair.com +intim-uslugi.info +int.search.tb.ask.com +iscblog.info +ivanstroi.ru +iwantmyfreecash.com +iweblist.info +ixora.pro +jennyfire.ru +jerseyswholesalechinalimited.com +jerseywholesalebizchina.com +jobmarket.com.ua +journalhome.com +jyvopys.com +kamen-e.ru +kidd.reunionwatch.com kino2018.club -kinobaks.com -kinosed.net -klikbonus.com -krasivoe-hd.net -krasula.pp.ua -kreativperlen.ch -kunstaktien.info -landinez.co -letmacworkfaster.world -letmacwork.world -lifebyleese.com -lightinghomes.net -lignofix.ua -limads.men -limtu.ifmo.ru +kinogonew.ru +kinohit1.ru +kinotorka.ru +kleine-titten.biz +klin3952.ru +knowsitall.info +kredit-blog.ru +laapp.com +larutti.ru +law-check-nine.xyz +laxdrills.com +ldrtrack.com +lecbter-relationships.ga +lechtaczka.net +letslowbefast.site +lexixxx.com +lezbiyanki.net linerdrilling.com -liveshoppersmac.com -lkbennettoutlet.online -longadventure.com -louboutinshoes.xyz -luckyshop.net.ua -mafcards.ru -magento-crew.net -make-money-online.com -malwareremovalcenter.com -manifestation.betteroffers.review -manipulyator-peterburg.ru -marketingtechniques.info +linkrr.com +lion.conboy.us +lion.snapmap.us +lirunet.ru +localflirtbuddies.com +locustdesign.co +lombia.co +lonfon.xyz +lostaruban.com +lunamedia.co +m1media.net +m3gadownload.pl +mabdoola.blogspot.com +mall.uk +map028.com +markjaybeefractal.com masturbate.co.uk -matematikus.info -mbiologi.ru mearns-tractors.co.uk -meccadumps.net -medispainstitute.com.au +mecash.ru +medtherapy.ru +mega-bony-2017.pl megakino.net -middlerush-a.akamaihd.net -millionare.com -minecraft-neo.ru -mixed-wrestling.ru -mobicover.com.ua -mobilierland.com -modenamebel.ru -moncleroutletonline.xyz -moviemail-online.co.uk -moyakuhnia.ru -mrvideospornogratis.xxx -mrwhite.biz +meget.co.za +message-warning.net +michaelkorssaleoutletonline.net +minet.club +mixx.com +mlf.hordo.win +mmofreegames.online +mobile-appster.ru +mobile.ok.ru +moomi-daeri.com +moscow-clining.ru +motherboard.vice.com +msk-diplomat.com +mtmtv.info +musicas.baixar-musicas-gratis.com musicdaddy.net -my-floor.in.ua -mywallpaper.top +muzaporn.com +my-aladin.com +mybinaryoptionsrobot.com +mzdish.site najaden.no -narco24.me -nfvsz.com -nhl09.ru -nmrk.ru -noclegonline.info -nonews.co -notebook-pro.ru -novatech.vn +naj-filmy24.pl +narkologiya-orel.ru +na-telefon.biz +nationalbreakdown.com +new-apps.ru +nextconseil.com +nippon-bearings.ru +njkmznnb.ru +nlfjjunb5.ru +nl.netlog.com +nodup.ru +no-fuel.org obesidadealgarve.com -oktube.ru +offtopic.biz +oivcvx.website +okmedia.sk onclkads.com -oneshotdate.com -onetravelguides.com -oralsexfilme.net -ossmalta.com -oz-offers.com -paclitor.com -paidonlinesites.com -palocco.it -panchro.co.uk -paydayonlinecom.com -perfectpracticeweb.com -picquery.com -piski.top -pladform.ru -platesauto.com -pl.id-forex.com -poiskzakona.ru +onlinebay.ru +online-x.ru +open-odyssey.org +orel-reshka.net +origin-my.ru +ororodnik.goodbb.ru +ostrovtaxi.ru +owohho.com +ozoz.it +paramountmarble.co.uk +par-fallen.ga +paydayloanslocal.com +photorepair.ru +piluli.info +pinstake.com +plaff-go.ru +podshipniki-ntn.ru +polcin.de popads.net -porn555.com -pornhub-forum.ga -porno-dojki.net -pornogad.com -pornosemki.info -pornowarp.info -positive2b.ru -pourvous.info -powc.r.ca.d.sendibm2.com -privatefx.all4invest.info -prizesbook.online -productarium.com -profitwithalex.info -profolan.pl -progonrumarket.ru -projefrio.com.br -prosmibank.ru -prostitutki-kharkova.org -pussysaga.com -pwwysydh.com -questionmarque.ch -quickcashlimited.com -quick-seeker.com -ragecash.com -razamicroelectronics.com +popugauka.ru +popup.matchmaker.com +pornokorol.com +preparevideosafesystem4unow.site +pricheski-video.com +progress-upakovka.ru +promoheads.com +pron.pro +prostitutki-kiev.org +pshare.biz +pssucai.info +pussyfleet.com +puzo2arbuza.ru +quickloanbank.com +quizzitch.net +raavidesigns.com +randalljhoward.com +rank-checker.online +ranksays.com +rapidsites.pro realnye-otzyvy.info -rebuildermedical.com -refudiatethissarah.info -remontgruzovik.ru -renewablewealth.com -rent2spb.ru -rentehno.ru -restorator-msk.ru -retreatia.com -rfd-split.hr -rockma.se -romhacking.ru -rukino.org -runofilms.ru -ruscopybook.com +redirect.trafficreceiver.club +replica-watch.ru +rethinkwasteni.info +reversing.cc +ribieiendom.no +ringporno.com +rogervivierforsale.com +roofers.org.uk +ros-ctm.ru +ru-dety.ru +russkie-gorki.ru +rus-teh.narod.ru +s8-nowy-wygraj.comli.com +sack.net sad-torg.com.ua -salmonfishingsacramentoriver.com -sandhillsonline.com -saneitconsulting.com -sbdl.no -scanner-jack.top -sdelatmebel.ru -searchengineranker.email -see-your-website-here.com -seocheckupx.com -seoprofiler.com -serviporno.com -servisural.ru -sexblog.pw -sexiporno.net -sexkontakte-seite.com -sexuria.net -sexy-screen-savers.com -sexyteens.hol.es -sexytrend.ru -shopcheermakeup.info -shopperifymac.com -site-auditor.online -smart-balancewheel.com -sneakyboy.com -snts.shell-pmr.ru -soaksoak.ru -soda.media -space2019.top -spbchampionat.ru -spidtest.space -sptslmtrafms.com -spywarespy.com -sr-rekneskap.no -stoki.ru -strag-invest.ru -strana-solnca.ru -stroyhelp-dv.ru -stroymonolit.su -success-seo.com -sukarame.net -support.nopeas.sk -sureone.pro -swiped.su -tabakur77.com -taboola.com -tackletarts.co -talant-factory.ru -tamada69.com -td-33.ru -telefonsexi.com +santasgift.ml +sape.top +sarafangel.ru +sa-rewards.co.za +savefrom.com +scanmarine.info +scanner-barak.top +scanner-willy.top +scoopquest.com +semalt.com +seogadget.ru +seomarketings.online +seo-prof1.xyz +serialsway.ucoz.ru +servingnotice.com +sex-tracker.com +shama-rc.net +shillyourcoins.com +shlyahten.ru +shodanhq.com +shop.xz618.com +sibtest.ru +similardeals.net +siteonomy.com +sklad-24.ru +skyway24.ru +slim.sellany.ru +sms2x2.ru +snip.to +snworks.com +socialbutton.xyz +social-search.me +softlinesolutions.me +spamnuker.com +spb-plitka.ru +squidoo.com +stairliftsarea.com +standardchartered-forex.com +storehouse.ua +strv.se +studiokamyk.com.pl +sumo.com +superinterstitial.com +swinginwithme.ru +takeflyte.com +tanieaukcje.com.pl +tattomedia.com +teastory.co +technopellet.gr +tedy.su +telefonsexkostenlos.tk telemetryverification.net -themeforest.net -tizanidineingredients.blogspot.com -tizanidineonline.blogspot.com -tizanidinestreetprice.blogspot.com -tizanidinewithdrawal.blogspot.com -tk-assortiment.ru -tomck.com -top-study.work -trafaret74.ru -trafficjunky.com +terraclicks.com +thecounter.com +thefotosgratis.eu +theheroes.ru +thexart.club +thruport.com +tivolibasket.it +tizanidinecost.blogspot.com +tizanidinedosageforsleep.blogspot.com +tizanidinesleepaid.blogspot.com +tomatis.gospartner.com +torrentgamer.net +trafficmania.com trafficnetzwerk.de -treasuretrack-a.akamaihd.net -turkeyreport.tk -twu.com.ua -tx41tclega.ru -uasb.ru -uk-zheu20.ru -unisexjewelry.org -universals.com.ua +traffixer.com +triumf-realty.ru +truck-addzilla.life +ttrraacckkrr.com +ttsq.fr +tuberkulezanet.ru +tula.mdverey.ru +txxx.com +uchebavchehii.ru +uchetunet.su +underthesite.com +unece.org +unpredictable.ga +unrealcommander.com uptime-as.net -uptime-gamma.net +urdoot.win +urengoy.pro +urzedowski.eu +usadacha.net us-america.ru -usdx.us -uzungil.com -validdomain.xyz -valoresito.com -vbabule.net -vellings.info -vertaform.com -vesnatehno.ru -vezuviy.su -video-chat.cn -videooko.weebly.com -virtuagirl.com -wdrake.com -webextract.profound.net -webpromotion.ae -webs.com -website-speed-checker.site -websites-reviews.com -wedding0venues.tk -westermarkanjou.se -whatistizanidine2mg.blogspot.com +uvozdeckych.info +vacances-voyages.info +varikozdok.ru +vavilone.com +venerologiya.com +venta-prom.ru +verymes.xyz +via-energy-cumpara.com +viagraneggrx.com +videochat.world +videogamesecrets.com +vielporno.net +villakohlanta.nu +vinylvault.co.uk +visa-pasport.ru +vitalads.net +vitanail.ru +voloslove.ru +vonradio.com +vykup-avto-krasnodar.ru +waysbetter.cn +webix.biz +westvilletowingservices.co.za +whatsappdownload10.com wholesalejerseys-cheapest.com -wholesalenfljerseys.us.com -wildworld.site -wineitudes.wordpress.com -w-journal.ru -word-vorlagen.xyz -wowas31.ucoz.ru +world-mmo.com +writersgroup580.web.fc2.com +wttavern.com wufak.com -xmladserver.com -xn----7sbaaabaei0cc8aj5bj0bncejx.xn--p1ai +wzgyyq.com +x-diesel.com +x-diesel.info +xmronta.com xn----7sbifcamovvfggw9d.xn--p1ai -xn--b1agm2d.net -xn----dtbndd4ae7eub.top -x-rates.ru -xtraffic.plus -xz618.com -yhit.press -youradulthosting.com -yourdesires.ru -yugo-star.ru +xn--80adaggc5bdhlfamsfdij4p7b.xn--p1ai +xn--80aebbcbcdemfkhba4byaehoejh8dza3v.xn--p1ai +xn----btbdvdh4aafrfciljm6k.xn--p1ai +xn----ctbbcjd3dbsehgi.xn--p1ai +xn--e1afanlbnfckd7c3d.xn--p1ai +xwatt.ru +xxxguitars.com +yadro.ru +yeartwit.com +ynymnwbm.bloger.index.hr +youdesigner.kz +yubikk.info zagadki.in.ua -zarabotat-na-sajte.ru -zatjmuzu.info -zed21.net -zeroredirect11.com -zeroredirect8.com -zhuravlev.info +zaloro.com +zeleznobeton.ru +zeroredirect9.com zigarettenonlinekaufen2.drupalgardens.com -zocaparj.kz -zskdla.site +zixizop.net.ru +zrelaya.pw +zvooq.eu diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index d011417ef..6489ea255 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -1kinobig.ru +23kw.ru 3-letter-domains.net -4kpics.rocks -7yue.org -abclauncher.com -advancedcleaner.com -aero2.ru -airmaxshoes-2016.com -albuteroli.com -all4invest.ru -ameblo.jp -amospalla.es -amt-k.ru -apibrowsesmartne-a.akamaihd.net -api.stathat.com -apps-analytics.net -arendovalka.xyz -ascat.porn -ausmepa.org.au -balkanfarma.org -banan.tv -banki76.ru -basedecor.ru -best-way.men -bestwebsiteawards.com -bharatdefencekavach.com -bookhome.info +4k-player.pl +4ureyesonly.com +aa625d84f1587749c1ab011d6f269f7d64.com +acheterviagrafr24.com +adminshop.com +adsref.men +adultfriendfinder.com +advokateg.xyz +adzerg.com +amoi.tn +amyfoxfitness.com +android-systems.ru +apiluckyleapnet-a.akamaihd.net +appartement-stumm.at +approved.su +appsaurus.com +atelielembrancaqueencanta.com.br +aviav.ru +avkzarabotok.com +avon-ukraine.com +babosas.com +balans.shahterworld.org +beremenyashka.com +bestserials.com +betwinservice.com +bigcities.org +bluesalt.co +bombla.org brateg.xyz -buybest1.biz -buy-loft.ru -cakemediahq.com.au -cartujano-pre.de -catalogs-parts.com -cezartabac.ro -chcu.net -cheapjerseysap.com -chocoslim.pro -cinemaenergy-hd.ru -compliance-george.top -compliance-ivan.xyz -compliance-mark.top -coolbar.pro -coprofam.org -countbertwistdisp26.soup.io -countercrazy.com -crest-poloski.ru -danceuniverse.ru -datadepths.com -dataloading.net -data.vtc.pw -daydream-studio.ru -deniven.1bb.ru -dieta-personalna.pl -divci-hry.info -docsportal.net -domain.webkeyit.com -downloadkakaotalk.com +browsepulse-a.akamaihd.net +buchananshardware.com +camdolls.com +candiceloves.us +cardsharp1.ru +carrauterie.be +cartierreplica.pw +cheapbelstaff.online +cibpenonptib.flu.cc +club-lukojl.ru +collegeessay19.blogspot.ru +compliance-brian.xyz +dailyfinancefix.com +dangphoto.trade +darodar.com +dcj-nn.ru +directivepub.com +dms-sw.ru +domcran.net +dostavka-v-krym.com +downloadeer.net +download-of-the-warez.blogspot.com +druzhininevgeniy63.blogspot.com dstroy.su dtnlyss.com -duplicashapp.com -dwomlink.info -eandsgallery.com earncash.com.ua -earthmagic.info -education-cz.ru -elektrischezigarette1.blog.pl -eloxal.ru -eluxer.net -errorfixing.space -excitacion.info -extrabot.com -ez8motelseaworldsandiego.com -farmingworm.com -fba-mexico.com -fefo.gdn -flex4launch.ru +elektrozigaretten1.tumblr.com +erot.co +exchangeit.gq +executehosting.com +extstat.com +facebook-mobile.xyz +familyholiday.ml +familyphysician.ru +fertilitetsradgivningen.se +fikasound.tk floridahuntingfishingadventures.com -footballfarrago.com -forensicpsychiatry.ru -forumprofi.de -forums.toucharcade.com -forum.tvmir.org -frankofficial.ru -freeuploader.ml -fungirlsgames.net -gambarkatabaru.com -gemgrab-a.akamaihd.net -getresponse.com -get-seo-domain.com -globalscam.ga +focalink.com +foodcrafts.website +fotosfotos.eu +fr-bearings.ru +gallerily.com +gambarkataku.co +gamewrath.com +gettpromos.com +gloverid.site +golmau.host.sk +gombita.info googlefeud.com -greatzip.com -guod.me -gwebtools.com -healing-dysplasia.ru -heartofpayne.xyz -helicalpile.us -host-protection.com -hoverboard360.es -ideashome.id -ifirestarter.ru -iflycapetown.co.za -informatiecentro.be -int.search.mywebsearch.com -investyb.com -iradiology.ru -it-max.com.ua -jeremyeaton.co -kakablog.net -kerei.ru -klejonka.info -koleso24.com.ua -kolotiloff.ru -lacapilla.info -legalrc.biz -letto.by -lifehacĸer.com -louboutinshoes.xyz -lovelycraftyhome.com -ltvperf.com -macresource.co.uk -mafcards.ru -magicdiet.gq -map028.com -martlinker.com -mastershef.club -matsdale.com -medicine-4u.org -megahdporno.net -meget.co.za -meltwater.com -monsterdivx.com -moskva.nodup.ru -mrwhite.biz -muabancantho.info -muycerdas.xxx -mxgetcode.com -myhealthcare.com -mylesosibirsk.ru -mypornfree.ru -narco24.me -narkologiya-peterburg.ru -net-profits.xyz -novosibirsk.xrus.org -ntic.fr -offer.wpsecurity.website -oktube.ru -ok-ua.info -onescreen.cc -oneshotdate.com -opinionreelle.com -orenburg-gsm.ru +gotcher.us +greatdealshop.com +gsasearchengineranker.xyz +gwebtools.com.br +hanwei.us +hdapp1008-a.akamaihd.net +helvetia.com.ua +hentai-manga.porn +herokuapp.com +highspeed5.net +hmywwogw.bloger.index.hr +homeandhealth.ru +hostsshop.ru +hotloans.ru +inet-traffic.com +insider.pro +integritylandscapeservices.com +istripper.com +iyasimasennka.com +izhevsk.zrus.org +janavibekken.no +katjimej.blog.fc2.com +kazinogames.lv +ketoanhanoi.info +kinopolet.net +kit-opt.ru +kredit-blog.ru +kung-fu-ru.com +kutikomi.net +laapp.com +lerporn.info +lesbian.xyz +lmrauction.com +loadingpages.me +m0r0zk0-krava.ru +macdamaged.space +mailemedicinals.com +mall.uk +marinetraffic.com +meduza-consult.ru +mistr-x.org +mobile-appster.ru +mp3films.ru +msk-diplomat.com +mulberryoutletonlineeu.com +mymobilemoneypages.com +mypets.by +news-readers.ru +nfljerseyscheapchinabiz.com +ohmyrings.com +oil-td.ru +onlywoman.org +o-o-6-o-o.com +optibuymac.com +oralsexfilme.net +orgasmatrix.com +osnova3.ru ourville.info -oynat.info -paramountmarble.co.uk -partner-host.men -pensplan4u.com -percin.biz.ly -petsblogroll.com -picturesboss.com -picturesfrom.com -pokemon-go-play.online -pornobrazzers.biz -pornotubs.com -productarium.com -promover.org +partner-trustworthy.men +petedrummond.com +phobia.us +phpdevops.com +piccdata.com +picphotos.net +picturesify.com +piter.xrus.org +pixell.club +pizdopletka.club +pontiacsolstice.info +popugauka.ru +predmety.in.ua +preventheadacheguide.info +privacylocationforloc.com +privat-girl.net +prombudpostach.com.ua propranolol40mg.blogspot.com -prostitutki-almata.org -prostitutki-belgoroda.org -prostitutki-rostova.org -prostitutki-tyumeni.org -puzzleweb.ru -py100.ru -rainbowice.ru -randalljhoward.com -realnye-otzyvy.info -renecaovillasale.online -research.ifmo.ru -rewardit.com -ru-dety.ru -runstocks.com -ryetaw.com -salmonfishingsacramentoriver.com -sanatorrii.ru -saneyes.com -seinterface.com -semalt.com -service.adtech.us -serw.clicksor.com -sex-pr.net -sfj-ror.no -shitmovs.com -site5.com -siteexpress.co.il -site-speed-checker.site -skypasss.com -slavkokacunko.de -slimcdn.com -slutloadlive.com -social-buttons.com -solitaire-game.ru -spb.ru -sports-supplements.us -spravka130.ru -stal-rulon.ru -sudexpert66.ru -tandvardshuset.net -telegramdownload10.com -thesmartsearch.net -timetorelax.biz -tizanidinestreetvalue.blogspot.com -toon-families.com -top-karkas.ru -top-study.work -triumf-realty.ru +pro-tec.kz +pshare.biz +puserving.com +quizzitch.net +rankings-analytics.com +ranksignals.com +rasteniya-vs-zombi.ru +recinziireale.com +remontgruzovik.ru +remont-komputerov-notebook.ru +remont-ustanovka-tehniki.ru +ruex.org.ua +rusprostitute.com +rvi.biz +sadaholding.com +salut-camp.ru +saneitconsulting.com +sarf3omlat.com +saveindex.xyz +sbf441.com +scanner-jack.top +scanner-marwin.top +scanner-olga.top +schoolfiles.net +scmor.ilxc.cc +search-goo.com +seocheckupx.com +seorankinglinks.us +sibtest.ru +silverage.ru +sirpornogratis.xxx +skylta.com +sladkoevideo.com +small-game.com +sobecjvuwa.com.ru +socialmonkee.com +socialseet.ru +soc-proof.su +spabali.org +spacash.com +space4update.pw +spinnerco.ca +spywarebegone.com +stairliftstrue.com +stroy-portal22.ru +sumo.com +superlist.biz +surcentro.com +svetlotorg.ru +tattomedia.com +theautoprofit.ml +threecolumnblogger.com +tizanidine4mgstreetvalue.blogspot.com +tizanidineinfo.blogspot.com +tizanidinerestlesslegsyndrome.blogspot.com +tizanidinevssoma.blogspot.com +top-deal.com.pl +tour-line.net +toys.erolove.in +tracklead.net +trafficfactory.biz +tri-slona.org tsyndicate.com -tvand.ru -twinderbella.com -twodollarshows.com -ukkala.xyz -urdoot.win -vapomnoncri.tk -via-gra.webstarts.com -videochat.ph -videochaty.ru -video--production.com -vvon.co.uk -wallpapersdesk.info -watch-movies.ru -website-audit.com.ua -webtherapy.ru -wfb.hatedriveapart.com -woman-orgasm.ru -womens-journal.net -worldis.me -wosik-dach.service-for-web.de -www.888.com -x-diesel.biz -x-lime.net -xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai -xn----9sbebi2bvzr7h.xn--p1ai -xn----dtbndd4ae7eub.top -xsion.net -youdesigner.kz -yourothersite.com -yousense.info -zaimhelp.ru -zaobao.com.sg -znaniyapolza.ru -zryydi.com +twojebook.pl +u-cheats.ru +ufa.xrus.org +uhod-za-sobakoj.ru +umnovocaminho.com +validccseller.com +video-hollywood.ru +vigrx-original.ru +viveresaniesnelli.it +voloo.ru +warezaccess.com +warezkeeper.com +warningzscaler.heraeus.com +waycash.net +webshoppermac.com +weclipart.com +whengirlsgowild.com +wholesalejerseys-cheapest.com +winner7777.net +worldtraveler.world +xchat26.myfreecams.com +xcombear.ru +xev.ru +xn--24-glceagatoq7c2a6ioc.xn--p1ai +xn--b1adccaf1bzj.xn--p1ai +x-true.info +xwatt.ru +yebocasino.co.za +youbrainboost.asia +zaloro.com +zed21.net diff --git a/.dev-tools/test_units/whitelist-ips.conf b/.dev-tools/test_units/whitelist-ips.conf index e288dd266..c7afa9b6e 100644 --- a/.dev-tools/test_units/whitelist-ips.conf +++ b/.dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.231.58.0 0; +34.73.34.132 0; 127.0.0.1 0; diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index 731feebd0..43c0fda7c 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index b2787fba9..5b2bd04bf 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 3c704601c..e90fbc07a 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2020.04.2054 +#### Version: V4.2020.04.2055 #### Bad Referrers Blocked: [6831](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [571](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [573](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index b70d519cc..d24938961 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2020.04.2054 +#### Version: V4.2020.04.2055 #### Bad Referrers Blocked: [6831](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [571](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [573](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index b36f10411..97ff316c1 100755 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2020.04.2054 +#### Version: V4.2020.04.2055 #### Bad Referrers Blocked: [6831](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [571](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [573](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index 076650e32..25c3f153f 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,5 +1,3 @@ -Domains\ Project -domainsproject.org 360Spider 404checker 404enemy @@ -117,6 +115,8 @@ DnyzBot DomainAppender DomainCrawler DomainSigmaCrawler +Domains\ Project +domainsproject.org DomainStatsBot Dotbot Download\ Wonder diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 1581d86ea..6703aa12c 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2020.04.2054 +#### Version: V4.2020.04.2055 #### Bad Referrers Blocked: [6831](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [571](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [573](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 9097ea73b..4af9c93f5 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.04.2054 -### Updated: Sat Apr 25 22:39:30 SAST 2020 +### Version: V4.2020.04.2055 +### Updated: Sun Apr 26 10:24:14 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 6ba43101a..d73ca912e 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.04.2054 -### Updated: Sat Apr 25 22:39:31 SAST 2020 +### Version: V3.2020.04.2055 +### Updated: Sun Apr 26 10:24:15 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 9097ea73b..4af9c93f5 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.04.2054 -### Updated: Sat Apr 25 22:39:30 SAST 2020 +### Version: V4.2020.04.2055 +### Updated: Sun Apr 26 10:24:14 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 571 +### Bad Bot Count: 573 ################################################### ### VERSION INFORMATION ## @@ -264,6 +264,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)DomainAppender(?:\b)" 3; "~*(?:\b)DomainCrawler(?:\b)" 3; "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)Domains\ Project(?:\b)" 3; + "~*(?:\b)domainsproject.org(?:\b)" 3; "~*(?:\b)DomainStatsBot(?:\b)" 3; "~*(?:\b)Dotbot(?:\b)" 3; "~*(?:\b)Download\ Wonder(?:\b)" 3; diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 747c0da74..cd77336c5 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2020.04.2054 -### Updated: Sat Apr 25 22:39:32 SAST 2020 -### Bad Bot Count: 571 +### Version: V4.2020.04.2055 +### Updated: Sun Apr 26 10:24:16 SAST 2020 +### Bad Bot Count: 573 ################################################### ### Version Information ## @@ -249,6 +249,10 @@ User-agent: DomainCrawler Disallow:/ User-agent: DomainSigmaCrawler Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: domainsproject.org +Disallow:/ User-agent: DomainStatsBot Disallow:/ User-agent: Dotbot