diff --git a/.dev-tools/_robots_input/robots-input.txt b/.dev-tools/_robots_input/robots-input.txt index ac886d4c6..a40a851e4 100644 --- a/.dev-tools/_robots_input/robots-input.txt +++ b/.dev-tools/_robots_input/robots-input.txt @@ -209,6 +209,7 @@ internetVista monitor ips-agent Iria IRLbot +isitwp.com Iskanie IstellaBot JamesBOT @@ -389,6 +390,7 @@ Rogerbot RSSingBot s1z.ru SalesIntelligent +satoristudio.net SBIder ScanAlert Scanbot @@ -415,6 +417,8 @@ Shodan Siphon SISTRIX Sitebeam +SiteCheckerBotCrawler +sitechecker.pro SiteExplorer Siteimprove SiteLockSpider diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 7121d5fec..886e6d1ed 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.07.2107 -### Updated: Tue Jul 28 08:53:56 SAST 2020 +### Version: V3.2020.07.2108 +### Updated: Tue Jul 28 09:06:54 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index ad23b3d8a..49090b431 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.227.97.188 1; +34.74.91.53 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 511678455..272faad99 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.227.97.188 0; +34.74.91.53 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 7121d5fec..886e6d1ed 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.07.2107 -### Updated: Tue Jul 28 08:53:56 SAST 2020 +### Version: V3.2020.07.2108 +### Updated: Tue Jul 28 09:06:54 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index ad23b3d8a..49090b431 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.227.97.188 1; +34.74.91.53 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 511678455..272faad99 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.227.97.188 0; +34.74.91.53 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 7121d5fec..886e6d1ed 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.07.2107 -### Updated: Tue Jul 28 08:53:56 SAST 2020 +### Version: V3.2020.07.2108 +### Updated: Tue Jul 28 09:06:54 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index ad23b3d8a..49090b431 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.227.97.188 1; +34.74.91.53 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 972cfbda6..9c1f82a5a 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -270,6 +270,7 @@ "~*(?:\b)ips-agent(?:\b|)" 0; "~*(?:\b)Iria(?:\b|)" 0; "~*(?:\b)IRLbot(?:\b|)" 0; + "~*(?:\b)isitwp.com(?:\b|)" 0; "~*(?:\b)Iskanie(?:\b|)" 0; "~*(?:\b)IstellaBot(?:\b|)" 0; "~*(?:\b)JamesBOT(?:\b|)" 0; @@ -450,6 +451,7 @@ "~*(?:\b)RSSingBot(?:\b|)" 0; "~*(?:\b)s1z.ru(?:\b|)" 0; "~*(?:\b)SalesIntelligent(?:\b|)" 0; + "~*(?:\b)satoristudio.net(?:\b|)" 0; "~*(?:\b)SBIder(?:\b|)" 0; "~*(?:\b)ScanAlert(?:\b|)" 0; "~*(?:\b)Scanbot(?:\b|)" 0; @@ -476,6 +478,8 @@ "~*(?:\b)Siphon(?:\b|)" 0; "~*(?:\b)SISTRIX(?:\b|)" 0; "~*(?:\b)Sitebeam(?:\b|)" 0; + "~*(?:\b)SiteCheckerBotCrawler(?:\b|)" 0; + "~*(?:\b)sitechecker.pro(?:\b|)" 0; "~*(?:\b)SiteExplorer(?:\b|)" 0; "~*(?:\b)Siteimprove(?:\b|)" 0; "~*(?:\b)SiteLockSpider(?:\b|)" 0; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 511678455..272faad99 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.227.97.188 0; +34.74.91.53 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 7121d5fec..886e6d1ed 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.07.2107 -### Updated: Tue Jul 28 08:53:56 SAST 2020 +### Version: V3.2020.07.2108 +### Updated: Tue Jul 28 09:06:54 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 7121d5fec..886e6d1ed 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.07.2107 -### Updated: Tue Jul 28 08:53:56 SAST 2020 +### Version: V3.2020.07.2108 +### Updated: Tue Jul 28 09:06:54 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index f1c8c60fa..ce8581278 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.07.2107 -### Updated: Tue Jul 28 08:53:55 SAST 2020 +### Version: V4.2020.07.2108 +### Updated: Tue Jul 28 09:06:52 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 3d83c4983..33f94c3ed 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.74.79.111 0; +35.231.58.0 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index f1c8c60fa..ce8581278 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.07.2107 -### Updated: Tue Jul 28 08:53:55 SAST 2020 +### Version: V4.2020.07.2108 +### Updated: Tue Jul 28 09:06:52 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 2a28fc9d0..33da169b9 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.74.79.111 1; +35.196.82.30 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index a8c6bf5bf..c7d199157 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.74.79.111 0; +35.196.82.30 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index f1c8c60fa..ce8581278 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.07.2107 -### Updated: Tue Jul 28 08:53:55 SAST 2020 +### Version: V4.2020.07.2108 +### Updated: Tue Jul 28 09:06:52 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 2a28fc9d0..33da169b9 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.74.79.111 1; +35.196.82.30 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index a8c6bf5bf..c7d199157 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.74.79.111 0; +35.196.82.30 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index f1c8c60fa..ce8581278 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.07.2107 -### Updated: Tue Jul 28 08:53:55 SAST 2020 +### Version: V4.2020.07.2108 +### Updated: Tue Jul 28 09:06:52 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 2a28fc9d0..33da169b9 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.74.79.111 1; +35.196.82.30 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 972cfbda6..9c1f82a5a 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -270,6 +270,7 @@ "~*(?:\b)ips-agent(?:\b|)" 0; "~*(?:\b)Iria(?:\b|)" 0; "~*(?:\b)IRLbot(?:\b|)" 0; + "~*(?:\b)isitwp.com(?:\b|)" 0; "~*(?:\b)Iskanie(?:\b|)" 0; "~*(?:\b)IstellaBot(?:\b|)" 0; "~*(?:\b)JamesBOT(?:\b|)" 0; @@ -450,6 +451,7 @@ "~*(?:\b)RSSingBot(?:\b|)" 0; "~*(?:\b)s1z.ru(?:\b|)" 0; "~*(?:\b)SalesIntelligent(?:\b|)" 0; + "~*(?:\b)satoristudio.net(?:\b|)" 0; "~*(?:\b)SBIder(?:\b|)" 0; "~*(?:\b)ScanAlert(?:\b|)" 0; "~*(?:\b)Scanbot(?:\b|)" 0; @@ -476,6 +478,8 @@ "~*(?:\b)Siphon(?:\b|)" 0; "~*(?:\b)SISTRIX(?:\b|)" 0; "~*(?:\b)Sitebeam(?:\b|)" 0; + "~*(?:\b)SiteCheckerBotCrawler(?:\b|)" 0; + "~*(?:\b)sitechecker.pro(?:\b|)" 0; "~*(?:\b)SiteExplorer(?:\b|)" 0; "~*(?:\b)Siteimprove(?:\b|)" 0; "~*(?:\b)SiteLockSpider(?:\b|)" 0; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index a8c6bf5bf..c7d199157 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.74.79.111 0; +35.196.82.30 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index f1c8c60fa..ce8581278 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.07.2107 -### Updated: Tue Jul 28 08:53:55 SAST 2020 +### Version: V4.2020.07.2108 +### Updated: Tue Jul 28 09:06:52 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 7a31c6e3c..5f04cdae8 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.196.57.92 0; -34.74.79.111 0; +34.73.34.132 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 890a359c4..3a312b27b 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.07.2107 -### Updated: Tue Jul 28 08:53:55 SAST 2020 +### Version: V4.2020.07.2108 +### Updated: Tue Jul 28 09:06:52 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index ea359fd02..21f4798b7 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.196.57.92 0; -35.237.56.208 0; +34.73.66.97 0; +34.73.34.132 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 23a434e94..8f9e903f4 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.07.2107 -### Updated: Tue Jul 28 08:53:55 SAST 2020 +### Version: V4.2020.07.2108 +### Updated: Tue Jul 28 09:06:52 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index 169608f63..cd8c7272d 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.196.57.92 0; -35.196.158.85 0; -35.227.97.188 0; +35.231.58.0 0; +34.73.34.132 0; +35.227.58.83 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index f1c8c60fa..ce8581278 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.07.2107 -### Updated: Tue Jul 28 08:53:55 SAST 2020 +### Version: V4.2020.07.2108 +### Updated: Tue Jul 28 09:06:52 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 80067881a..808a5380a 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.196.158.85 0; -35.196.72.151 0; -35.237.56.208 0; +35.227.58.83 0; +34.74.91.53 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index f1c8c60fa..ce8581278 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.07.2107 -### Updated: Tue Jul 28 08:53:55 SAST 2020 +### Version: V4.2020.07.2108 +### Updated: Tue Jul 28 09:06:52 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index ad23b3d8a..49090b431 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.227.97.188 1; +34.74.91.53 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/blacklist-user-agents.conf b/.dev-tools/test_units/blacklist-user-agents.conf index 972cfbda6..9c1f82a5a 100644 --- a/.dev-tools/test_units/blacklist-user-agents.conf +++ b/.dev-tools/test_units/blacklist-user-agents.conf @@ -270,6 +270,7 @@ "~*(?:\b)ips-agent(?:\b|)" 0; "~*(?:\b)Iria(?:\b|)" 0; "~*(?:\b)IRLbot(?:\b|)" 0; + "~*(?:\b)isitwp.com(?:\b|)" 0; "~*(?:\b)Iskanie(?:\b|)" 0; "~*(?:\b)IstellaBot(?:\b|)" 0; "~*(?:\b)JamesBOT(?:\b|)" 0; @@ -450,6 +451,7 @@ "~*(?:\b)RSSingBot(?:\b|)" 0; "~*(?:\b)s1z.ru(?:\b|)" 0; "~*(?:\b)SalesIntelligent(?:\b|)" 0; + "~*(?:\b)satoristudio.net(?:\b|)" 0; "~*(?:\b)SBIder(?:\b|)" 0; "~*(?:\b)ScanAlert(?:\b|)" 0; "~*(?:\b)Scanbot(?:\b|)" 0; @@ -476,6 +478,8 @@ "~*(?:\b)Siphon(?:\b|)" 0; "~*(?:\b)SISTRIX(?:\b|)" 0; "~*(?:\b)Sitebeam(?:\b|)" 0; + "~*(?:\b)SiteCheckerBotCrawler(?:\b|)" 0; + "~*(?:\b)sitechecker.pro(?:\b|)" 0; "~*(?:\b)SiteExplorer(?:\b|)" 0; "~*(?:\b)Siteimprove(?:\b|)" 0; "~*(?:\b)SiteLockSpider(?:\b|)" 0; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 63e38cdb1..2b8fe93a7 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -DomainCrawler -Ebingbong -InfoNaviRobot -Msrabot -Nibbler -PECL::HTTP -SEOprofiler -Spanner -Telesphorep -Widow +IRLbot +MarkMonitor +NextGenSearchBot +POE-Component-Client-HTTP +scan.lol +SEOkicks-Robot +Spammen +Spinn3r +Whacker +Wprecon diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index a281d2913..7453904d5 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -404checker +80legs Abonti -Acunetix -Aipbot +ADmantX AllSubmitter AlphaBot -arquivo.pt +Anarchie +Ankit +archive.org_bot +Aspiegel +ASPSeek Attach AwarioRssBot BackDoorBot backlink-check -Badass -Bandit -Battleztar Bazinga -BDCbot +Barkrowler +BBBike BDFetch BetaBot -Bitacle +Bigfoot Black Hole -BlackWidow -Blow BlowFish -Boardreader -Bolt BotALot +Brandprotect +Brandwatch BuiltBotTough BuiltWith -Bullseye Calculon -CATExplorador -CherryPicker +CazoodleBot +CheeseBot +ChinaClaw +Chlooe Claritybot +Cloud mapping coccocbot-web -Cogentbot cognitiveseo +Collector com.plumanalytics -CopyRightCheck Cosmos -crawler.feedback -crawl.sogou.com +crawler4j CrazyWebCrawler -CSHttp -Curious +Crescent +DataCha0s DBLBot -demandbase-bot -Demon Deusu -Dispatch -DnyzBot +DIIbot +Disco +Discobot Domains Project +DomainStatsBot Dotbot +Download Wonder Dragonfly -Drip -DSearch DTS Agent -EasyDL +Ebingbong ECCP/1.0 +Ecxi EirGrabber -EMail Siphon -EMail Wolf +EroCrawler evc-batch -Evil Exabot +ExtLinksBot ExtractorPro -EyeNetIE +Ezooms facebookscraper -FemtosearchBot +FHscan +Fimap Foobot -GermCrawler -GetRight -GetWeb +FyberSpider +Fyrebot +GalaxyBot +Genieo +Getintent +G-i-g-a-b-o-t +Go-Ahead-Got-It +Gotit Go!Zilla Grabber -Grafula +GrabNet GrapeFX GridBot -Harvest -HEADMasterSEO -heritrix -Heritrix -Hloader +Haansoft +HaosouSpider HTMLparser HTTP::Lite -HTTrack -Humanlinks -IDBot -Indy Library +Image Fetch +Image Sucker InfoTekies -Intelliseek InterGET -Iria -IRLbot +Internet Ninja +InternetSeer +internetVista monitor +isitwp.com JamesBOT Jbrofuzz -JetCar -Jetty -JikeSpider -Joomla +JennyBot Jorgee JustView +Kenjin Spider Keyword Density +Kinza Larbin +Lftp +LibWeb +Libwhisker LinkpadBot +LinksManager +LinqiaMetadataDownloaderBot LinqiaScrapeBot +Lipperhey +Lipperhey Spider Litemage_walker +Lmspider Ltx71 -lwp-request LWP::Simple -Magnet magpie-crawler Mail.RU_Bot -Majestic SEO -MarkMonitor +Majestic-SEO MarkWatch +Masscan +MauiBot Mb2345Browser meanpathbot -mediawords -MegaIndex.ru -MFC_Tear_Sample +MeanPath Bot +Mediatoolkitbot MicroMessenger -Microsoft URL Control -MJ12bot -Morfeus Fucking Scanner +MIIxpc +Mister PiX +Mojeek MQQBrowser -MSFrontPage Msrabot -Nameprotect -Needle +muhstik-scan Nessus +NetAnts netEstate NE Crawler -Net Vampire -NetZIP -Nibbler -NICErsPRO -Nimbostratus -Nmap -NPbot -Openfind +NextGenSearchBot +Niki-bot +Ninja +oBot +Octopus +Offline Explorer +Offline Navigator +OnCrawl OpenLinkProfiler Openvas OpenVAS +OPPO A33 +OrangeBot OrangeSpider +OutclicksBot +OutfoxBot PageAnalyzer -PageScorer Pandalytics -Papa Foto -Pavuk -PECL::HTTP +PeoplePal +PHPCrawl +Picscout +Picsearch PictureFinder -Pimonster PleaseCrawl +plumanalytics Pockey +POE-Component-Client-HTTP polaris version -Probethenet ProPowerBot Psbot -Pump -Quick-Crawler -RealDownload +PyCurl +RankActiveLinkBot +RankingBot2 +Rankivabot RebelMouse -RepoMonkey +RedesScrapy +ReGet +Ripper RocketCrawler -RSSingBot -s1z.ru +Rogerbot SalesIntelligent +SBIder +ScanAlert Scanbot ScoutJet -Screaming -ScreenerBot +Scrapy +Searchestate Semrush -SEOkicks -SEOkicks-Robot -SEOprofiler -SEOstats -serpstatbot -sexsearcher -SiteExplorer -SMTBot -Snapbot -SocialRankIOBot +seoscanners +Shodan +SISTRIX +SiteCheckerBotCrawler +Siteimprove +SiteSucker +Site Sucker +Sitevigil +SmartDownload +Sociscraper Sogou web spider Sottopop +Spammen SpankBot sp_auditbot -Spinn3r -SputnikBot -Sqlmap Sqlworm -Sqworm -Stripper +Steeler Sucuri SuperBot Surfbot -SurveyBot -Suzuran -T0PHackTeam -T8Abot -Telesoft +Telesphoreo +Telesphorep +TheNomad +Titan +Toweyabot +Tracemyfile Trendiction +Trendictionbot trendiction.com -True_Robot +trendiction.de Turnitin -TurnitinBot TwengaBot Typhoeus -UnisterBot -Upflow -URLy Warning URLy.Warning -Vacuum +VB Project +VCI VeriCiteCrawler -VoidEYE -Voil Voltron WBSearchBot Webalta -WebAuto +Web Auto WebBandit -WebCollage -WebCopier -WEBDAV -WebEnhancer +Web Enhancer +WebFuck +WebGo IS WebImageCollector -WebmasterWorldForumBot -WebPix +WebLeacher +webmeup-crawler WebSauger -Web Sauger -Webshag -WebsiteQuester -Webster -Web Sucker -WebZIP +WebSucker Whack -Who.is Bot +Whatweb +Widow WinHTTrack -Woobot -Wotbox -Wprecon +WiseGuys Robot +Wonderbot WPScan +WWW-Collector-E WWW-Mechanize -x22Mozilla -Xaldon_WebSpider +x09Mozilla +Xaldon WebSpider Xenu -YoudaoBot -Zade Zauba -zauba.io Zermelo -zgrab -Zitebot +Zeus +ZmEu +ZumBot +ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 590fad4f1..f62b2adf9 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -360Spider -404enemy -80legs -Aboundex -Aboundexbot Acunetix -ADmantX AhrefsBot -Alexibot Alligator Anarchie -Apexoo archive.org_bot arquivo.pt Aspiegel -Attach -autoemailspider -Bandit -Barkrowler +AwarioRssBot +BackDoorBot +Backlink-Ceck +backlink-check BatchFTP -Battleztar Bazinga -BBBike -BDCbot +BDFetch +BetaBot Bigfoot Bitacle Blackboard -BlackWidow +Black Hole +BLEXBot Blow +BlowFish Boardreader Bolt BotALot -Buddy BuiltBotTough BuiltWith -BunnySlippers -Calculon -CCBot -CheeseBot -CheTeam -ChinaClaw -coccocbot-web +BuzzSumo +Chlooe +Claritybot +Cliqzbot +Cloud mapping Cogentbot -cognitiveseo -Copier -Cosmos +com.plumanalytics +CopyRightCheck crawler.feedback -CrazyWebCrawler -Curious +CrunchBot +CSHttp +Custo +DatabaseDriverMysqli DBLBot -DigitalPebble +Deusu +Digincore DIIbot -Dirbuster -Discoverybot -Dispatch +Disco DittoSpyder +DnyzBot DomainAppender DomainSigmaCrawler -Dotbot +domainsproject.org Download Wonder -Dragonfly DSearch DTS Agent +EasyDL eCatch +ECCP/1.0 Ecxi +EirGrabber evc-batch -Exabot Express WebPictures -Extractor -Extreme Picture Finder -EyeNetIE +ExtLinksBot +ExtractorPro Ezooms -FDM -FemtosearchBot +FHscan +Fimap Firefox/7.0 -FlashGet -Flunky +Foobot +FrontPage FyberSpider -Genieo -Getintent +GalaxyBot Gigablast -Go!Zilla +Gigabot +G-i-g-a-b-o-t +Go-Ahead-Got-It +Gotit +Grabber +Grafula GrapeFX GrapeshotCrawler -GridBot +GT::WWW Haansoft -HaosouSpider -Harvest -Havij -Heritrix +heritrix HMView -Humanlinks -HybridBot -Iblog +HTMLparser +HTTP::Lite +HTTrack IDBot -Id-search -Image Fetch -IndeedBot -Intelliseek -Internet Ninja -ips-agent -Iria -IRLbot +IlseBot +InterGET +InternetSeer +isitwp.com Iskanie -IstellaBot +JamesBOT +Jbrofuzz JennyBot -Jetty -JikeSpider -Joomla +Jorgee +JustView +Jyxobot +Kenjin Spider Kinza +Larbin LeechFTP LeechGet -Lftp +LexiBot Libwhisker +LieBaoFast +Linkdexbot LinkextractorPro -LinqiaScrapeBot -LWP::Simple +LinkpadBot +LinkWalker +Lmspider +LNSpiderguy +Ltx71 +lwp-request Magnet +Mag-Net +magpie-crawler +Mail.RU_Bot +Majestic SEO Majestic-SEO MarkMonitor MarkWatch -MauiBot -meanpathbot +Meanpathbot MeanPath Bot -mediawords +MegaIndex.ru +MFC_Tear_Sample +Microsoft Data Access +Microsoft URL Control MIDown tool MIIxpc Mojeek -MSIECrawler -muhstik-scan +Mojolicious +Morfeus Fucking Scanner Musobot -Nameprotect Navroad NearSite +Needle Nessus +NetAnts Netcraft netEstate NE Crawler -NetLyzer -NetSpider -NICErsPRO -Nimbostratus -Nutch -Octopus -Offline Explorer +Net Vampire +Netvibes +NextGenSearchBot +Nibbler +NimbleCrawler +Nmap +oBot +Offline Navigator OnCrawl -Openfind OpenLinkProfiler Openvas OpenVAS -OPPO A33 -OutfoxBot +OrangeSpider +Page Analyzer PageGrabber -page scorer Panscient -Pavuk +pcBrowser PECL::HTTP -PeoplePal -Pimonster +PHPCrawl +Pi-Monster Pixray -PleaseCrawl -ProPowerBot -PyCurl -QueryN Metasearch -Quick-Crawler -RankActive -RankingBot2 -Ripper -Rogerbot +Pockey +polaris version +Probethenet +Pump +Rankivabot +RealDownload +ReGet +ScanAlert +Scanbot scan.lol -ScreenerBot +Screaming Searchestate -Semrush +SemrushBot SEOkicks SEOkicks-Robot +Seomoz +SEOprofiler +SeoSiteCheckup serpstatbot -Shodan -SiteExplorer +Sitebeam +sitechecker.pro Siteimprove -SiteSucker -Site Sucker +Sitevigil +SlySearch SmartDownload -SMTBot -Snake -Snapbot -SocialRankIOBot -Sociscraper -sogouspider -Spammen +Sosospider +SpankBot +Spanner +Spbot Spinn3r SputnikBot +Sqlmap +Sqlworm Sqworm -Steeler -Sucker -Sucuri -SuperBot +Stripper Surfbot -SurveyBot -T0PHackTeam +T8Abot tAkeOut -Teleport -TeleportPro -Telesoft Telesphorep -TheNomad -Tracemyfile +The Intraformant +Titan Trendiction -Trendictionbot -Turingos +trendiction.de +True_Robot +Turnitin TurnitinBot -Upflow +Typhoeus +UnisterBot +URLy.Warning Vacuum -Vagabondo +VCI VeriCiteCrawler -VidibleScraper -Voil -WallpapersHD +Virusdie +Voltron +Wallpapers/3.0 WBSearchBot Webalta -Web Auto +WebBandit +WebCollage Web Collage -WebCopier -Web Enhancer +WebEnhancer WebFetch Web Fetch -WebFuck -Web Fuck WebImageCollector +WebmasterWorldForumBot WebPix -WebReaper Web Sauger -Website Quester -WebStripper -WebSucker +Webshag +WebsiteExtractor +WebsiteQuester Web Sucker WeSEE Whacker Whatweb -Who.is Bot Widow +WinHTTrack WiseGuys Robot -WISENutbot -Wonderbot Woobot Wotbox +WPScan +WWW-Mechanize +WWWOFFLE x22Mozilla -Xaldon WebSpider -Xenu xpymep1.exe +YoudaoBot +Zauba zauba.io -zgrab -ZumBot +Zermelo +Zitebot diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index d43d0b7e3..362f7c2cd 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -crystalslot.com -czat.wp.pl -doxyporno.com -extractorandburner.com -hplaserjetpdriver8y.pen.io -kladrus.ru -lunchrock.co -mhtr.be -nosecret.com.ua -yellowads.men +bfz.biz +check-host.net +clickso.com +creditwell.ru +detalizaciya-tut.biz +kirov.zrus.org +stair.registrydomainservices.com +warningwar.ru +whatsappdownload10.com +zeikopay.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index be43d0926..293a51c13 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -08800.top -14b.info -256bit.by -91abcw.com -abtasty.com -abwa.tk -acortarurl.es -activepr.ru +00it.com +0912701309f8ce.com +0daymusic.org +101flag.ru +10bet.com +123any.com +1688.com +23kw.ru +2pxg8bcf.top +3dracergames.com +5forex.ru +7makemoneyonline.com +7search.com +a96527gi.beget.tech +aac500b7a15b2646968f6bd8c6305869d7.com +abcguru.xyz +adamoads.com addtoadd.men -adktrailmap.com -adsland.men -adultfullhd.com -agardomains.com -aibolita.com -alekseevec.ru -alert-jdh.xyz -alfa9.com -alice.it -alldezservice.kz +adobereader-free.ru +adultgalls.com +aerogo.com +air-edem.ru +airlandsea.info +alfa-img.com +alfapro.ru +algarveglobal.com +allbizne.co.ua +allcredits.su +alldownload.pw allnews24.in -allornamenti.com -allstatesugarbowl.org -allwomen.info -alveris.ru -an-donut.com -animalia-life.club -animenime.ru -apartmentbay.ru -apibetweenlinesn-a.akamaihd.net -apidiamondatanet-a.akamaihd.net -apioasisspacenet-a.akamaihd.net -apishops.ru -api.stathat.com -apper.de -applicationg29.com -arenanews.com.ua -arendakvartir.kz -arendovalka.xyz -asiengirls.net -availit.weebly.com -aversis.net -avkzarabotok.com -avramstroy.ru -awaybird.ru +altamayoztourism.com +amung.us +anal-acrobats.hol.es +analytics-ads.xyz +android-style.com +animal36.com +animalrank.com +animebox.com.ua +anonymizeme.pro +apartmentratings.com +apparel-offer.com +apps-analytics.net +apptester.tk +arendas.net +arewater.com +asiavirtualsolutions.com +asscenihotosidea.blogspot.co.za +astana.xxxkaz.org +atleticpharm.org +autoblogger24.info +autobrennero.it +avto-oligarh.ru ayerbo.xhost.ro +babyfactory.fr +baikaleminer.com baltgem.com -banan.tv -bannerspace.com -bashtime.ru -bestdooz.com -bestimagecoollibrary.com -besttorrentknifta.weebly.com +banki76.ru +bausparen.bz.it +beauby.ru +bestpriceninja.com bestwaystogetpaid.us -bjetjt.com -blockety.co -bonkers.name -bonky.biz -boostmyppc.com -bosefux.esy.es -brandedleadgeneration.com -brianhenry.co -brk-rti.ru -brokergid.ru -brus.city -buchananshardware.com -bzero1jewelry.net -cakemediahq.com.au -call-of-duty.info -calstaterealty.us -carsplate.com -cartierwatch.xyz -casinos4dummies.co.uk -cdncash.org -c-gainsbourg.com -chatroulette.online -cheapmoncler.pw -chidporn.com -children-learningreading.info -christianlouboutinsaleonline.us -citetick.com -classiquebijoux.ru -clickcash.com -clodo.ru -club-lukojl.ru -club-samodelkin.ru -coderstate.com -codq.info -colobit.biz -co.lumb.co -comissionka.net -compliance-alex.top -compliance-barak.top +bezsporno.ru +biketank.ga +bikini-image.com +bildsuche.ru +billiard-classic.com.ua +biographiya.com +biznesrealnost.ru +bkgr.se +blogsfunky672.weebly.com +bmwhighperformers.com +board.f00d.de +boleznikogi.com +bombla.org +bonux.nextview.ru +breget74.ru +bupropion-sr-150-mg.us +burger-tycoon.com +buron.pw +cand.jp +candlehandmade.com +canvas.gloverid.site +cartierring.xyz +cbozhe.com +centrdebut.ru +centre-indigo.org.ua +cheapjerseysa.com +cheapmoncler.xyz +check-host.net +chee-by.biz +chelnytruck.ru +chelyabinsk.xrus.org +christianlouboutinoutlet.win +cinemaenergy-hd.ru +clarithromycin500mg.com +claytransformations.info +coldfilm.ru +colehaanoutlet.store +community.allhiphop.com +compiko.info compliance-brian.xyz -compliance-don.top -compliance-elena.top -compliance-margo.top -comsysnet.com -conocer-sanabria.com -contextualyield.com -coolbar.pro -copenergo.ru -copro.pw -countbertwistdisp26.soup.io -country-chic.ru -crazyboost.pro -crazyprotocol.com -custodieva.ru -daptravel.com -daretodonate.co -datsun-do.com +compliance-fred.top +compliance-irvin.top +compliance-mary.top +connectingsingles.com +cookie-law-enforcement-hh.xyz +cordstrap.cc +counter.bloke.com +counterbot.com +creatives.adbetclickin.pink +creditnation.ru +crirussian.ru +custom-product-labels.com +dafatiri.com ddpills.com -delfin-aqua.com.ua dellalimov.com -diegolopezcastan.com -digest-project.ru -digilander.libero.it -digital-video-processing.com -dimaka.info -doctissimo.fr -dojki-hd.com -domainsatcost.com -domcran.net -doxysexy.com -dpihatinh.gov.vn -druzhininevgeniy63.blogspot.com -dyshagi.ru -earl-brown.info -eas-seo.com +dengi-pod-zalog-nedvizhimosti.ru +detective01.ru +dfiles.me +dimkino.ru +dinosaurus.site +dipstar.org +directrev.com +discountliv.com +djonwatch.ru +dogsrun.net +domoysshop.ru +dostavka-v-krym.com +drivotracker.com +drunkmoms.net +easycommerce.cf ec-file.info -e-collantes.com -e-commerce-seo.com -ecommerce-seo.org +edudocs.net ege-russian.ru -ekn-art.se -elektrischezigaretten1.blogse.nl -elektronischezigarettekaufen2.tumblr.com +egovaleo.it +ekaterinburg.xrus.org +ekto.ee +elektronischezigarettekaufen1.beeplog.com elektrozigaretten1.postbit.com -ero-advertising.com -erotikchat-24.com -eu-cookie-law.blogspot.com -euronis-free.com -evening-dating-club.info -ewebarticle.info -exlarseva.webblog.es +elektrozigaretten1.webs.com +elvel.com.ua +elvenmachine.com +emergencyneeds.org +energydiet-info.ru +englate.com +eonpal.com +erectile.bid +exonline.info extlinks.com -ezigarettekaufen.myblog.de -ezigaretteshop2.vefblog.net -f00kclan.de -falllow.gq -falopicm.pw -fapgon.com -feel-planet.com -ferrotodo.com -ff30236ddef1465f88547e760973d70a.com -fiuxy.com -flirt4free.com -fordsonmajbor.cf -fototravel.eu +ezigarettekaufen2.mpbloggar.se +ezigaretteshop2.mywapblog.com +filesmonster.porn +filmidivx.com +findanysex.com +findpik.com +finejewelryshop.ru +finteks.ru +flowwwers.com +forpostlock.ru +fortevidyoze.net +forum.tvmir.org foxydeal.com -freegamesplay.online -freelifetimefuckbook.com -free-today.com -freshberry.com.ua -freshsuperbloop.com -freza-sverlo.ru -frighteningremain.cf -frustrated-favorable.gq -fsakhalin.ru -fuckbuddybestgilf.info -fun2cell.net -gazporno.com -gcup.ru -germetiki.com.ua -getmyads24.com -getpopunder.com -glcomputers.ru -gne8.com -go2mike.ru +free-laptop-reward.com +freewhatsappload.com +freewlan.info +friendflnder.com +ftns.ru +fuckingawesome.com +gael-s.ru +gamesprite.me +game-top.su +gateway.zscalertwo.net +gay-file.com +gaz-voshod.ru +ge0ip.org +gemgrab-a.akamaihd.net +ggiaro.com +gidonline.one +gigixo.com +gilbertbanda.net +glastecfilms.com.my +gloverid.site +good-mummy.ru +googlepositions.com +gopixdatabase.com gosarhivrt.ru -gpirate.com -gq-catalog.gq -grand-chlen.ru -groupmoney.ru -h2monline.com -hackers-crackers.tk -hamacapty.com -healmytrauma.info -heliko.no +gotcher.us +graphid.com +greendream.com.ua +gsmlab.pl +gsmtlf.ru +guitar-master.org +hamilton.ca +hauleddes.com +hayate.biz help.tpu.ru -hetmanship.xyz -hiwibyh.bugs3.com -hmywwogw.bloger.index.hr -holidaypics.org -hotenergy.ru -hotloans.ru -howtoclean.club -htmlcorner.com -ie.57883.net -igithab.com -ilikevitaly.com -impisr.ru -incep.imagine-ex.co -indetiske.ya.ru -indiakino.net -inet-traffic.com -infektsii.com -insurple.com -interesnie-faktu.ru -interferencer.ru -iphantom.com -irkutsk.zrus.org -irunfar.com -jackpotchances.com -jav-fetish.site -jbl-charge.info -jerseywholesalechinabiz.com +historichometeam.com +hledejvshopech.cz +hotkeys.com +hoverboardforsaledirect.com +hulfingtonpost.com +hundejo.com +iccornacircri.cf +ideashome.id +imadedinner.net +imagecoolpub.com +imfamous.info +imgarcade.com +immobiliaremassaro.com +imperiafilm.ru +indiasourcemart.in +infokonkurs.ru +intelhdgraphicsgtdrive6w.metroblog.com +iscblog.info +itservicesthatworkforyou.com +ixora.pro +jackwolfskinoutlet.online +janerikholst.se +jeremyeaton.co +jerseysbizwholesalecheap.com +jerseyschinabizwholesale.us +jerseyssportsshop.com +jestr.org jewelryandfiligree.com -juliadiets.com -justdating.online -justprofit.xyz +journalhome.com +jyvopys.com +karadene.com +katadhin.co kaz.kz -kerei.ru -keywordsking.com -kino-rating.ru -kletkimehan.ru -knogg.net -knowyournextmove.com -koleso24.com.ua -konica.kz -krasnodar.ru -krassh.ru -lampokrat.ws -lasvegaslockandsafe.com -lesbian.xyz -letmacworkfaster.world -letslowbefast.today -lida-ru.com -lineshops.biz -lizace.com -lkbennettstore.online +kenaba.su +kerch.site +kino2018.club +kinomaniatv.pl +klitimg.pw +knigonosha.net +komputernaya-pomosh-moscow.ru +ktotut.net +kumuk.info +kurwa.win +kvartira-sutochno.com +kw21.org +lapitec.eu +laulini.soclog.se +law-check-eight.xyz +lazy-z.com +lescinq.com +letmacworkfaster.site +lider-zhaluzi.kiev.ua +lifeinsurancekb.com +linkdebrideur.xyz +linkwithin.com +live-sexcam.tk +live-sexchat.ru +llastbuy.ru loadopia.com -lob.com.ru -lovi-moment.com.ua -lutherstable.org -macrotek.ru -mac-shield.com -manualterap.roleforum.ru -map028.com -mazda-roadsters.com -mcar.in.ua -mebelcomplekt.ru -mebel-ekb.com -medicine-4u.org -mega-bony-2017.pl -mesto-x.com +lollypopgaming.com +lol-smurfs.com +lomza.info +longadventure.com +lsex.xyz +m1media.net +mamasuper.prom.ua +maranbrinfo.com.br +m.b00kmarks.com +mcnamaratech.com +medicinacom.ru +meendo-free-traffic.ga michaelkorsoutletstore.net -minegam.com -miniads.ca -mitrasound.ru -modenamebel.ru -mojaocena.com -moneyviking-a.akamaihd.net -monsterdivx.tv -morlat.altervista.org -mossmesi.com +microstatic.pl +millionare.com +mindbox.co.za +minecraft-rus.org +minet.club +mir-business-24.ru +misandesign.se +missvietnam.org +moesen-ficken.com +moje-recenze.cz +mojpregled.com +moncleroutletonline.xyz +moneytop.ru +moroccosurfadventures.com most.gov.iq -most-kerch.org -moyaterapiya.ru -mustwineblog.com -muzaporn.com -muz-shoes.ru -my-aladin.com -mygameplus.com -mypornfree.ru -netvouz.com -newsperuse.com -newstudio.tv +motherboard.vice.com +movies-in-theaters.net +mp3films.ru +mrbojikobi4.biz +musicstock.me +muz-baza.net +my-big-family.com +mycouponizemac.com +mydoctorok.ru +myfreetutorials.com +myhealthcare.com +myindospace.com +myiptest.com +naj-filmy24.pl +naturalbreakthroughsresearch.com +nerudlogistik.ru +net-profits.xyz +nextrent-crimea.ru +nfljerseysforsalewholesaler.com njkmznnb.ru -nlfjjunb5.ru -nonews.co -norththeface.store +nl.netlog.com +nordstar.pro +nosecret.com.ua notfastfood.ru -ntic.fr -nuup.info -olvanto.ru -o-o-8-o-o.ru -open-odyssey.org -optikremont.ru -optitrade24.com -osb.se11.ru -outrageousdeal-a.akamaihd.net -ownshop.win -ozas.net -pandroid.co +nottyu.xyz +novatech.vn +obsessionphrases.com +odesproperty.com +odoratus.net +oil-td.ru +okout.ru +onblastblog.online +one-gear.com +onlineku.com +onlinetvseries.me +ops.picscout.com +oralsexfilme.net +oranga.host.sk +oren-cats.ru +orion-v.com paradontozanet.ru -pateaswing.com -pathwhelp.org -percin.biz.ly -perso.wanadoo.es -petrovka-online.com -pharmacyincity.com -php-market.ru -physfunc.ru -picmoonco.pw -pila.pl -pinsdaddy.com -pixshark.com -pladform.ru -pliks.pl -pl-top.pl -pobeiranie.pl -popads.net -popugauka.ru -pornofiljmi.com -pornohd1080.online -porno-play.net -porno-video-chati.ru -portside.cc -pr0fit-b0x.com -prchecker.info -priscilarodrigues.com.br -privatbank46.ru -produkto.net -profit-opportunity.com -progonrumarket.ru +parajumpersstore.online +partnersafe.men +pcimforum.com +pdns.cz +pearlisland.ru +pestomou.info +petedrummond.com +pisanieprac.info +piski.top +pizda.lol +pizdeishn.com +platezhka.net +playboyfiles.xblog.in +play-movie.pl +pornhub-ru.com +pornoforadult.com +preparevideosafesystem4unow.space +primedice.com +princevc.com +profitwithalex.info +prokotov.com pronekut.com -pussyspace.net -pxhdwsm.com -pyramidlitho.webs.com -qpypcx.com +prostitutki-yaroslavlya.org +qitt.ru quickcashlimited.com -ranksonic.org -rarbg.to -realizmobi.com -reeyanaturopathy.com -registrationdomainsite.com -resant.ru -rfd-split.hr -rfid-locker.co +raavidesigns.com +ratemodels.net +realitykings.com +realnye-otzyvy.info +realresultslist.com +recipedays.com +redirect.trafficreceiver.club +rednise.com +regionshop.biz +reklama-i-rabota.ru +remont-komputerov-notebook.ru +rent2spb.ru +rezeptiblud.ru +rff-cfal.info rique.host.sk -rockprogblog.com -royal-investments.net -rukino.org -ruscoininvest.company +rn-to-bsn.com +robot-forex.biz +rollercoin.com +rossanasaavedra.net +royalvegascasino.com +rp9.ru russia-tao.ru -russintv.fr -russkie-gorki.ru -saletool.ru -savefrom.com -saveriopiazza.it -savetubevideo.com -sax-sex.com -sayyoethe.blogspot.co.za -scanner-jack.top -scenicmissouri.us -searchencrypt.com -searchimage.co +rvtv.ru +s8-nowy-wygraj.comli.com +saecsa.co +sanidumps.com +santasgift.ml +scanspyware.net +sdelai-prosto.ru searchkut.com -seemoreresultshu-a.akamaihd.net -semrush.com -senger.atspace.co.uk -seoexperimenty.ru -seo-platform.com -seo-smm.kz +search.xtconnect.com +secretscook.ru +seocheckupx.com +seojokes.net +seomarketings.online +seoprofiler.com +seorankinglinks.com +service-core.ru serviporno.com -sexy.babes.frontend-stack.top -shiksabd.com -shinikiev.com.ua -shopsellcardsdumps.com -shop.xz618.com -signx.info +sexkontakte-seite.com +sexpornotales.com +sexpornotales.net +share-buttons-for-free.com +ship-marvel.co.ua +shitmovs.com sildenafilcitratemed.com -silvercash.com -siteripz.net -site.ru -sitevaluation.com -sitevalued.com -sitiz.club -sitopreferito.it -si-unique.com -s-iwantyou.com -skatestick.bid -slavkokacunko.de -sms2x2.ru -socialbutton.xyz -softlinesolutions.me -solnplast.ru -sotkal.lark.ru -soundfrost.org -space4updating.win -spamnuker.com -special-porn.com -speeddream.xyz -spywarenuker.com -sribno.net -stal-rulon.ru -start.myplaycity.com -steelmaster.lv -stillmiracle.com -stjamesschool.info -strag-invest.ru -strana-krasoty.ru -streha-metalko.si -strfls.com -superlist.biz -supers.com.ua -superstarfloraluk.com -surcentro.com -surflinksmedical.com -surintech.ac.th -taaaak.com -tacbelarus.ru -testingads.pro -textads.men -thebluffs.com -thefotosgratis.eu -thegameriders.com -thesmartsearch.net -thexart.club -tizanidineandhydrocodone.blogspot.com -tizanidineforopiatewithdrawal.blogspot.com -tizanidinerestlesslegsyndrome.blogspot.com -tmtrck.com -tomatis.gospartner.com -top10-online-games.com +sispe.com.br +siteonomy.com +sixcooler.de +sobecjvuwa.com.ru +soc-econom-problems.ru +socialsignals24.com +soserfis.com +spasswelt.net +spyfu.com +standardchartered-forex.com +stathat.com +statoutlook.info +strongholdsb.ru +sudexpert66.ru +supermama.top +svbur.ru +svetodiodoff.ru +svtrd.com +taihouse.ru +technopellet.gr +teslathemes.com +thegioixekhach.com +ticketsys.inetwd.com +timer4web.com +titslove.yopoint.in +tizanidine4mgstreetvalue.blogspot.com +tizanidinecost.blogspot.com +tizanidinehcl4mgtab.blogspot.com +tizanidinehcl4mgtabinfo.blogspot.com +tizanidinehcl4mgtablet.blogspot.com +tizanidinehydrochloride2mg.blogspot.com +tizanidinemusclerelaxant.blogspot.com +tizanidinepills.blogspot.com +tizanidinestreetprice.blogspot.com top10registrycleaners.com -topanasex.com -topsy.com -torontoplumbinggroup.com -torrentred.games -totu.info -trafficgenius.xyz -trafficnetzwerk.de -traffique.net -truck-land.life -trumpetedextremes.com -trustl.life -tuberkuleznik.ru -twelvevisionspartyofcolorado.com -twiclub.in -u555u.info -uchebavchehii.ru -unblocksit.es -underthesite.com -unmaroll.ya.ru -unrealcommander.org -uptime.com -urlopener.blogspot.com.au -uroffer.link -valkiria-tk.ru -velpanex.ru -video-chat.love -viel.su -villacoloniale.com -vintontech.info -vip-parfumeria.ru -vladimir.xrus.org -vod.com.ua -vsexkontakte.net -vykup-avto-krasnodar.ru -wallpapers-best.com -watchmyfb.pl -webix.me -wfb.hatedriveapart.com -whereverdesperate.gq -whos.amung.us -williamrobsonproperty.com -win-spy.com -wnhjavlhezp.gq -www.kabbalah-red-bracelets.com -xblognetwork.com -xfire.com -xfluro.com -x-lime.com -xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai -xn--80aebbcbcdemfkhba4byaehoejh8dza3v.xn--p1ai -xn----8sblgmbj1a1bk8l.xn----161-4vemb6cjl7anbaea3afninj.xn--p1ai -xn--c1acygb.xn--p1ai -xnxx-n.com -xportvusbdriver8i.snack.ws +top-deal.com.pl +top-karkas.ru +top-l2.com +touchmods.fr +track2.shop +tradedeals.biz +traffic100.com +traffixer.com +truck-addzilla.life +truck-rece.life +tt-ipd.info +turbabitload.weebly.com +turizmus.us +tvteleport.ru +ufa.xrus.org +unrealcommander.biz +uptime-gamma.net +url2image.com +urll.eu +usiad.net +ussearche.cf +validdomain.xyz +vclicks.net +vellings.info +via-gra.webstarts.com +videochat.cafe +vip-dom.in +vksex.ru +vmnmvzsmn.over-blog.com +vodaodessa.com +vrnelectro.ru +vseuznaem.com +vzubah.com +warningwar.ru +website-analyzer.info +websites-reviews.com +webstatsdomain.org +westvilletowingservices.co.za +whengirlsgowild.com +whipme.yopoint.in +windowssearch-exp.com +withstandingheartwarming.com +womens-journal.net +worldis.me +worldlovers.ru +woweb.com.ua +wurr.voila.net +xchangetrak.com +xgames-04.com +xjrul.com +xlovecam.com +xn-----7kcabaipgeakzcss7bjdqdwpfnhv.xn--p1ai +xn--b1ag5cfn.xn--p1ai +xuki.us +xvideosbay.com xxx-treker.ru -xxxtubesafari.com -yaaknaa.info -yellocloud.be -youbloodyripper.com -youradulthosting.com -yourothersite.com -yoursite.com -yugo-star.ru -yun56.co -zapatosenventa.info -zarabotat-na-sajte.ru -zdesformula.ru -zeroredirect.com -zetmaster.ru -znakom.sibtest.ru -znaniyapolza.ru -zoogdinsney.com -zscalerone.net -zverokruh-shop.cz +yoopsie.com +yopoint.in +youdao.com +yourads.website +yourporn.com +yousense.info +zacreditom.ru +zaimhelp.ru +zaloro.com +zaobao.com.sg +zarajbuilders.com +zeroredirect1.com +zigarettenonlinekaufen2.drupalgardens.com +znakomstvaonlain.ru diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index e58e629c6..963f58cae 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -007angels.com -123locker.com -2015god.org -3dracergames.com -777-club.ru -8gold.com -aad1f4acb0a373420d9b0c4202d38d94fa.com -acmebtn.ml -adclickservice.com -adimmix.com -adsland.men -afslankpillen2017nl.eu -ahrntal.verymes.xyz -air-edem.ru -alphacarolinas.com -ampower.me -apartment.ru -appleid-verification.com -arcadeplayhouse.com -artefakct.com -asrvvv-a.akamaihd.net -astrochicks.com -b00kmarks.com -bestkfiledxd.cf -bget.ru -binaryoptionscops.info -bioinnovate.co -bio-market.kz -bitporno.sx -bjetjt.com -bloggers.nl -bobinoz.com -boc.kz -bongiornos.info -bookmark4you.com -brains2.biz -brakehawk.com -brg8.com -buyantiviralwp.com -buyviagraa.blogspot.com -cakesplus.com.au -camdolls.com -cartierbraceletsreplica.pw -cellfun.mobi -cenokos.ru -cenoval.ru -cheapcigarettesc.info -cheapmarmot.online -cheatcode-lita12.rhcloud.com -cialis-samples.com -cicaki.net -citizenclsdriveri7.pen.io -compliance-ivan.top -compliance-josh.top -cottageofgrace.com -cryptoswap.biz -cypernhuset.se -deanmoore.ie -demian.kz -dev78.dioniqlabb.se -digitalfaq.com -discountliv.com -djstools.com -downloadkakaotalk.com -dvrlists.com -easyshoppermac.com -ecookna.com.ua -elektroniksigaraankara.info -elektrozigarettekaufen2.blogse.nl -elitesportsadvisor.com -elstal.com.pl -emergencyneeds.org -eropho.net -ezigarettenkaufen1.pagina.gr -ezigarettenshop2.myblog.de -fermersovet.ru -filesclub.net -floridamhca.org -forpackningsutveckling.se -forsex.info -free-laptop-reward.com -fuckmill.com -gamerextra.com -ge0ip.net -glavtral.ru -goodhousekeeping.com -gsasearchengineranker.xyz -hatedriveapart.com -highspeed5.net -hitsbox.info -hiwibyh.bugs3.com -honyaku.yahoofs.jp -house.sieraddns.com -hubbble.com -ibmdatamanagement.co -inbabes.sexushost.com -infospot.pt -jerseywholesalebizchina.com -kadashihotel.com -karaganda.xkaz.org -kimcurlrvsms.com -konoplisemena.com -krasivye-devushki.net -landoftracking.com -lasvegaslockandsafe.com -ledpolice.ru -leftofcentrist.com -lexixxx.com -libertybilisim.com -lietaer.com -lifeinsurancekb.com -lkbennettoutlet.online -lob.com.ru -localmatchbook.com -locimge.pw -lombardfinder.ru -maju.bluesalt.co -makis.nu -mejoresfotos.eu -metabar.ru -mindbox.co.za -minecraft-rus.org -miniature.io -misterjtbarbers.com -moncleroutletonline.pw -monitorwebsitespeed.com -moxo.com -msfsaar.de -musicdaddy.net -muz-shoes.ru -my-floor.in.ua -mykings.pw -nakozhe.com -nhl17coins.exblog.jp -nlfjjunb5.ru -novodigs.com -nucia.biz.ly -nudo.ca -okroshki.ru -ok.ru -okuos.com -oneminutesite.it -online-hit.info -onlinemeetingnow.com -online-templatestore.com -oranga.host.sk -ordernorxx.com -orgasmus-virtual.com -outrageousdeal-a.akamaihd.net -pastaleads.com -photosaga.info +00go.com +08800.top +1stat.ru +3w24.com +5tey7463.weebly.com +8lufu.com +a14download.com +aaa38852e886ac4af1a3cff9b47cab6272.com +aaronabel.com +actulite.com +adcash.com +adelly.bg +adloads.com +adtech.fr +adtiger.tk +adultmeetonline.info +agahinameh.com +ahmedabadwebs.com +aktivator-windows10.blogspot.com +allkrim.com +apartmentbay.ru +apilinkswiftco-a.akamaihd.net +arpelsreplica.xyz +atleticpharm.org +ausergrubhof.info +auto4style.ru +azte.ch +bag77.ru +bestmarriages.com +bestmobilityscooterstoday.com +bigpictures.club +blavia.00author.com +bloglag.com +brainsandeggs.blogspot.com +budilneg.xyz +bum.com.ru +burn-fat.ga +buyk.host.sk +calcularpagerank.com +callawaygolfoutlet.online +call-of-duty.info +candytiner.com +capture-room.com +carapics.com +carson.getenjoyment.net +cartierlovebracelet.xyz +cdncash.com +cementaresearch.se +cercacamion.it +changde.58.com +charmstroy.info +cheapestjerseys-wholesale.com +chiptuninger.com +christianlouboutinoutlet.win +colobit.biz +compliance-irvin.xyz +converse.ddsoldes.fr +coolwallpapers4k.info +creams.makeforum.eu +cwetochki.ru +ddpills.com +de.zapmeta.com +discovertreasure-a.akamaihd.net +distver.ru +diy-handmade-ideas.com +dogsrun.net +domain-tracker.com +dumuelave.xyz +dustyorate.com +ege-essay.ru +elektrischezigaretten1.blogse.nl +enternet.ee +ereko.ru +errorfixing.space +esfchat.tk +estibot.com +euromasterclass.ru +extads.net +extremal-blog.com +faptitans.com +fashion.stellaconstance.co +fbdownloader.com +fefo.gdn +fettefrauen.net +fifa-coins.online +filunika.com.ru +flash4fun.com +formularz-konkurs.tk +foxydeal.com +freesitetest.com +fym.com.ru +gamerscorps.com +gaz-voshod.ru +gelezki.com +getridofstretchmarks.org +gg-arena.ru +ghazel.ru +gk170.ru +gosmeb.ru +gsasearchengineranker.space +habermetre.com +hauleddes.com +heatpower.ru +hebr.myddns-flir.com +helvetia.com.ua +hotblognetwork.com +hotgirlhdwallpaper.com +hotloans.ru +hotxnights.info +idc.com.ua +images-free.net +investmac.com +inzn.ru +iqupdatetmz.win +janavibekken.no +javspace.net +jikoman.info +jrcigars.com +kalandranis.gr +kidd.reunionwatch.com +krassh.ru +kriokomora.info +lacasamorett.com +lineavabit.it +linkbuddies.com +lsp-awak-perikanan.com +mackeeper-center.club +mactechinfo.info +magicdiet.gq +marblestyle.ru +masserect.com +medosmotr-ufa.ru +metarip.ru +micasainvest.com +mirobuvi.com.ua +mobplayer.net +monsterdivx.tv +moscow-clining.ru +musezone.ru +mystats.xyz +narkologiya-voronezh.ru +narosty.com +neobux-bg.info +net-radar.com +newhairstylesformen2014.com +nippon-bearings.ru +notaria-desalas.com +notfastfood.ru +oakridgemo.com +ok-ua.info +onlinadverts.com +onlineserialy.ru +online-x.ru +origin-my.ru +pammik.ru +papasdelivery.ru +paphoselectricianandplumber.com +pcgroup.com.uy picture-group.com -pk-pomosch.ru -platesauto.com -play-movie.pl -plusnetwork.com -popugaychiki.com -pornolook.net +pills24h.com +pinstake.com +piter.xrus.org +pizda.lol +polimga.pw +poptool.net porno-video-chati.ru -profitkode.com -promalp-universal.ru -promoheads.com -pshare.biz -purplestats.com -quickchange.cc -rabotaetvse.ru -realresultslist.com -rebelmouse.com -redirect.trafficreceiver.club -reining.lovasszovetseg.hu -respublica-otel.ru -rosbalt.com.ua -rusoft-zone.ru -russia-tao.ru -russia-today-video.ru -saneyes.com -sax-sex.com -sayyoethe.blogspot.co.za -scanner-margo.top -searchinquire.com -searchwebknow-a.akamaihd.net -semrush.com -serpstat.com -servicecenter.co.ua -sexyteens.hol.es -sharpchallenge.com -shemalegalls.blogporn.in -simon3.ru -s-iwantyou.com -slim.sellany.ru -slonechka.ru -slutloadlive.com -snegozaderzhatel.ru -socialmonkee.com -soc-proof.su -space-worry.ml -superkanpo.com -survival.betteroffers.review -svarkagid.com -svnuppsalaorebro.se -taqplayer.info -tattooha.com -thebluenoodle.com -timer4web.com -tizanidinecapsules.blogspot.com -top-karkas.ru -topvidos.ru -totu.us -treasuretrack-a.akamaihd.net -try-rx.com -ultimateclassicrock.com -unisexjewelry.org -upstore.me -utrolive.ru -vchulkax.com -video-chat.in -videogamesecrets.com -vip2ch.com -voloo.ru -voloslove.ru -vsexkontakte.net -waysbetter.cn -webenlace.com.ar -webmasterhome.cn -westvilletowingservices.co.za -whatsupinfoley.com -whipme.yopoint.in -womensterritory.ru -works.if.ua -xcc24.pl -xn----7sbifcamovvfggw9d.xn--p1ai -xn--80ab4aa2g.xn--p1ai -xn----dtbndd4ae7eub.top -xnxx699.com -xportvusbdriver8i.snack.ws -xtrafficplus.com -yes-com.com +postclass.com +printie.com +profitwithalex.info +proftests.net +psa48.ru +puzzleweb.ru +rabot.host.sk +rangjued.com +realitykings.com +reeyanaturopathy.com +remorcicomerciale.ro +runofilms.ru +russintv.fr +saveindex.xyz +scanner-don.top +scanner-julianna.top +sexstream.pl +sextracker.be +sexy-screen-savers.com +shama-rc.net +shoppingjequiti.com.br +shtaketniki.ru +signx.info +simul.co +sisiynas.ru +skypasss.com +social-search.me +softlinesolutions.me +solitaire-game.ru +soserfis.com +soundfrost.org +souvenirua.com +sps-shop.com +srdrvp.com +startwp.org +stroimajor.ru +stroyplus.ru +swinger-mobil.net +swiped.su +syndicate.fun +testingads.pro +thejournal.ru +tiandeural.ru +tizanidine6mg.blogspot.com +tizanidinehcl4mgtablet.blogspot.com +tizanidineinfo.blogspot.com +tomck.com +tootoo.to +top10-way.com +topbestgames.com +topclickguru.com +trustaffs.com +tula.howotorg.ru +turizmus.us +uchil.net +uhdtv.website +universals.com.ua +uvozdeckych.info +uzungil.com +velpanex.ru +videochat.mx +vipps.com.my +vonradio.com +voprosotvet24.ru +wallpapers-best.com +website-speed-up.top +wesharepics.com +westermarkanjou.se +wineitudes.wordpress.com +winwotgold.pl +worldinternetauthority.com +wowas31.ucoz.ru +xev.ru +xn--80ajjbdhgmudixfjc8c5a9df8b.xn--p1ai +xxlargepop.com +xz618.com +yginekologa.com yourporngay.com -yuweng.info -zacreditom.ru -zaloadi.ru -zarabotat-na-sajte.ru -zarabotok--doma.ru +ypmuseum.ru +yugk.net +zaloro.com zarajbuilders.com -zatjmuzu.info -zeroredirect11.com -zigzog.ru -zophim.me -zryydi.com -zvetki.ru +znakomstva-piter78.ru +zooggames.com diff --git a/.dev-tools/test_units/whitelist-ips.conf b/.dev-tools/test_units/whitelist-ips.conf index 511678455..272faad99 100644 --- a/.dev-tools/test_units/whitelist-ips.conf +++ b/.dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.227.97.188 0; +34.74.91.53 0; 127.0.0.1 0; diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index 1b167a8e6..d1bf6f683 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index 77c3bb131..deba8132b 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index fda0e3127..03b3b3e6e 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2020.07.2107 +#### Version: V4.2020.07.2108 #### Bad Referrers Blocked: [6831](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [574](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [578](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index b209e0d49..2272f545d 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2020.07.2107 +#### Version: V4.2020.07.2108 #### Bad Referrers Blocked: [6831](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [574](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [578](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index 84c50644e..c4c76fa6f 100755 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2020.07.2107 +#### Version: V4.2020.07.2108 #### Bad Referrers Blocked: [6831](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [574](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [578](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index 2cae4a2c8..09900bad0 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,7 +1,3 @@ -isitwp.com -SiteCheckerBotCrawler -sitechecker.pro -satoristudio.net 360Spider 404checker 404enemy @@ -213,6 +209,7 @@ internetVista\ monitor ips-agent Iria IRLbot +isitwp.com Iskanie IstellaBot JamesBOT @@ -393,6 +390,7 @@ Rogerbot RSSingBot s1z.ru SalesIntelligent +satoristudio.net SBIder ScanAlert Scanbot @@ -419,6 +417,8 @@ Shodan Siphon SISTRIX Sitebeam +SiteCheckerBotCrawler +sitechecker.pro SiteExplorer Siteimprove SiteLockSpider diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 14002e8b5..ab9a6ca41 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2020.07.2107 +#### Version: V4.2020.07.2108 #### Bad Referrers Blocked: [6831](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [574](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [578](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index f1c8c60fa..ce8581278 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.07.2107 -### Updated: Tue Jul 28 08:53:55 SAST 2020 +### Version: V4.2020.07.2108 +### Updated: Tue Jul 28 09:06:52 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 7121d5fec..886e6d1ed 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2020.07.2107 -### Updated: Tue Jul 28 08:53:56 SAST 2020 +### Version: V3.2020.07.2108 +### Updated: Tue Jul 28 09:06:54 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index f1c8c60fa..ce8581278 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2020.07.2107 -### Updated: Tue Jul 28 08:53:55 SAST 2020 +### Version: V4.2020.07.2108 +### Updated: Tue Jul 28 09:06:52 SAST 2020 ### Bad Referrer Count: 6831 -### Bad Bot Count: 574 +### Bad Bot Count: 578 ################################################### ### VERSION INFORMATION ## @@ -358,6 +358,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)ips-agent(?:\b)" 3; "~*(?:\b)Iria(?:\b)" 3; "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)isitwp.com(?:\b)" 3; "~*(?:\b)Iskanie(?:\b)" 3; "~*(?:\b)IstellaBot(?:\b)" 3; "~*(?:\b)JamesBOT(?:\b)" 3; @@ -538,6 +539,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)RSSingBot(?:\b)" 3; "~*(?:\b)s1z.ru(?:\b)" 3; "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)satoristudio.net(?:\b)" 3; "~*(?:\b)SBIder(?:\b)" 3; "~*(?:\b)ScanAlert(?:\b)" 3; "~*(?:\b)Scanbot(?:\b)" 3; @@ -564,6 +566,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)Siphon(?:\b)" 3; "~*(?:\b)SISTRIX(?:\b)" 3; "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteCheckerBotCrawler(?:\b)" 3; + "~*(?:\b)sitechecker.pro(?:\b)" 3; "~*(?:\b)SiteExplorer(?:\b)" 3; "~*(?:\b)Siteimprove(?:\b)" 3; "~*(?:\b)SiteLockSpider(?:\b)" 3; diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index b75a0817f..2264ade40 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2020.07.2107 -### Updated: Tue Jul 28 08:53:57 SAST 2020 -### Bad Bot Count: 574 +### Version: V4.2020.07.2108 +### Updated: Tue Jul 28 09:06:55 SAST 2020 +### Bad Bot Count: 578 ################################################### ### Version Information ## @@ -437,6 +437,8 @@ User-agent: Iria Disallow:/ User-agent: IRLbot Disallow:/ +User-agent: isitwp.com +Disallow:/ User-agent: Iskanie Disallow:/ User-agent: IstellaBot @@ -797,6 +799,8 @@ User-agent: s1z.ru Disallow:/ User-agent: SalesIntelligent Disallow:/ +User-agent: satoristudio.net +Disallow:/ User-agent: SBIder Disallow:/ User-agent: ScanAlert @@ -849,6 +853,10 @@ User-agent: SISTRIX Disallow:/ User-agent: Sitebeam Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ User-agent: SiteExplorer Disallow:/ User-agent: Siteimprove