diff --git a/.dev-tools/_robots_input/robots-input.txt b/.dev-tools/_robots_input/robots-input.txt index 22db28765..b7febc356 100644 --- a/.dev-tools/_robots_input/robots-input.txt +++ b/.dev-tools/_robots_input/robots-input.txt @@ -277,7 +277,6 @@ Mr.4x3 MSFrontPage MSIECrawler Msrabot -MS Web Services Client Protocol muhstik-scan Musobot Name Intelligence diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index fa023001e..8a81cdc81 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1808 -### Updated: Fri Aug 16 11:07:57 SAST 2019 +### Version: V3.2019.08.1809 +### Updated: Fri Aug 16 14:26:28 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 746c3f03f..42c86bb7d 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.224.112.202 1; +35.193.7.13 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 500584a94..e42e64139 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.224.112.202 0; +35.193.7.13 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index fa023001e..8a81cdc81 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1808 -### Updated: Fri Aug 16 11:07:57 SAST 2019 +### Version: V3.2019.08.1809 +### Updated: Fri Aug 16 14:26:28 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 746c3f03f..42c86bb7d 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.224.112.202 1; +35.193.7.13 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 500584a94..e42e64139 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.224.112.202 0; +35.193.7.13 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index fa023001e..8a81cdc81 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1808 -### Updated: Fri Aug 16 11:07:57 SAST 2019 +### Version: V3.2019.08.1809 +### Updated: Fri Aug 16 14:26:28 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 746c3f03f..42c86bb7d 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.224.112.202 1; +35.193.7.13 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 81227f5b8..8b33ddea3 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -338,7 +338,6 @@ "~*(?:\b)MSFrontPage(?:\b|)" 0; "~*(?:\b)MSIECrawler(?:\b|)" 0; "~*(?:\b)Msrabot(?:\b|)" 0; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 0; "~*(?:\b)muhstik-scan(?:\b|)" 0; "~*(?:\b)Musobot(?:\b|)" 0; "~*(?:\b)Name\ Intelligence(?:\b|)" 0; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 500584a94..e42e64139 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.224.112.202 0; +35.193.7.13 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index fa023001e..8a81cdc81 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1808 -### Updated: Fri Aug 16 11:07:57 SAST 2019 +### Version: V3.2019.08.1809 +### Updated: Fri Aug 16 14:26:28 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index fa023001e..8a81cdc81 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1808 -### Updated: Fri Aug 16 11:07:57 SAST 2019 +### Version: V3.2019.08.1809 +### Updated: Fri Aug 16 14:26:28 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index c1392c506..c02a7ef97 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1808 -### Updated: Fri Aug 16 11:07:56 SAST 2019 +### Version: V4.2019.08.1809 +### Updated: Fri Aug 16 14:26:27 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index c0fc69668..503b1de7a 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.85.2 0; +35.224.112.202 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index c1392c506..c02a7ef97 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1808 -### Updated: Fri Aug 16 11:07:56 SAST 2019 +### Version: V4.2019.08.1809 +### Updated: Fri Aug 16 14:26:27 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 5e7c0ed8e..465ec1626 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.184.226.236 1; +35.188.1.99 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 3e60bde86..290c48e19 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.184.226.236 0; +35.188.1.99 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index c1392c506..c02a7ef97 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1808 -### Updated: Fri Aug 16 11:07:56 SAST 2019 +### Version: V4.2019.08.1809 +### Updated: Fri Aug 16 14:26:27 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 5e7c0ed8e..465ec1626 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.184.226.236 1; +35.188.1.99 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 3e60bde86..290c48e19 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.184.226.236 0; +35.188.1.99 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index c1392c506..c02a7ef97 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1808 -### Updated: Fri Aug 16 11:07:56 SAST 2019 +### Version: V4.2019.08.1809 +### Updated: Fri Aug 16 14:26:27 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 5e7c0ed8e..465ec1626 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.184.226.236 1; +35.188.1.99 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 81227f5b8..8b33ddea3 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -338,7 +338,6 @@ "~*(?:\b)MSFrontPage(?:\b|)" 0; "~*(?:\b)MSIECrawler(?:\b|)" 0; "~*(?:\b)Msrabot(?:\b|)" 0; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 0; "~*(?:\b)muhstik-scan(?:\b|)" 0; "~*(?:\b)Musobot(?:\b|)" 0; "~*(?:\b)Name\ Intelligence(?:\b|)" 0; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 3e60bde86..290c48e19 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.184.226.236 0; +35.188.1.99 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index c1392c506..c02a7ef97 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1808 -### Updated: Fri Aug 16 11:07:56 SAST 2019 +### Version: V4.2019.08.1809 +### Updated: Fri Aug 16 14:26:27 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 16f37cf31..739e5138b 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.193.7.13 0; 35.192.85.2 0; +35.224.112.202 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 95c78cfda..c3dfdc02d 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1808 -### Updated: Fri Aug 16 11:07:56 SAST 2019 +### Version: V4.2019.08.1809 +### Updated: Fri Aug 16 14:26:27 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index c95bc9ad9..3618b5e9a 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.188.1.99 0; -35.193.7.13 0; +35.202.145.110 0; +35.184.226.236 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 7f8ee15fc..d862f9ad0 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1808 -### Updated: Fri Aug 16 11:07:56 SAST 2019 +### Version: V4.2019.08.1809 +### Updated: Fri Aug 16 14:26:27 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index a53d3a500..b88d4faf7 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.184.226.236 0; -35.193.7.13 0; -35.202.145.110 0; +35.224.112.202 0; +35.192.85.2 0; +35.188.1.99 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index c1392c506..c02a7ef97 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1808 -### Updated: Fri Aug 16 11:07:56 SAST 2019 +### Version: V4.2019.08.1809 +### Updated: Fri Aug 16 14:26:27 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 5bfb5e340..926bb913d 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.188.1.99 0; 35.224.112.202 0; +35.193.7.13 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index c1392c506..c02a7ef97 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1808 -### Updated: Fri Aug 16 11:07:56 SAST 2019 +### Version: V4.2019.08.1809 +### Updated: Fri Aug 16 14:26:27 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 746c3f03f..42c86bb7d 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.224.112.202 1; +35.193.7.13 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/blacklist-user-agents.conf b/.dev-tools/test_units/blacklist-user-agents.conf index 81227f5b8..8b33ddea3 100644 --- a/.dev-tools/test_units/blacklist-user-agents.conf +++ b/.dev-tools/test_units/blacklist-user-agents.conf @@ -338,7 +338,6 @@ "~*(?:\b)MSFrontPage(?:\b|)" 0; "~*(?:\b)MSIECrawler(?:\b|)" 0; "~*(?:\b)Msrabot(?:\b|)" 0; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 0; "~*(?:\b)muhstik-scan(?:\b|)" 0; "~*(?:\b)Musobot(?:\b|)" 0; "~*(?:\b)Name\ Intelligence(?:\b|)" 0; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 906d58d35..73225b764 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -CrazyWebCrawler -DIIbot -eCatch -Lmspider -Nibbler -Recorder -serpstatbot -SlySearch -Sqlworm -ZmEu +Alligator +Cogentbot +Cosmos +DittoSpyder +Pimonster +Searchestate +Snake +Sottopop +Szukacz +YoudaoBot diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 1b20fb91c..d6b1a4043 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ +404checker 404enemy -Aboundex +Abonti +Acunetix AhrefsBot -AIBOT +Alexibot +Alligator AlphaBot -Anarchie -Apexoo archive.org_bot -Asterias -Backlink-Ceck -BackStreet +ASPSeek +backlink-check +Bandit Barkrowler -BatchFTP +Battleztar Bazinga BBBike -Bigfoot +BDFetch Bitacle Black Hole -Blow -BlowFish -Brandprotect -BuiltBotTough -Bullseye -BunnySlippers +BlackWidow +BotALot +Brandwatch +BuzzSumo +Calculon CATExplorador -Cegbfeieh -CheeseBot +CazoodleBot CherryPicker -CheTeam ChinaClaw -Cloud mapping -cognitiveseo +Claritybot Collector -com.plumanalytics -Cosmos +CopyRightCheck Craftbot -crawl.sogou.com -CrunchBot CSHttp Curious -Digincore +DatabaseDriverMysqli +DataCha0s +DBLBot +demandbase-bot +Deusu +Devil DIIbot -Dirbuster +Discobot Dispatch DittoSpyder -DnyzBot -DomainCrawler -DomainStatsBot +DomainAppender Drip -DSearch -ECCP/1.0 -Ecxi -EMail Siphon -Exabot -Extreme Picture Finder -Ezooms +Ebingbong +eCatch +EirGrabber +evc-batch facebookscraper -FHscan Fimap Firefox/7.0 FlashGet +Flunky Foobot +Freeuploader +FrontPage FyberSpider GalaxyBot GermCrawler +Getintent GetRight -GetWeb +Gigablast Gigabot -G-i-g-a-b-o-t -Gotit -GrabNet +Go-Ahead-Got-It +Grabber GrapeFX -GrapeshotCrawler -GT::WWW Haansoft +HaosouSpider Harvest -HTMLparser +HEADMasterSEO +Hloader +HMView HTTP::Lite -HTTrack -IDBot -Id-search -IlseBot -Image Fetch +Iblog IndeedBot Indy Library -Internet Ninja -internetVista monitor -Iria -JamesBOT +InfoNaviRobot +InfoTekies +instabid +InterGET +IRLbot JennyBot +JetCar JikeSpider -Joomla -Jorgee -Jyxobot -Keyword Density +JustView +Kozmosbot +Lanshanbot +Larbin LeechFTP -LeechGet LexiBot +Lftp LibWeb Libwhisker +Likse LinkextractorPro -LinkpadBot LinkScan -Lipperhey Spider -Litemage_walker -Lmspider +LinqiaScrapeBot +Lipperhey LNSpiderguy -Mag-Net +Ltx71 +LWP::Simple +lwp-trivial +Magnet magpie-crawler +Mail.RU_Bot +Majestic12 +Majestic SEO +MarkMonitor MarkWatch -meanpathbot -Meanpathbot +Mass Downloader +Mata Hari +MauiBot MeanPath Bot -Mediatoolkitbot mediawords MegaIndex.ru -Metauri -MIDown tool +MFC_Tear_Sample +MIIxpc Mister PiX -Mojeek +MJ12bot +Morfeus Fucking Scanner MSFrontPage MSIECrawler -Msrabot -MS Web Services Client Protocol -muhstik-scan Musobot -Nameprotect NearSite Nessus NetAnts Netcraft netEstate NE Crawler NetSpider -Nettrack -Net Vampire -NetZIP NextGenSearchBot Nibbler NICErsPRO +Nikto +Nimbostratus Ninja -Octopus +NPbot +oBot Offline Explorer Offline Navigator -OnCrawl -OpenVAS -OrangeBot -OrangeSpider +Openvas OutclicksBot -page scorer -PageScorer -Panscient +PageAnalyzer +Page Analyzer +PageGrabber pcBrowser PECL::HTTP -PHPCrawl -Picscout -Pimonster +PeoplePal +PictureFinder +Pixray PleaseCrawl -Pockey POE-Component-Client-HTTP -Psbot +ProWebWalker +Pump PxBroker +PyCurl QueryN Metasearch RankActive -RankActiveLinkBot -RankingBot +RankFlex +RankurBot +RealDownload Reaper -RedesScrapy -ReGet +Recorder +Ripper +RocketCrawler Rogerbot SalesIntelligent +SBIder ScanAlert -ScoutJet -Searchestate -Semrush -SemrushBot -SEOkicks-Robot -SEOlyticsCrawler -SEOprofiler -sexsearcher +Scanbot +Screaming +SEOkicks +seoscanners +SEOstats Siphon SISTRIX Sitebeam -SiteLockSpider -SiteSnagger -SiteSucker +SiteExplorer +Siteimprove Site Sucker Sitevigil -SmartDownload SMTBot -Snapbot -Snoopy -SocialRankIOBot -Sociscraper +Sogou web spider Sosospider -Sottopop -SpaceBison -Spammen -SpankBot -sp_auditbot -SputnikBot +Spanner +spyfu Sqworm -Steeler -Stripper -Sucker Sucuri -SuperBot -Suzuran +SuperHTTP Swiftbot +sysscan +Szukacz T0PHackTeam -T8Abot +TeleportPro Telesoft The Intraformant +Titan Toweyabot -Tracemyfile -Trendictionbot +trendiction.com +True_Robot +Turnitin TurnitinBot TwengaBot +Twice Typhoeus -Upflow -VeriCiteCrawler -Virusdie -Voil +URLy.Warning +Vacuum +Vagabondo +VB Project +VidibleScraper +Voltron +Wallpapers/3.0 WASALive-Bot -WebAuto +Webalta +WebBandit WebCollage +Web Collage +WEBDAV Web Fetch -Web Fuck +WebFuck WebImageCollector +WebLeacher WebmasterWorldForumBot +webmeup-crawler WebPix -WebsiteExtractor -WebsiteQuester +WebSauger Website Quester +WebStripper WebWhacker +WebZIP +WeSEE Whacker Whatweb Who.is Bot +Widow +WinHTTrack +WiseGuys Robot WISENutbot -Wprecon -WWW-Collector-E +Wonderbot +WPScan WWW::Mechanize -Xaldon WebSpider -Xaldon_WebSpider -Zade -Zauba +WWWOFFLE +x22Mozilla +Xenu +xpymep1.exe zauba.io Zeus +zgrab Zitebot -ZmEu -ZumBot ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index e5b7254d6..ca6a89b3e 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -360Spider +404checker 80legs -Aboundex +Abonti Aboundexbot Acunetix -AfD-Verbotsverfahren +ADmantX AIBOT -Apexoo +AiHitBot +Aipbot +AlphaBot +Asterias Attach Backlink-Ceck BacklinkCrawler -BackWeb -Badass +BackStreet +Bandit Barkrowler BatchFTP -Battleztar Bazinga -BetaBot +BDFetch Blackboard -Black Hole BotALot Brandprotect -Buddy -BuiltBotTough -BuiltWith -BuzzSumo +Brandwatch Calculon +CazoodleBot CCBot -Claritybot +Cegbfeieh +CheeseBot +ChinaClaw +Cloud mapping +coccocbot-web Cogentbot -Copier +cognitiveseo +Collector CopyRightCheck -Copyscape -crawler.feedback -crawl.sogou.com -Crescent -CrunchBot +crawler4j Curious Custo DatabaseDriverMysqli -DataCha0s -DBLBot -Deusu -Digincore +demandbase-bot DIIbot -Disco +Dirbuster Discoverybot -DittoSpyder +DnyzBot DomainSigmaCrawler -DomainStatsBot -Dragonfly -DSearch -DTS Agent +Drip +EasyDL Ebingbong ECCP/1.0 -EMail Wolf -EroCrawler +EirGrabber evc-batch Evil +Exabot +Express WebPictures +ExtLinksBot Extreme Picture Finder +EyeNetIE Ezooms -facebookscraper -FDM -FemtosearchBot -Firefox/7.0 +Fimap FlashGet +Freeuploader +FrontPage GalaxyBot -Genieo -GermCrawler Getintent -Gigablast -Gigabot -Go-Ahead-Got-It -GrapeFX +GetRight +GetWeb +G-i-g-a-b-o-t +GridBot +GT::WWW +Haansoft HaosouSpider -Harvest -HEADMasterSEO -Heritrix Hloader -HTTP::Lite -HTTrack +HTMLparser +Humanlinks HybridBot Iblog -IDBot -IlseBot -Image Fetch -Image Sucker IndeedBot +Indy Library InfoTekies +instabid Intelliseek +InterGET Internet Ninja -InternetSeer -ips-agent -Jbrofuzz -JennyBot -JikeSpider -JOC Web Spider +internetVista monitor +IRLbot Joomla -Jorgee +Jyxobot Kenjin Spider +Keyword Density Kozmosbot -LeechGet +Larbin +LeechFTP +Libwhisker Linkdexbot -LinksManager +LinkextractorPro +LinkpadBot +LinqiaMetadataDownloaderBot LinqiaRSSBot -LinqiaScrapeBot -Lipperhey Lipperhey Spider -Litemage_walker Lmspider Ltx71 LWP::Simple -Mag-Net +Magnet magpie-crawler -Majestic12 +Mail.RU_Bot +Majestic-SEO MarkMonitor MarkWatch +Mass Downloader Mata Hari +meanpathbot Meanpathbot -MeanPath Bot -Microsoft Data Access +MFC_Tear_Sample +Microsoft URL Control MIDown tool -MJ12bot -MS Web Services Client Protocol +Mister PiX +Morfeus Fucking Scanner +MSIECrawler Nameprotect -NearSite -Needle -NetAnts -Nettrack -NetZIP -NextGenSearchBot -Niki-bot -NimbleCrawler -Ninja -NPbot -Nutch -oBot +Nessus +netEstate NE Crawler +NetMechanic +Netvibes +Nibbler +NICErsPRO +Nikto +Nimbostratus OnCrawl -Openfind -OpenLinkProfiler Openvas -OpenVAS -OrangeBot OrangeSpider +OutclicksBot +OutfoxBot Page Analyzer +PageGrabber PageScorer +Pandalytics +Panscient Papa Foto -PeoplePal -Picscout +PECL::HTTP Pi-Monster -Pixray +Pockey POE-Component-Client-HTTP Probethenet ProPowerBot ProWebWalker Pump -PyCurl +PxBroker +Quick-Crawler RankActive -RankActiveLinkBot +RankFlex Rankivabot +RealDownload +Reaper RebelMouse -Recorder +ReGet RepoMonkey +Ripper RocketCrawler +Rogerbot RSSingBot s1z.ru -SBIder +SalesIntelligent ScanAlert -Scanbot -ScoutJet -SemrushBot +Scrapy +Screaming +ScreenerBot +Searchestate +SearchmetricsBot +Semrush SEOlyticsCrawler -Seomoz seoscanners SeoSiteCheckup -SEOstats -sexsearcher +Shodan SISTRIX -Sitebeam -SiteExplorer -SiteLockSpider +Siteimprove +SiteSucker +Site Sucker +Sitevigil SlySearch -SmartDownload SMTBot Snake -Snapbot -Snoopy -SocialRankIOBot Sociscraper sogouspider Sogou web spider +Sottopop +SpaceBison +Spammen SpankBot -Spanner -Spbot +sp_auditbot SputnikBot spyfu +Sqlmap Sqlworm -Sqworm -Steeler -Sucuri +Surfbot SurveyBot +Suzuran Swiftbot +sysscan Szukacz T0PHackTeam -T8Abot Teleport TeleportPro -Telesphorep -TightTwatBot +Titan +Toweyabot Trendiction -Trendictionbot trendiction.com -TurnitinBot +trendiction.de +Turnitin TwengaBot -Typhoeus -Upflow -Vagabondo -VCI -VeriCiteCrawler -Virusdie +URLy.Warning +VB Project +VidibleScraper VoidEYE -Voltron +Wallpapers/3.0 +WASALive-Bot +WBSearchBot +Webalta WebAuto -WebCopier +Web Auto +WebCollage +Web Collage WebFuck WebGo IS +WebImageCollector +WebLeacher WebmasterWorldForumBot -webmeup-crawler -WebSauger +WebPix Web Sauger -WebsiteExtractor -Website Quester -WebStripper +Webshag +WebsiteQuester +Webster WebSucker -Whack +WebWhacker +WebZIP Whacker -Widow -WinHTTrack -WWW-Collector-E -WWW::Mechanize +Who.is Bot +Wonderbot +Wotbox +WPScan +WWW-Mechanize +WWWOFFLE x22Mozilla -Xaldon WebSpider Xaldon_WebSpider -xpymep1.exe -Zade +Xenu Zauba zauba.io +Zermelo Zeus -zgrab -ZmEu +ZumBot ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index edfc3738b..5ab8d2c1f 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -1bet.com -aero2.ru -apple.com-webbrowsing-security.review -bupropion-sr-150-mg.us -greatdealshop.com -keywordhut.com -portside.xyz -tdsing.ru -xitjw.info -zlatnajesen.com +4webmasters.com +elektronischezigarettekaufen1.myblog.de +el-nation.com +neobux-bg.info +okmusic.jp +quit-smoking.ga +scansafe.net +sundrugstore.com +vzlomtw.com +yeartwit.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 37e05d263..274c8a519 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -00webcams.com -1-99seo.com -1-free-share-buttons.com -1millionusd.xyz -2345.com -3weekdiet.com -4replicawatch.net -5i2.net -7xc4n.com -7yue.org -98oi.ru -aa9d046aab36af4ff182f097f840430d51.com -aasoldes.fr -absugars.com -admatic.com.tr -adservme.com -adtech.de -adultactioncam.com -adultmeetonline.info -ahmedabadwebs.com -akusherok.ru -alarmobninsk.ru -alert-fdm.xyz -alibestsale.com -alkoravto.ru -amateurgalls.com -anabolics.shop -animal36.com -animebox.com.ua -anticrawler.org -apiappenableinfo-a.akamaihd.net -apibatbrowsecom-a.akamaihd.net -apiholdingmypage-a.akamaihd.net -apparel-offer.com -apple.com-webbrowsing-security.science -arcadeplayhouse.com -arclk.net -arcteryxsale.online -arenanews.com.ua -arius.tech -arkartex.ru -asophoto.com -asrvrep-a.akamaihd.net -atley.eu.pn -avek.ru -avtoarenda.by -ayodhya.co -azartclub.org -balitouroffice.com -balla-boo.se -baltgem.com -bankcrediti.ru -bannerads.de -baoxaydung.com.vn +01apple.com +03e.info +1001watch.com.ua +1flag.co.za +2nt.ru +4kpics.rocks +4webmasters.com +6128786.com +777-club.ru +8xv8.com +aa9bd78f328a6a41279d0fad0a88df1901.com +aadroid.net +abcdefh.xyz +abcdeg.xyz +above.com +acmebtn.ml +adamoads.com +ads-seo.men +ads.tremorhub.com +affrh2015.com +akuhni.by +alekseevec.ru +alfapro.ru +allcredits.su +allfinweb.com +allkrim.com +all-number.com +allornamenti.com +alltheviews.com +amanda-porn.ga +amateurmatch.com +amung.us +antons-transporte.de +aosexkontakte.net +apiluckyleapnet-a.akamaihd.net +ardimobileinfo.ml +arenda-avtoprokat-krasnodar.ru +ascat.porn +asrv-a.akamaihd.net +atelielembrancaqueencanta.com.br +aussie-prizes.com +autovideobroadcast.com +avitocars.ru +ayeartoforget.com +bala.getenjoyment.net +banan.tv +bankmib.ru battleforupdating.site -beauty-clinic.ru -bestadbid.com -bestofpicture.com -bestofupload.info +bdsmgalls.net +beachfix.co +bestfortraders.com +besthomemadepornsites.com best-seo-offer.com -bestserials.com -bestsexyblog.com -beztuberkuleza.ru -bigcities.org -bin-brokers.com -bioca.org -bird1.ru -bjetjt.com -bloggedporn.com -blog.remote-computer.de -bloke.com -boole.onlinewebshop.net +bet-prognoz.com +betwinservice.com +beyan.host.sk +bezlimitko.xyz +bidbuy.co.kr +binomo.kz +bioscorp.ru +bio.trade-jp.net +bjanshee.ru +bjorkbacken.nu +blavia.00author.com +boazpower.com +bobinoz.com +boltalko.xyz +bookhome.info bouda.kvalitne.cz -builtwith.com -burger-imperia.com -burn-fat.ga -buyantiviralwp.com -cacheimages.com -callejondelpozo.es -camdenmemorials.com -candlehandmade.com +braintobe.win +bravegirlsclub.com +brg8.com +bristolhostel.com +bristolhotel.com +browserprotecter.com +btvn.ru +buchananshardware.com +bumble.cheapwebsitehoster.com +buron.pw +buypanicdisorderpill.com +canvas.gloverid.site cardiosport.com.ua cardsharp1.ru -cartierring.xyz -cartierwatch.xyz -cartujano-pre.de -cascadelink.org -caveavins.fr -cb.iphantom.com -cbox.ws -centre-indigo.org.ua -ceotrk.com -cfacarrosserie74.com -chatroulette.life -cheapcigarettesc.info -cheapestjerseys-wholesale.com -cheapjerseysbizwholesale.us -chinese-amezon.com +cartierbracelet.xyz +cartierlovebracelet.xyz +cartierloveringreplica.xyz +cashkitten-a.akamaihd.net +cbozhe.com +cdncash.net +cdncash.org +chatroulette.online +chatroulette.world +cheapjerseysa.com +cheap-trusted-backlinks.com chocolateslim-original.com -cialischstgerts.com -cialisndbrx.com -cjs.com.ru -cleanersoft.com -clicksotrk.com -clips.ua.ac.be -clubfashionista.com -codq.info -compliance-fred.top -compliance-jess.top -compliance-julia.top -compliance-mary.top -compliance-willy.top -coprofam.org -counciltally.com -countbertwistdisp26.soup.io -crd.clan.su -custom-electric-guitar.com -customerguru.in -danceuniverse.ru -dangphoto.trade -dawlenie.com -deadlinkchecker.com +chorus.terakeet.com +cialisovercounteratwalmartusa.com +ciekawinki.pl +circlesl.com +classicakuhni.ru +clayvasedesigns.tk +clicksor.com +clksite.com +club-samodelkin.ru +colehaanoutlet.store +collegeessay19.blogspot.ru +comeondog.info +compliance-alex.top +compliance-josh.top +compliance-mark.top +compliance-viktor.top +contentsexpress.com +converse.ddsoldes.fr +cosmeticswomens-womensports.rhcloud.com +covetnica.com +cpajunkies.com +cranly.net +creatives.adbetclickin.pink +credit-respect.ru +customchocolate.business-for-home.com +customcollegeessays.net +danashop.ru +dating-time-now.com dealwifi.com +deda-moroza-zakaz.ru +dekoration.us dentalpearls.com.au -destinationrealestate.com -dev33.dioniqlabb.se -dfiles.me -diamond-necklace.info -dieswaene.com -digital-sale.su -dikqlyremy.info -directivepub.com -djonwatch.ru -dkvorota.ru -docsportal.net -doeco.ru -dostavimvdom.ru -download-of-the-warez.blogspot.com -downtuptv.gq -drugspowerstore.com -dstroy.su -eandsgallery.com -e-commerce-seo1.com -ekobata.ru -eldiariodeguadalajara.com -elexies.info -elite-sex-finders.com -enews.tech -enthuse.computernetworksonline.com -eropho.com -esfchat.tk -essayassist.com -essay-edu.biz -essayservicewriting.org -estelight.ru -evogarage.com -exchangeit.gq -excitacion.info -exdocsfiles.com -exportshop.us -extlinks.com -extrabot.com -eyelike.com.ua -eyessurgery.ru -ezb.elvenmachine.com -ezigarettekaufen.myblog.de -facebook-mobile.xyz -farm26.ru -fashion-mk.net -fatmaelgarny.com -fba-mexico.com -feel-planet.com -fefo.gdn -fickkontakte.org -fil.ru -firma-legion.ru +deutschehobbyhuren.net +dktr.ru +dnepr-avtospar.com.ua +dogbreedspicture.net +dolohen.com +domainsphoto.com +domain.webkeyit.com +domination.ml +dood.live +dostavka-v-krym.com +dostavka-v-ukrainu.ru +dotnetdotcom.org +easytuningshop.ru +e-collantes.com +editors.choice6912650.hulfingtonpost.com +election.interferencer.ru +electronicadirect.com +elektrischezigarette2.onsugar.com +elektrozigarettekaufen2.blogse.nl +elitedollars.com +encodable.com +energy-ua.com +erotikchat-24.com +escort-russian.com +este-line.com.ua +eu-cookie-law.info +eurosamodelki.ru +event-tracking.com +excitacao.com +exmasters.com +extstat.com +ezigarettenshop2.postbit.com +f0815.de +falllow.gq +fanoboi.com +fashionavenuegame.com +fearcrow.com +fettefrauen.net +fickenprivat.info +fickkontaktehobbyhuren.com +fidelityfunding.com +filesclub.net +fishtauto.ru +fitfloponline.store +florida-tourism.net +foto-basa.com +franch.info free411games.com -free-fb-traffic.com -freemags.cc -freewareseek.com -frivgame250.com -fr.netlog.com -ftns.ru -gambarkatabaru.com -gamebackyard.com -game-top.su -gardene.ru -generalporn.org -genericviagrasildenafiled.net -geoads.com -glcomputers.ru -gojiberriess.apishops.ru +free-deals.faith +freshmac.space +fullgirl.ru +fxgallery.com +gael-s.ru +gagrasector.ru +gallery.rennlist.com +gamedayassist.com +gamedayhouse.com +getlamborghini.ga +getoutofdebtfree.org gok-kasten.net +golden-catalog.pro goldenggames.com -gonextmedia.com -goodhousekeeping.com -goo.ne.jp -gopro-online.info -gosmeb.ru -greamimgo.pw -greenidesign.co -green-tea.tv -gsasearchengineranker.xyz -gsbs.com.ua -gymi.name -hd-film.pl -hdimagelib.com -hdpixent.com -hdpixion.com -healmytrauma.info -help.tpu.ru -helvetia.com.ua -hermesreplica.win -hjaoopoa.top -homedecorpicture.us -homeinns.com -home.myplaycity.com -hoporno.com -hotblognetwork.com -hybrid.ru -hydropump.su -idealtits.net -iklysha.ml -ilmen.net -ilovevitaly.ru -imguramx.pw -img.wallpaperstock.net -imk.com.ua -incest-ru.com -incomekey.net -indianmedicaltourismshop.com +gomusix.com +gq-catalog.gq +gsasearchenginerankerdiscount.com +gta-club.ru +gwebtools.com +gyffu.com +hamacapty.com +hawaiisurf.com +health-medical-portal.info +heatpower.ru +hebr.myddns-flir.com +hem.passagen.se +hentai-manga.porn +hidefiles.org +hildinghr.se +histock.info +homeandhealth.ru +housekuba.org +hoverboard360.es +howtowhitenteethfast.xyz +hully.altervista.org +humanorightswatch.org +humbmosquina.tk +hut1.ru +iccornacircri.cf +ideashome.id +igtools.club +images.gyffu.com +imagui.eu +immobiliaremassaro.com +importchinacoach-teach.com +import-sales.com +impotentik.com +increasewwwtraffic.info instabid.tech -interesnie-faktu.ru -int.search.tb.ask.com -ipornox.xxx -iqoption-bin.com -iscblog.info -iskalko.ru -isoveti.ru -ivoiretechnocom.ci -japan-bearings.ru -jav-fetish.com -jerseychinabizwholesale.us -jestr.org -jetsli.de +instakink.com +int.search.mywebsearch.com +iqoption.com +istanbulit.com +istizanidineanopiate.blogspot.com +itag.pw +itis4you.com +itronics.ca +jabimgo.pw +jackpotchances.com +jackwolfskinoutlet.online +javcoast.com +javidol.site +jjbabskoe.ru jobmarket.com.ua -joingames.org -jose.mulinohouse.co -juliadiets.com -juliaworld.net -justdating.online -kerwinandcariza.com +kemerovo.zrus.org +kevblog.top +keywordglobal.co.za kichenaid.ru -kladrus.ru -kliksaya.com -kupiproday.com.ua -laexotic.com -la-fa.ru +kiskinhouse.com +knogg.net +koral.se +kovesszucs.atw.hu +kozhniebolezni.com +kunstaktien.info +kupit-adenu.ru +kwzf.net +labplus.ru languagecode.com -laptop-4-less.com -laudit.ru -lenvred.org -letmacwork.world -levaquin750mg.blogspot.com -lightinghomes.net -likesuccess.com -loadingpages.me -lollypopgaming.com -lomb.co -longlifelomilomi.info -lookingglassemb.com -machicon-akihabara.info -magda-gadalka.ru -maglid.ru -managerpak204.weebly.com -markjaybeefractal.com -martlinker.com -matpre.top -maxthon.com -mb140.ru -medanestesia.ru -mediaoffers.click -mex-annushka.ru -mielec.pl -mindbox.co.za -minneapoliscopiers.com -mirzonru.net +lankarns.com +larutti.ru +lexixxx.com +libertybilisim.com +life.biz.ua +linkarena.com +lisque.batcave.net +livingcanarias.com +lkbennettoutlet.online +longadventure.com +lovelycraftyhome.com +lyngdalhudterapi.no +mackeeper-land-672695126.us-east-1.elb.amazonaws.com +magazin-pics.ru +mararoom.ru +marketingtechniques.info +matsdale.com +mebel-ekb.com +medicinacom.ru +meetingrainstorm.bid +megakino.net +methodsmarketing.com +mexicotravelnet.com +minegam.com +miniature.io +misandesign.se +mixed-wrestling.ru mmofreegames.online -mobile.ok.ru -mojpreskumanie.com -monetizationking.net -money-every-day.com -morf.snn.gr -mossmesi.com -most-kerch.org -mototsikl.org -movies-in-theaters.net -mp3ringtone.info -mrinsidesales.com -muztops.ru -mxgetcode.com -myprintscreen.com -naj-filmy24.pl -naughtyconnect.com -netfacet.net -net-radar.com -nfljerseyscheapchinabiz.com -nfljerseysforsalewholesaler.com -nikkiewart.ru -nippon-bearings.ru +mobsfun.net +modabutik.ru +moje-recenze.cz +mojowhois.com +monarchfind-a.akamaihd.net +moncleronline.xyz +most.gov.iq +motherboard.vice.com +mrbojikobi4.biz +mrmoneymustache.com +mug-na-chas-moscow.ru +musicas.baixar-musicas-gratis.com +mylesosibirsk.ru +mylida.org +mypornfree.ru +nagdak.ru +nailsimg.com +naphukete.ru +nastroyke.net +naturtreenspicerx.pw +navalwiki.info +netallergy.ru +newsperuse.com +next-dentists.tk +nfljerseyscheapbiz.us +no-rx.info nosecret.com.ua -oddamzadarmo.eu -offergroup.info -okmedia.sk -okonich.com.ua -old-rock.com +nsatc.net +o333o.com +oballergiya.ru +oda.as +okroshki.ru +ok.ru +okuos.com omoikiri-japan.ru -onlinefilmz.net -onlinemegax.com -onlineporno.site -online-sbank.ru -online-x.ru -orel-reshka.net -oren-cats.ru -ostrovtaxi.ru +onefilms.net +onemantrip.com +online.ktc45.ru +onlinemeetingnow.com +onlineslotmaschine.com +o-o-6-o-o.ru +ostroike.org paceform.com -pathwhelp.org -perl.dp.ua -perso.wanadoo.es -piccshare.com -pictures-and-images.net -picturesmania.com -picurams.pw -pinwallpaper.xyz -pisanieprac.info -piulatte.cz -porn555.com -porndroids.com -pornosemki.info -pornotubexxx.name -pornozhara.com -poshiv-chehol.ru -potoideas.us -powenlite24.ru -pravoholding.ru -priscilarodrigues.com.br -privacyassistant.net -privhosting.com -promodj.com -proposal-engine.com -prostitutki-tolyatti.org -psoriasis-file.trade -qitt.ru +paidonlinesites.com +paintingplanet.ru +pandroid.co +paparazzistudios.com.au +pavlodar.xkaz.org +pcads.ru +perosan.com +phuketscreen.com +pics-group.com +picture-group.com +pierrehardysale.online +pixshark.com +poppen-nw.net +porno-chaman.info +pornoklad.ru +preventheadacheguide.info +princevc.com +prizefestival.mobi +professionalwritingservices15.blogspot.ru +promalp-universal.ru +pxhdwsm.com +qualitymarketzone.com +quick-offer.com rada.ru -razyboard.com -rebrand.ly -refererx.com -regionshop.biz -rentehno.ru -rique.host.sk -roma-kukareku.livejournal.com -rootandroid.org -royal-investments.net -russian-postindex.ru -russia-tao.ru -sajatvelemeny.com -sammlungfotos.online -savefrom.com -sbornik-zakonov.ru -scanner-fred.top -scanner-george.top -scanner-jess.top -scrnet.biz.ua -sc-specialhost.com -searchimpression.com -searchlock.com -seasaltwithfood.com -seocheckupx.com -seorankinglinks.us -seorankinglinks.xyz -seryeznie-znakomstva.ru -sexad.net -sexs-foto.com -sexytrend.ru -shillyourcoins.com -shinikiev.com.ua -shlyahten.ru +rebuildermedical.com +reckonstat.info +redirect.trafficreceiver.club +reklama1.ru +reklama-i-rabota.ru +relayblog.com +renecaovillasale.online +replicalouboutin.xyz +resant.ru +resellerclub.com +rezeptiblud.ru +rfd-split.hr +rvi.biz +safe-app.net +sardinie.us +satellite.maps.ilovevitaly.com +scanmyphones.com +scanner-barak.top +scanner-walt.top +seansonline24.pl +secretscook.ru +seksvideoonlain.com +semprofile.com +seo-tools-optimizing.com +serialsway.ucoz.ru +servethis.com +sevendays.com.ua +sexkontakte-seite.com +sexy.babes.frontend-stack.top +sexy-screen-savers.com +share-buttons-for-free.com shoesonlinebuy.cn -shymkent.xkaz.org -signal03.ru -site-speed-checker.site -skladvaz.ru -sladkoevideo.com -slavia.info -slimcdn.com -socas.pluto.ro -socialbuttons.xyz -sofit-dmd.ru -spacash.com -sportbetfair.com -sps-shop.com -statustroll.com -stauga.altervista.org -superlist.biz -surcentro.com -surgut.zrus.org -susanholtphotography.com -swingerseiten.com -takethatad.com -tanieaukcje.com.pl -tarad.com -td-33.ru -tech4master.com -template-kid.com -terraclicks.com -tfxiq.com -thebluenoodle.com -thegolfclub.info -tiens2010.ru -titan-cloud.life -titelhelden.eu -tizanidine4mg.blogspot.com -tizanidine4mgtablets.blogspot.com +sigmund-freud.co.uk +sispe.com.br +site-speed-check.site +sitevaluation.org +s-iwantyou.com +slavkokacunko.de +smsactivator.ru +softlinesolutions.me +softonicads.com +solitaire-game.ru +solution4u.com +songoo.wz.cz +spb-plitka.ru +specialfinanceoffers.com +spravka130.ru +spywarebegone.com +spywareit.com +squidoo.com +standardchartered-forex.com +statistici.ro +sticken.co +streetfire.net +studiofaca.com +tandvardshuset.net +teguh.info +tetracsaudi.com +thefotosgratis.eu +timdreby.com tizanidine6mg.blogspot.com -tizanidinecost.blogspot.com -tizanidinepills.blogspot.com -tizanidineshowupondrugtest.blogspot.com -tizanidinesideeffects.blogspot.com -tobeyouday.win -token-lab.org -toloka.hurtom.com -top1-seo-service.com -topdownloads.ru -top-study.work -toyota.7zap.com -traffic100.com -traffic2money.com -trubywriting.com -ttsq.fr -tubo360.com +tizanidineandhydrocodone.blogspot.com +tizanidinecapsules.blogspot.com +tizanidinedosageforsleep.blogspot.com +tizanidinehcl2mgsideeffects.blogspot.com +tizanidineingredients.blogspot.com +tizanidineinteractions.blogspot.com +tizanidineonline.blogspot.com +tizanidinevsvalium.blogspot.com +toondinsey.com +top10registrycleaners.com +trafficmania.com +tubeoffline.com tupper-posuda.ru -twittruth.com -twu.com.ua -uasb.ru -uchebavchehii.ru -ugguk.online -unblocksit.es +tvand.ru +tversvet.ru +typimga.pw +u555u.info +udsgame.online +uggsale.online +ugogo.info underthesite.com -unmaroll.ya.ru -urengoy.pro -userequip.com -vapeface.club -varbergsvind.se -v-doc.co -veles.shop -veselokloun.ru -vezuviy.su -videotuber.ru -vip.51.la -vipps.com.my -vipsexfinders.com -vipsiterip.org -vkgaleria.com -vkonche.com -vrnelectro.ru -w7s.ru -website-speed-check.site -wedding0venues.tk -welck.octopis.com -whereiskentoday.com +unionmarkt.de +unrealcommander.org +uptimechecker.com +upupa.net +urldelivery.com +url-img.link +uroki.net +us-america.ru +vacuumscleaner.com +vasileostrovsky-rayon.ru +vchulkax.com +vduplo.ru +vedomstvo.net +verymes.xyz +via-energy-order.com +videtubs.pl +vielporno.net +vip-dom.in +vladimir.xrus.org +vpnmouse.com +vriel.batcave.net +vrotike.ru +vzlomfb.com +wallpapers-best.com +watch-movies.ru +waterefficiency.co +webalan.ru +webjam.com +webmasters.stackexchange.com +weclipart.com +wedding-salon.net +weprik.ru where-toget.com -whereverdesperate.gq -wholesalejerseychinaoutlet.com -wholesalenfljerseys.us.com -wineration.com -worldinternetauthority.com -worldoffiles.ru -wowas31.ucoz.ru -woweb.com.ua -wwwadultcheck.com -www.kabbalah-red-bracelets.com -x-diesel.com +whiteproduct.com +wildworld.site +wiosenny-bon-1500.pl +worldhistory.biz +wygraj-skiny.win +xboxster.ru xev.ru -xjlottery.com -x-musics.com -xxxrus.org -yougetsignal.com -your-bearings.com -yourdesires.ru -yuarra.pluto.ro -zapiszto.pl -zarabotat-na-sajte.ru -zarabotok--doma.ru -zastroyka.org -zawyna.ua -zeroredirect6.com -zeroredirect8.com -zetgie.com.pl -zocaparj.kz -zona-aqua.ru +xgames-04.com +xlolitka.com +xlovecam.com +xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai +xn----7sbho2agebbhlivy.xn--p1ai +xn--80aanaardaperhcem4a6i.com +xn--80ajjbdhgmudixfjc8c5a9df8b.xn--p1ai +xn----9sbubg3ambdfl1j.xn--p1ai +xn--b1ag5cfn.xn--p1ai +xn----ctbigni3aj4h.xn--p1ai +xn----dtbndd4ae7eub.top +xn--hxazdsfy.blogspot.com +xn--l1aengat.xn--p1ai +xolodremont.ru +xxx-cam.webcam +xxxguitars.com +xxxnatelefon.ru +yogamatsexpert.com +youbrainboost.asia +youporn-forum.ga +you-shall-not-pass.is74.ru +yuweng.info +zaapplesales.blogspot.com +zarabiaj-dzis.pl +zarenica.net +zeroredirect5.com +zoogdiesney.com +zoosexart.com +zx6.ru +zzbroya.com.ua diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index 8b3ae0faa..16fe23ba1 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -08800.top -0lovespells0.blogspot.com -2728fb936f0.com -4rent.ru -8xv8.com -abiente.ru -abovetherivernc.com -accessoires-mode.in -addbin.men -adelly.bg -adloads.net -angkortours.vn -animal-drawings.com -aosheng-tech.com -apiwebwebgetcom-a.akamaihd.net -arenanews.com.ua -as5000.com -ausmepa.org.au -autoseo-traffic.com +2020iscoming.info +2345.com +5-steps-to-start-business.com +999webdesign.com +agadelha.com.br +alfa-medosmotr.ru +aliexpress.com +amateurmatch.com +ample-awards-today.us +anaksma.info +animebox.com.ua +anmysite.com +a.pr-cy.ru +arclk.net +arewater.com +arkartex.ru +artefakct.com +artpicso.com +asdfg.pro +asscenihotosidea.blogspot.com +aviapanda.ru avtolombard-krasnodar.com -awency.com -axbocz.net -bambi.ck.ua -banan.tv -baoxaydung.com.vn -beachtoday.ru -beremenyashka.com -betwinservice.com -bfz.biz -bizru.info -blobar.org -blogos.kz -brakehawk.com -brus.city -bugof.gq -buy-meds24.com -buzzurl.jp -canacopegdl.com -candypeople.se -celejihad.info -cete.ru -cialischmrx.com -cibpenonptib.flu.cc -citetick.com -classiquebijoux.ru -clickaider.com -coin-hive.com -compliance-barak.xyz -compliance-fred.xyz -compliance-olga.top -compliance-walter.top -copyrightclaims.org -counter.yadro.ru -creditmoney.com.ua -culmimg.pw -customergrowthsystems.com -cutalltheshit.com -dailystrength.org -daisye.top -daooda.com -data-mining.tk -ddospanel.com -dickssportinggoods.com -dildofotzen.net -docsportal.net -dogclothing.store -domcran.net -downtuptv.gq -ecommerce-seo.org -ekspertmed.com -emailaccountlogin.co -embedle.com -essaypro.com -eugenevaultstorage.com -expresstoplivo.ru -familyphysician.ru -fatfasts-4tmz.com -felizporno.com -filesmonster.porn -filmgo.ru -findpics.pw -flashahead.info -f-loaded.de -forsex.info -fortevidyoze.net -front.ru -fyl.com.ru -gerhardhealer.com -getyourimage.club -gifspics.com -glicol.kz -gta-club.ru -gwagka.com -gwebtools.com -hoste.octopis.com -hostsshop.ru -hotdl.in -housediz.com -humanorightswatch.org -imperiafilm.ru -inboundlinks.win -incest-ru.com -infogame.name -infostatsvc.com -inspiring-desperate.tk -instakink.com +back.dog +baksman.com +battleforupdating.site +bbsoldes.fr +beachdriveblog.com +beauty-b0x.pl +bestdooz.com +bestwrinklecreamnow.com +blogqpot.com +boltalko.xyz +bouda.kvalitne.cz +btc4u.ru +buypuppies.ca +buytizanidineonline.blogspot.com +cactussoft.biz +cand.jp +carabela.com.do +cardsdumps.com +cardul.ru +carloans.com +cashonads.com +casinos4dummies.co.uk +cdncash.net +cerev.info +cjs.com.ru +cleanallvirus.com +clickso.com +clicksor.net +cobaltpro.ru +codq.info +companies-catalog.com +compliance-irvin.xyz +cookie-law-enforcement-ee.xyz +counciltally.com +cs-passion.pl +cupday.com +delfin-aqua.com.ua +detki-opt.ru +diminishedvalueoforegon.com +djonwatch.ru +domain2008.com +domik-derevne.ru +downloader12.ru +dpihatinh.gov.vn +droidlook.net +dtnlyss.com +earncash.com.ua +e-c.al +e-commerce-seo1.com +elektrischezigarette2.onsugar.com +elektrozigarettekaufen2.blogse.nl +enskedesquashclub.se +eropornosex.ru +eurosamodelki.ru +evogarage.com +exchanges-bet.com +excitacion.info +ezigaretteshop2.vefblog.net +fastfixing.tech +fenoyl.batcave.net +fidelityfunding.com +filmbokep69.com +filmci.pro +filmidivx.com +filunika.com.ru +flowwwers.com +forum20.smailik.org +forum.doctissimo.fr +free-deals.faith +freelifetimefuckbook.com +freemags.cc +free-share-buttons.com +frustrated-favorable.gq +fuckbuddybestgilf.info +gambarxkata.co +garciniaxt.us +gazobeton-p.com.ua +global-ics.co.za +gok-kasten.net +golfresa.lucania.se +google-liar.ru +gosreg.amchs.ru +growmyfunds.ca +guitar-master.org +handicapbathtubarea.com +hatdc.org +hchha.com +hdhc.ru +homelygarden.com +houston-vikings.com +hoverboard360.de +howlongdoestizanidinestayinyoursystem.blogspot.com +ilte.info ivanstroi.ru -ivoiretechnocom.ci javlibrary.cc -jerseywholesalebizchina.com -justbcause.com -kalandranis.gr -keenoutlet.online -keywords-monitoring-success.com -kicknights.gq -kino-key.info -kosynka-games.ru -larger.io -letmacwork.world -lflash.ru -linksharingt.com -lomb.co -lookingglassemb.com -lumb.co -malwareremovalcenter.com -manyresultshub-a.akamaihd.net -matb3aa.com -matpre.top -metaxalonevstizanidine.blogspot.com -mobilemedia.md -modenamebel.ru -moje-recenze.cz -money-every-day.com -muschisexbilder.com -mustat.com -myfreemp3.eu -myliveblog.ru -myseoconsultant.com +jav-way.site +javxxx18.com +jennyfire.ru +j-times.ru +juliaworld.net +kakadu-interior.com.ua +kihi.gdn +kiwi237au.tk +komp-pomosch.ru +kwzf.net +lashstudia.ru +legalrc.biz +linkpulse.com +lion.misba.us +loadopia.com +lostfilm-online.ru +louboutinreplica.xyz +lyrster.com +m5home.ru +mackeeper-land-672695126.us-east-1.elb.amazonaws.com +martlinker.com +matchpal-a.akamaihd.net +mazda-roadsters.com +mediafresh.online +menstennisforums.com +minecraft-neo.ru +mir-business-24.ru +mistr-x.org +mmofreegames.online +mobplayer.ru +musicspire.online +mygameplus.com +myhealthcare.com +mylesosibirsk.ru +narkologiya-orel.ru +naturalpharm.com.ua net-profits.xyz -nevansk.ru -oddamzadarmo.eu -o.light.d0t.ru -onlineku.com -onlineserialy.ru -osb.se11.ru +njpalletremoval.com +nofreezingmac.work +nowtorrents.com +oecnhs.info +ok.ru +oneclickfiles.com +one-gear.com +onthemarch.co +ooo-gotovie.ru +orion-code-access.net +osnova3.ru ossmalta.com -paleohub.info -panamaforbeginners.com -paperwritingservice17.blogspot.ru -partnersafe.men -pawli.eu -paydayloanslocal.com -petedrummond.com -physfunc.ru -picturesify.com -pijoto.net -pintattoos.com -plastweb.ru -playmsn.com -playtap.us -poffet.net -popup-jdh.xyz -pornoelita.info -pornogratisdiario.com -pornonik.com -pornosee.info -portside.xyz -privatbank46.ru -profolan.pl -proftests.net -prombudpostach.com.ua -prostitutki-novgoroda.org -puzzleweb.ru -pyramidlitho.webs.com -rabotaetvse.ru -realting-moscow.ru -recipedays.com -referencemoi.com -reklamuss.ru -repeatlogo.co.uk -retreatia.com -ringporno.com -rniaeba.ga -robertefuller.com -rossmark.ru -rusprostitute.com -satellite.maps.ilovevitaly.com -scanner-jack.top -secretscook.ru +pendelprognos.se +perm.xrus.org +pictures-and-images.com +picturesmania.com +pigrafix.at +plohaya-kreditnaya-istoriya.ru +pl-vouchers.com +podshipniki-ntn.ru +popunder.net +pornogad.com +portal-eu.ru +pretty-mart.com +pricheski-video.com +princeadvantagesales.com +propranolol40mg.blogspot.com +purplesphere.in +qwesa.ru +radiodigital.co +rapidgator-porn.ga +reeyanaturopathy.com +registry-cleaner.net +remontvsamare.su +rfserial.net +roleforum.ru +russia-tao.ru +sale-japan.com +savefrom.com +sbt-aqua.ru +scanner-olga.top +search.1and1.com securityallianceservices.com -segol.tv -seocheckupx.com -seofied.com -serpstat.com -share-buttons-for-free.com -shtaketniki.kz -sibdevice.ru -sigmund-freud.co.uk -signoredom.com -sildenafil-tadalafil.info -skypasss.com -smartadserver.com -smart-scripts.com -snjatie-geroinovoy-lomki.ru -social-search.me -sports-supplements.us -stjamesschool.info -sugarkun.com -sumo.com -superiends.org -swinon.site -templates.franklinfire.co -thegameriders.com -thegamerznetwork.com -the-torrent-tracker.blogspot.com -tizanidineandhydrocodone.blogspot.com -top10-online-games.com -trustl.life -twu.com.ua -tyumen.xrus.org -ucsol.ru -udayavani.com -urldelivery.com -usiad.net -varbergsvind.se -videochat.cafe -villacoloniale.com -vk-mus.ru -vzlomtw.com -watchdogs-2.ru +sel-hoz.com +service.adtech.fr +sex-watch.com +sfd-chess.ru +shop-electron.ru +sindragosa.comxa.com +siteaero.com +sklad-24.ru +sluganarodu.ru +social-s-hhh.xyz +softonicads.com +sotechco.co +spidtest.space +spyfu.com +stal-rulon.ru +steelmaster.lv +stpicks.com +success-seo.com +suggest-keywords.com +sygraem.com +t3chtonic.com +tackletarts.co +thefarmergame.com +the-trader.net +tizanidineduringpregnancy.blogspot.com +tizanidinefibromyalgia.blogspot.com +tizanidinehclsideeffects.blogspot.com +todohr.com +torrentdownloadhub.com +trafficgenius.xyz +traffixer.com +upproar.com +usadacha.net +veopornogratis.xxx +videojam.tv +visionwell.com.cn +vkontarkte.com +watch-movies.ru +waterpurifier.club weboptimizes.com -webstatsdomain.org -wholesalejerseyscheapjerseys.us.com -wleuaprpxuvr.ga -worldinternetauthority.com -wsgames.ru -wurr.voila.net -xfluro.com -xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai -xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai -xn--b1agm2d.net -xn----btbdvdh4aafrfciljm6k.xn--p1ai -youghbould.wordpress.com -yousense.info -zerocash.msk.ru -z-master.ru -zocaparj.kz -zona-aqua.ru -zoompegs.com -zx6.ru +workle.website +worldhistory.biz +x-diesel.org +xn----7sbho2agebbhlivy.xn--p1ai +xn----8sblgmbj1a1bk8l.xn----161-4vemb6cjl7anbaea3afninj.xn--p1ai +xsion.net +xxxtubesafari.com +yogamatsexpert.com +yougame.biz +yun56.co +zeroredirect5.com +znakomstva-piter78.ru +zoogdinsney.com diff --git a/.dev-tools/test_units/whitelist-ips.conf b/.dev-tools/test_units/whitelist-ips.conf index 500584a94..e42e64139 100644 --- a/.dev-tools/test_units/whitelist-ips.conf +++ b/.dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.224.112.202 0; +35.193.7.13 0; 127.0.0.1 0; diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index 74b11ee1b..2853b6b5d 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index 9abaf4538..3219fe701 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 6e073602a..b917dab50 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2019.08.1808 +#### Version: V4.2019.08.1809 #### Bad Referrers Blocked: [6750](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [557](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [556](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [212](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 307422567..1d676c02e 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2019.08.1808 +#### Version: V4.2019.08.1809 #### Bad Referrers Blocked: [6750](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [557](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [556](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [212](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index f8acf93f5..b0b68ec36 100755 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2019.08.1808 +#### Version: V4.2019.08.1809 #### Bad Referrers Blocked: [6750](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [557](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [556](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [212](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index b82ad0b49..314416f21 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2019.08.1808 +#### Version: V4.2019.08.1809 #### Bad Referrers Blocked: [6750](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [557](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [556](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [212](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index c1392c506..c02a7ef97 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1808 -### Updated: Fri Aug 16 11:07:56 SAST 2019 +### Version: V4.2019.08.1809 +### Updated: Fri Aug 16 14:26:27 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index fa023001e..8a81cdc81 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.08.1808 -### Updated: Fri Aug 16 11:07:57 SAST 2019 +### Version: V3.2019.08.1809 +### Updated: Fri Aug 16 14:26:28 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index c1392c506..c02a7ef97 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.08.1808 -### Updated: Fri Aug 16 11:07:56 SAST 2019 +### Version: V4.2019.08.1809 +### Updated: Fri Aug 16 14:26:27 SAST 2019 ### Bad Referrer Count: 6750 -### Bad Bot Count: 557 +### Bad Bot Count: 556 ################################################### ### VERSION INFORMATION ## @@ -426,7 +426,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)MSFrontPage(?:\b)" 3; "~*(?:\b)MSIECrawler(?:\b)" 3; "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; "~*(?:\b)muhstik-scan(?:\b)" 3; "~*(?:\b)Musobot(?:\b)" 3; "~*(?:\b)Name\ Intelligence(?:\b)" 3; diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 0a469368e..5f8cedb28 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2019.08.1808 -### Updated: Fri Aug 16 11:07:59 SAST 2019 -### Bad Bot Count: 557 +### Version: V4.2019.08.1809 +### Updated: Fri Aug 16 14:26:29 SAST 2019 +### Bad Bot Count: 556 ################################################### ### Version Information ## @@ -573,8 +573,6 @@ User-agent: MSIECrawler Disallow:/ User-agent: Msrabot Disallow:/ -User-agent: MS Web Services Client Protocol -Disallow:/ User-agent: muhstik-scan Disallow:/ User-agent: Musobot