diff --git a/.dev-tools/_robots_input/robots-input.txt b/.dev-tools/_robots_input/robots-input.txt index 4c5cea51f..f302c624d 100644 --- a/.dev-tools/_robots_input/robots-input.txt +++ b/.dev-tools/_robots_input/robots-input.txt @@ -336,7 +336,6 @@ NimbleCrawler Nimbostratus Ninja Nmap -node-fetch NPbot Nuclei Nutch diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index ba5790d93..f49141aa0 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.02.2275 -### Updated: Wed Feb 10 01:52:44 SAST 2021 +### Version: V3.2021.02.2276 +### Updated: Wed Feb 10 06:06:25 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 701efaf43..180bdc087 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 1; +34.122.173.211 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index d99c8cd8c..bb50a4a64 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 0; +34.122.173.211 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index ba5790d93..f49141aa0 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.02.2275 -### Updated: Wed Feb 10 01:52:44 SAST 2021 +### Version: V3.2021.02.2276 +### Updated: Wed Feb 10 06:06:25 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 701efaf43..180bdc087 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 1; +34.122.173.211 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index d99c8cd8c..bb50a4a64 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 0; +34.122.173.211 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index ba5790d93..f49141aa0 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.02.2275 -### Updated: Wed Feb 10 01:52:44 SAST 2021 +### Version: V3.2021.02.2276 +### Updated: Wed Feb 10 06:06:25 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 701efaf43..180bdc087 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 1; +34.122.173.211 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 43c0d5d85..fa14a3c69 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -397,7 +397,6 @@ "~*(?:\b)Nimbostratus(?:\b|)" 0; "~*(?:\b)Ninja(?:\b|)" 0; "~*(?:\b)Nmap(?:\b|)" 0; - "~*(?:\b)node-fetch(?:\b|)" 0; "~*(?:\b)NPbot(?:\b|)" 0; "~*(?:\b)Nuclei(?:\b|)" 0; "~*(?:\b)Nutch(?:\b|)" 0; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index d99c8cd8c..bb50a4a64 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 0; +34.122.173.211 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index ba5790d93..f49141aa0 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.02.2275 -### Updated: Wed Feb 10 01:52:44 SAST 2021 +### Version: V3.2021.02.2276 +### Updated: Wed Feb 10 06:06:25 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index ba5790d93..f49141aa0 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.02.2275 -### Updated: Wed Feb 10 01:52:44 SAST 2021 +### Version: V3.2021.02.2276 +### Updated: Wed Feb 10 06:06:25 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 29888c55a..94946308c 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.02.2275 -### Updated: Wed Feb 10 01:52:43 SAST 2021 +### Version: V4.2021.02.2276 +### Updated: Wed Feb 10 06:06:23 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 52422de58..c4667b0ac 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.154.182.187 0; +34.122.208.80 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 29888c55a..94946308c 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.02.2275 -### Updated: Wed Feb 10 01:52:43 SAST 2021 +### Version: V4.2021.02.2276 +### Updated: Wed Feb 10 06:06:23 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 3bd4c7e2e..701efaf43 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 1; +34.122.208.80 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index cc2baf875..d99c8cd8c 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 0; +34.122.208.80 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 29888c55a..94946308c 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.02.2275 -### Updated: Wed Feb 10 01:52:43 SAST 2021 +### Version: V4.2021.02.2276 +### Updated: Wed Feb 10 06:06:23 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 3bd4c7e2e..701efaf43 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 1; +34.122.208.80 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index cc2baf875..d99c8cd8c 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 0; +34.122.208.80 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 29888c55a..94946308c 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.02.2275 -### Updated: Wed Feb 10 01:52:43 SAST 2021 +### Version: V4.2021.02.2276 +### Updated: Wed Feb 10 06:06:23 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 3bd4c7e2e..701efaf43 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 1; +34.122.208.80 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 43c0d5d85..fa14a3c69 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -397,7 +397,6 @@ "~*(?:\b)Nimbostratus(?:\b|)" 0; "~*(?:\b)Ninja(?:\b|)" 0; "~*(?:\b)Nmap(?:\b|)" 0; - "~*(?:\b)node-fetch(?:\b|)" 0; "~*(?:\b)NPbot(?:\b|)" 0; "~*(?:\b)Nuclei(?:\b|)" 0; "~*(?:\b)Nutch(?:\b|)" 0; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index cc2baf875..d99c8cd8c 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.193.184.18 0; +34.122.208.80 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 29888c55a..94946308c 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.02.2275 -### Updated: Wed Feb 10 01:52:43 SAST 2021 +### Version: V4.2021.02.2276 +### Updated: Wed Feb 10 06:06:23 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 4b93e0faa..c4667b0ac 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.154.182.187 0; 34.122.208.80 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index aae2488f5..a3b039e50 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.02.2275 -### Updated: Wed Feb 10 01:52:43 SAST 2021 +### Version: V4.2021.02.2276 +### Updated: Wed Feb 10 06:06:23 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 5e80f4199..c8638035f 100644 --- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.193.184.18 0; -34.122.208.80 0; +104.154.182.187 0; +34.122.173.211 0; diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 825aebfb3..8634d3c30 100644 --- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.02.2275 -### Updated: Wed Feb 10 01:52:43 SAST 2021 +### Version: V4.2021.02.2276 +### Updated: Wed Feb 10 06:06:23 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index 964763461..5e80f4199 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST +35.193.184.18 0; 34.122.208.80 0; -34.122.173.211 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index 29888c55a..94946308c 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.02.2275 -### Updated: Wed Feb 10 01:52:43 SAST 2021 +### Version: V4.2021.02.2276 +### Updated: Wed Feb 10 06:06:23 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index a398860de..2d9c3c5be 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -46,5 +46,4 @@ # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST 35.193.184.18 0; -34.122.208.80 0; 104.154.182.187 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index 29888c55a..94946308c 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.02.2275 -### Updated: Wed Feb 10 01:52:43 SAST 2021 +### Version: V4.2021.02.2276 +### Updated: Wed Feb 10 06:06:23 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 701efaf43..180bdc087 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 1; +34.122.173.211 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/blacklist-user-agents.conf b/.dev-tools/test_units/blacklist-user-agents.conf index 43c0d5d85..fa14a3c69 100644 --- a/.dev-tools/test_units/blacklist-user-agents.conf +++ b/.dev-tools/test_units/blacklist-user-agents.conf @@ -397,7 +397,6 @@ "~*(?:\b)Nimbostratus(?:\b|)" 0; "~*(?:\b)Ninja(?:\b|)" 0; "~*(?:\b)Nmap(?:\b|)" 0; - "~*(?:\b)node-fetch(?:\b|)" 0; "~*(?:\b)NPbot(?:\b|)" 0; "~*(?:\b)Nuclei(?:\b|)" 0; "~*(?:\b)Nutch(?:\b|)" 0; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index f6cb224a9..ffe8f6eb8 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Acunetix -AwarioSmartBot -GridBot -HEADMasterSEO -JikeSpider -Lipperhey Spider -MTRobot -polaris version -ScoutJet -SiteExplorer +Apexoo +Calculon +Express WebPictures +HTTP::Lite +LeechFTP +NearSite +Rogerbot +SputnikBot +WWW::Mechanize +x09Mozilla diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 18927f92c..35edffcee 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ 01h4x.com 360Spider +Aboundexbot ADmantX -AfD-Verbotsverfahren -AiHitBot -Aipbot -AlphaBot -Anarchy -Ankit -Anthill -arquivo.pt -arquivo-web-crawler -ASPSeek +AhrefsBot +Apexoo +autoemailspider AwarioSmartBot +Backlink-Ceck +BacklinkCrawler BackStreet BackWeb Badass -Bandit -BDCbot -BDFetch +BBBike BlackWidow -BlowFish Boardreader -Brandwatch -Buck +BotALot Buddy -BuiltBotTough -BunnySlippers -Chlooe -Cliqzbot -CODE87 +BuiltWith +Calculon +CATExplorador +Cegbfeieh +CheeseBot +ChinaClaw +Cocolyzebot Collector -Copyscape -crawler4j crawl.sogou.com +Curious DatabaseDriverMysqli -demandbase-bot -Devil -Dirbuster -Disco +Digincore +DigitalPebble +Discobot +Discoverybot Dispatch DittoSpyder -DnyzBot DomainAppender DomainSigmaCrawler -domainsproject.org -Download Wonder +DomainStatsBot +Dragonfly +Drip DTS Agent EasyDL -ECCP/1.0 -Ecxi -EirGrabber -EMail Siphon -EroCrawler -Evil +Ebingbong +EMail Wolf Exabot ExtLinksBot -EyeNetIE -FemtosearchBot -FHscan -FlashGet -FrontPage -FyberSpider +Extractor +Ezooms +Foobot +Fuzz Genieo -GetRight -GetWeb +GermCrawler G-i-g-a-b-o-t +gopher Gotit -GoZilla -Go!Zilla +GrabNet +GrapeFX GrapeshotCrawler GridBot -GT::WWW -Haansoft -heritrix +HaosouSpider +Harvest +HEADMasterSEO Heritrix -HMView +Hloader HTMLparser -HTTP::Lite +Humanlinks Iblog +IDBot IDBTE4M Image Fetch -Image Sucker IndeedBot -InfoTekies +instabid +Intelliseek +InternetSeer internetVista monitor -ips-agent Iria +IRLbot isitwp.com -JennyBot -JikeSpider +Iskanie +JamesBOT +JOC Web Spider Jorgee Keyword Density -Kinza -LeechFTP -LexiBot +Larbin +Lftp +LibWeb +LieBaoFast Lightspeedsystems -Likse -Linkdexbot LinkextractorPro LinkpadBot -LinkWalker +LinqiaScrapeBot Lipperhey Spider -Lmspider -Ltx71 -lwp-request -LWP::Simple -lwp-trivial Magnet Mag-Net Mail.RU_Bot -Majestic12 -Majestic-SEO -Mb2345Browser -meanpathbot +Majestic SEO +Masscan +Mass Downloader +MauiBot +Meanpathbot MeanPath Bot -Mediatoolkitbot +MegaIndex.ru Metauri -MFC_Tear_Sample MicroMessenger MIDown tool Mister PiX MJ12bot -Mozlila -MQQBrowser +Mojeek +Mojolicious +Morfeus Fucking Scanner MSFrontPage +Msrabot MTRobot Name Intelligence Navroad -Nessus -netEstate NE Crawler +NearSite NetMechanic NetSpider -Nettrack -Net Vampire -NetZIP NextGenSearchBot -Nibbler -NICErsPRO +Nikto NimbleCrawler -Nimbostratus -node-fetch +Ninja +Nmap Nuclei -OpenLinkProfiler +Nutch +oBot +Offline Explorer +Openfind +OrangeSpider OutclicksBot -OutfoxBot -PageGrabber +page scorer +PageScorer +Pandalytics Panscient -Papa Foto Pavuk -pcBrowser -PeoplePal Petalbot -Picscout +PHPCrawl Piepmatz -Pi-Monster -Pixray -POE-Component-Client-HTTP +Pimonster +PleaseCrawl +plumanalytics +probe-image-size ProWebWalker Psbot -RankFlex +Pu_iN +Pump +PyCurl +Quick-Crawler +RankActiveLinkBot +RankingBot RankingBot2 -Rankivabot RankurBot RealDownload -Recorder -ReGet -RepoMonkey +RebelMouse +RedesScrapy +Re-re Ripper Rogerbot +RSSingBot +s1z.ru +SalesIntelligent satoristudio.net +SBIder scalaj-http +Scanbot scan.lol ScreenerBot SearchmetricsBot -Semrush SemrushBot -SentiBot SEOkicks SEOkicks-Robot -SEOprofiler +SEOlyticsCrawler +Seomoz seoscanners -Shodan -Siphon -Sitebeam +sitechecker.pro SiteExplorer +Siteimprove SiteLockSpider +siteripz SiteSucker -Site Sucker Sitevigil -SlySearch -Snapbot +SmartDownload +SMTBot +Snake SocialRankIOBot +Sociscraper +sogouspider Sogou web spider Sosospider -SpaceBison -Spammen -Spanner +SpankBot Spbot +Sqlmap Sqlworm -Sqworm +Stripper +Sucker +Sucuri Suzuran -Swiftbot sysscan -T0PHackTeam -T8Abot +tAkeOut +Teleport +TeleportPro Telesoft Thumbor -Titan -Toweyabot Trendiction trendiction.com trendiction.de True_Robot Turnitin -VB Project +TurnitinBot +UnisterBot +Upflow +URLy Warning +URLy.Warning +Vacuum +Vagabondo VCI VelenPublicWebCrawler -VidibleScraper Virusdie VoidEYE -Voltron -WallpapersHD -WASALive-Bot +Voil +Wallpapers/3.0 WebAuto -WebBandit -Web Collage WebCopier -Web Fetch +WEBDAV +Web Enhancer +WebmasterWorldForumBot webmeup-crawler +WebPix +Web Pix WebReaper Web Sauger -Webster -WebSucker -Web Sucker +Webshag +WebsiteQuester +WebStripper WeSEE -Whack -Whatweb -Wonderbot +Whacker +Who.is Bot +Wprecon WPScan -WWW-Mechanize -x09Mozilla +WWW::Mechanize Xaldon WebSpider -Xaldon_WebSpider -Zauba +Xenu +YoudaoBot zauba.io Zermelo Zeus +Zitebot ZoomBot -ZumBot +ZoominfoBot +ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 68c56d79b..bbaaf52d2 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -01h4x.com -360Spider 404checker -404enemy -AfD-Verbotsverfahren +80legs +AIBOT +AiHitBot +Aipbot Alexibot Alligator AllSubmitter -Anarchie -Ankit -archive.org_bot +Anthill +Apexoo +arquivo.pt ASPSeek -BackStreet -BatchFTP +Attach +BackDoorBot +Backlink-Ceck +backlink-check +BacklinkCrawler +BackWeb +Bandit +Barkrowler Battleztar Bazinga -BBBike +BDCbot BetaBot -Bitacle -BLEXBot Boardreader -Bolt BotALot Brandwatch -Buck +Bullseye +BuzzSumo +CATExplorador check1.exe CheeseBot CherryPicker +ChinaClaw Chlooe -coccocbot-web -CODE87 -Cogentbot +Claritybot +Cliqzbot +Cloud mapping +Cocolyzebot +cognitiveseo Copier -Copyscape +CopyRightCheck +Cosmos crawler4j crawler.feedback -CrazyWebCrawler CSHttp +Curious +Custo DatabaseDriverMysqli DBLBot -Demon +demandbase-bot Deusu -Devil -DigitalPebble -DIIbot -Disco +Dirbuster Discobot -Discoverybot -Dispatch DittoSpyder -DnyzBot -DomainAppender -DomainSigmaCrawler -Domains Project -EasyDL +DomainCrawler +domainsproject.org +DTS Agent +Ebingbong ECCP/1.0 EirGrabber EMail Siphon -EMail Wolf +EroCrawler Exabot -Express WebPictures Extractor -Extreme Picture Finder -facebookscraper -FHscan -Fimap +ExtractorPro +EyeNetIE +FlashGet Foobot Freeuploader -GalaxyBot -Genieo -GetWeb -Gigablast +Fuzz +GetRight G-i-g-a-b-o-t -Go-Ahead-Got-It gopher -Gotit GoZilla Go!Zilla Grabber -GrapeFX -GrapeshotCrawler +Grafula GridBot +HaosouSpider +Harvest HEADMasterSEO heritrix Heritrix -Hloader HTMLparser -HTTP::Lite -Humanlinks -Iblog +HTTrack +HybridBot IDBot Id-search -Image Sucker -IndeedBot -Indy Library -InfoNaviRobot -instabid -Intelliseek -Internet Ninja -InternetSeer -internetVista monitor +InterGET ips-agent IRLbot Iskanie -JikeSpider +JamesBOT +JOC Web Spider +Joomla +Jorgee JustView -Kenjin Spider +Jyxobot Keyword Density Kozmosbot +Larbin +LeechFTP LeechGet LexiBot -LieBaoFast +Lftp +Lightspeedsystems Likse -LinkpadBot LinkWalker LinqiaMetadataDownloaderBot -LinqiaRSSBot -Ltx71 +Lipperhey Spider +Litemage_walker +Lmspider +lwp-request lwp-trivial -Magnet Mag-Net -Mail.RU_Bot -Majestic12 -MarkMonitor +magpie-crawler +Majestic-SEO +MarkWatch +Masscan +Mass Downloader meanpathbot -Microsoft Data Access -MIIxpc +Meanpathbot +Mediatoolkitbot +MicroMessenger +Microsoft URL Control +MIDown tool +Mister PiX +MJ12bot Mojeek +Mojolicious Mozlila -MQQBrowser +Mr.4x3 MSFrontPage Msrabot +MTRobot muhstik-scan +Musobot +Name Intelligence +Nameprotect Navroad -Needle NetAnts -Netcraft Nettrack Net Vampire -Netvibes -NetZIP -NextGenSearchBot -Nibbler -NimbleCrawler -Ninja -Nmap -node-fetch +NICErsPRO NPbot Nutch Offline Explorer -OrangeSpider +OpenVAS +OutfoxBot +PageAnalyzer +Page Analyzer PageGrabber -Panscient -Papa Foto +Pandalytics Pavuk PECL::HTTP -Petalbot +PeoplePal PHPCrawl +Picscout PictureFinder -Pimonster +Pi-Monster +Pixray PleaseCrawl +Pockey +POE-Component-Client-HTTP probe-image-size -Probethenet ProWebWalker Psbot -Pu_iN Pump -PxBroker +PyCurl Quick-Crawler -RankFlex -Rankivabot +RankActiveLinkBot +RebelMouse Recorder -RepoMonkey -RSSingBot -Scanbot +Re-re +Ripper +RocketCrawler +SalesIntelligent +scalaj-http +scan.lol ScoutJet +Scrapy Screaming -Searchestate -SEOkicks-Robot +SemrushBot +SEOkicks Seomoz -SEOprofiler -seoscanners SeoSiteCheckup -sexsearcher +serpstatbot Shodan +SISTRIX sitechecker.pro Siteimprove -SiteLockSpider -SiteSnagger -SiteSucker SlySearch -Snoopy -Sociscraper +Snake +SocialRankIOBot +sogouspider Sogou web spider -Sottopop -SpaceBison -Spammen -SpankBot -Spanner sp_auditbot +Spbot +SputnikBot Sqlmap -Sqlworm Sucker -SuperHTTP -SurveyBot +Sucuri +SuperBot Swiftbot Szukacz -tAkeOut +T0PHackTeam Teleport -The Intraformant -Tracemyfile -Turnitin -Upflow -Vagabondo +Telesoft +TheNomad +Toweyabot +Trendiction +trendiction.de +True_Robot +TwengaBot +Twice +Typhoeus +URLy Warning VCI VelenPublicWebCrawler -Virusdie +VidibleScraper VoidEYE Voil Voltron -Wallpapers/3.0 -WallpapersHD WASALive-Bot -WebAuto -WebCollage +WBSearchBot +Webalta Web Collage -WebEnhancer -Web Fuck -WebImageCollector +WebCopier +WebFuck +WebGo IS WebLeacher -WebmasterWorldForumBot -WebPix +webmeup-crawler +Web Pix WebsiteExtractor -Website Quester +Webster +Web Sucker WebWhacker +WebZIP WeSEE -Whack -Whacker -Woobot -WPScan +Whatweb +Who.is Bot +WinHTTrack +WISENutbot WWW-Collector-E -WWWOFFLE -x09Mozilla +WWW-Mechanize +WWW::Mechanize Xaldon_WebSpider +xpymep1.exe YoudaoBot -Zauba -zauba.io -zgrab -ZmEu -ZoominfoBot -ZyBorg +Zade diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index 587960690..f0d77e361 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -aa8b68101d388c446389283820863176e7.com -belstaffstore.online -directivepub.com -easyukraine.com -elektroniksigaraankara.info -gojiberriess.apishops.ru -ip-guide.com -landinez.co -tizanidinefibromyalgia.blogspot.com -ucoz.ru +carstrends2015.com +datract.com +freegamesplay.online +itch.io +paparazzistudios.com.au +sell-fb-group-here.com +seo-smm.kz +shopsellcardsdumps.com +videos-for-your-business.com +vtcdns.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 3d4b144b6..d56a58fe8 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -01apple.com -12masterov.com -1688.com -2020iscoming.info -3-letter-domains.net -4k-player.pl -4ureyesonly.com +00author.com +03e.info +03p.info +11235813.webzdarma.cz +123locker.com +13tabs.com +2ads.co.uk +3w24.com +3weekdiet.com +5forex.ru +6hopping.com 76brighton.co.uk -7yue.org aa08daf7e13b6345e09e92f771507fa5f4.com -abbanreddy.com -abcdefh.xyz -abcdeg.xyz -acmebtn.ml -actionnooz.com +aa625d84f1587749c1ab011d6f269f7d64.com +aa81bf391151884adfa3dd677e41f94be1.com +aa8780bb28a1de4eb5bff33c28a218a930.com +abovetherivernc.com activecampaign.dreamhosters.com -adexprts.com -adsfresh.men -adultfriendfinder.com -aimayangzhi.com -alert-fjg.xyz -alienwheel.es -all4invest.ru -all4wap.ru -allknow.info -allmarketsnewdayli.gdn -all-streaming-media.com -am15.net -ampower.me -an-donut.com -angigreene.com +adf.ly +adobereader-free.ru +ads.gold +advancedmassagebysara.com +akama.com +akuhni.by +albumsuper.info +alfa9.com +alfapro.ru +android-systems.ru +apartamentwroclaw.eu apelsinnik.website -apessay.com -apioutoboxnet-a.akamaihd.net -appfastplay.com -appiq.mobi -apple.com-cleaner.systems -applicationg29.com -applyneedy.xyz -as5000.com -asia-forum.ru -atelielembrancaqueencanta.com.br -auspolice.com -autorn.ru -aviav.eu +arcteryxstore.online +artdestshop.eu +articlesdirectoryme.info +asacopaco.tk +asophoto.com +asrvvv-a.akamaihd.net +atlasvkusov.ru +audiofree.ru avirasecureserver.com -avto-oligarh.ru -axbocz.net -ayakino.net -azarttoto.com -azazu.ru -azbaseimages.net -azte.ch -b2b-lounge.com -back.dog -backlinkwatch.com -baersaratov.ru -banksrf.ru -bank.uz -bannerads.de -bannertesting.com -battlecarnival.su -begalka.xyz -best-deals-products.com -bestofferswalkmydogouteveryday.gq -bestpornuha.com -bizcheapjerseyswholesalechina.com -blobar.org -blog.f00kclan.de -bmwhighperformers.com -board.f00d.de -bodybuilding-shop.biz -boleznikogi.com -brainzpod.win -break-the-chains.com -breget74.ru -bristolhotel.com -bronzeaid-a.akamaihd.net -burkesales.com -buy-cheap-online.info -buzz-porno.info -bwlx.prepedu.cn -carsdined.org -cbcseward.com -cdn.walkme.com -centre-indigo.org.ua -cfsrating.sonicwall.com -chiblackhawks-jerseys.com -chinacheapelitejerseys.com -christianlouboutinreplica.pw -christianlouboutinsaleoutletonline.us -cinemaenergy-hd.ru -clippingphotoindia.com -comissionka.net -compliance-alex.xyz +azartmsl.com +azbukafree.com +bad-stars.net +balans.shahterworld.org +balois.worldbreak.com +baltgem.com +bamo.xsl.pt +bannerspace.com +bausparen.bz.it +bbsoldes.fr +beauty-clinic.ru +beget.tech +berdasovivan.ru +bestdraws.com +bestmarriages.com +best-seo-offer.com +bet365.com +bezlimitko.xyz +bg6s0.com +bikini-image.com +bildsuche.ru +blackcurranthumidifiers.site +blog.koorg.ru +blogs.rediff.com +bluesalt.co +brateg.xyz +brewdom.ru +brillianty.info +bryansk.zrus.org +burn-fat.ga +buttons-for-website.com +buyfriend.ru +bvlgariring.xyz +cam-kontakte.org +candycrushshop.com +candy-glam-hp.com +carloans.com +cartierbraceletsreplica.pw +cdncash.com +censys.io +chatseo.com +chcu.net +cheapmoncler.pw +cherrypointplace.ca +chinaelitecheapjerseys.com +christianlouboutinshoes.xyz +cibpenonptib.flu.cc +cienum.fr +cigarpass.com +circlesl.com +citizenclsdriveri7.pen.io +clarithromycin500mg.com +cleanmypc.com +club-lukojl.ru +codysbbq.com +colobit.biz +compliance-andrew.top +compliance-don.top +compliance-elena.top +compliance-elena.xyz compliance-fred.top -com-supportcenter.website -connectionstrenth.com -consorzioilmosaico.org -coolbar.pro -coolyarddecorations.com -coslab.club -countbertwistdisp26.soup.io -countercrazy.com -cowlmash.com -cpabegins.ru -cubs-jerseys.us -customchocolate.business-for-home.com -cyse.tk -czat.wp.pl -dairyindia.in -damasarenai.info -dame-ns.kz -daooda.com -davebestdeals.com -d-black.bz -delayreferat.ru -demon-tweeks.com -deniven.1bb.ru -deutschland123.de -div.as +compliance-fred.xyz +compliance-ivan.xyz +compliance-mark.top +compliance-viktor.top +connectingsingles.com +contentlook.co +covetnica.com +cryptoswap.biz +cubook.supernew.org +dangphoto.trade +dentalpearls.com.au +de.zapmeta.com +diffbot.com +dikx.gdn +discountliv.com +divci-hry.info djekxa.ru -djonwatch.ru -dnepropetrovsk.mistr-x.org -domain.webkeyit.com -downloadeer.net -drpornogratisx.xxx -drunkenstepfather.com -dstroy.su -dverimegapolis.ru -eas-seo.com -e-commerce-seo.com -ecookna.com.ua -edudocs.net -elektir.ru -elektrischezigarette1.blog.pl -elitedollars.com -elite-sex-finders.com -elvel.com.ua -emediate.eu -englishdictionaryfree.com -essayservicewriting.org -etenininrade.ga -etm-consult.de -event-tracking.com -evidencecleanergold.com -exdocsfiles.com -expresstoplivo.ru -ez8motelseaworldsandiego.com -ezb.elvenmachine.com -ezigarettekaufen1.hpage.com +docstoc.com +doctormakarova.ru +dogoimage.com +dosug-lux.ru +downloaddy.net +downloadmefiranaratb1972.xpg.com.br +drillsaw.ru +drupa.com +dvrlists.com +dyshagi.ru +ed-shop01.ru +e-kwiaciarz.pl +elektronischezigarettekaufen2.tumblr.com +energy-ua.com +englate.com +eropho.net +erotikchat-24.com +escortslet.net +estelight.ru +f1nder.org +facialporntube.com +familienzahnaerzte.com family1st.ca -famix.xyz -fba-mexico.com -feel-planet.com -fickluder69.com -fikasound.tk -filmbokep69.com -findpik.com -footbalive.org -forex-procto.ru -fotzen-ficken.com -foxjuegos.com -frankofficial.ru -free-deals.faith -free-gluten.ru -freshwallpapers.info -frivgame250.com -fullfileaccess.com -g33.org -gaytube.com -ge0ip.com -ge0ip.org -geckoandfly.com +faptitans.com +farm26.ru +fashion.stellaconstance.co +femmesdenudees.com +fermersovet.ru +filmgo.ru +findthe.pet +formulaantiuban.com +forums.toucharcade.com +fotosfotos.eu +fr-bearings.ru +freeuploader.com +frequiry.com +frustrated-favorable.gq +fuck-paid-share-buttons.xyz +galeon.com +gemgrab-a.akamaihd.net getlamborghini.ga -gigablast.com -girlsfuckdick.com -glavtral.ru +getprismatic.com +gettpromos.com +getyourimage.club +glopages.ru +gloverid.site gne8.com -go2mike.ru -goodprotein.ru -grand-chlen.ru -graphics8.info +goforexvps.com +gogalleryawesome.com +goldpanningtools.com +goodnightjournal.com +googlemare.com +greendream.com.ua +gribkovye-zabolevaniya.com +grizzlysgrill.com +gsasearchenginerankerhelp.com gta-club.ru -guitar-master.org +gwebtools.com hacktougroup.ru -hard-porn.mobi -healmytrauma.info -hentai-manga.porn +heliko.no +highstairs-a.akamaihd.net +host-tracker.com hotkeys.com -https-legalrc.biz -hustoon.over-blog.com -ifirestarter.ru -ikearugs.xyz -i-midias.net.br +housekuba.org +houseofgaga.ru +houseofrose.com +howtoclean.club +hubbble.com +ihc2015.info +images-free.net +indianmedicaltourismshop.com +infektsii.com informatiecentro.be -intellego.info -intellektmedia.at -interferencer.ru -interfucks.net -iqoption.pro -isistaylorporn.info -islamtoday.co.za -ivanovo.zrus.org -jennyfire.ru -jillepille.com -jjbabskoe.ru -joy-penguin.com -kabbalah-red-bracelets.com -keywords-monitoring-your-success.com -keywordsuggest.org -kfon.eu -kinky-fetishes.com -kino2018.club -klitimg.pw -kol-energo.ru -komputernaya-pomosh-moscow.ru -kongruan.com -krasivye-devushki.net -krasnodar.zrus.org -ku6.com -labvis.host.sk -lankarns.com -leadwayau.com -lechtaczka.net +interesnie-faktu.ru +intermesh.net +inzn.ru +io9.com +irkutsk.online-podarki.com +it-max.com.ua +itronics.ca +iyasimasennka.com +jagg.info +janavibekken.no +javcoast.com +junglenet-a.akamaihd.net +jus0wil.pp.ua +kadashihotel.com +kakablog.net +kamorel.com +kandidos.com +kemerovo.zrus.org +keywords-monitoring-success.com +kinobaks.com +kinoflux.net +klin3952.ru +komp-pomosch.ru +kovrenok.ru +lecbter-relationships.ga +lechenie-gemorroya.com lenvred.org +lesbian.xyz +letmacworkfaster.site +levaquin750mg.blogspot.com +lidoradio.com lifebyleese.com likrot.com +limads.men linkpulse.com -liverpool.gsofootball.com +lisque.batcave.net ljusihus.se -locatellicorretor.com.br -lonely-mature.com -lonfon.xyz -lovasszovetseg.hu -low-format.ru -luxup.ru -machicon-akihabara.info -magicdiet.gq -mantramusic.ru -manualterap.roleforum.ru -marblestyle.ru -maridan.com.ua -mastershef.club +loftdigital.eu +lollypopgaming.com +longlifelomilomi.info +magnetic-bracelets.ru +mailemedicinals.com +mall.uk +masterseek.com matchpal-a.akamaihd.net -maxxtor.eu -mazda-roadsters.com -meble-bogart.info -meccadumps.net -medispainstitute.com.au -megaindex.ru +medicovi.com +mega-bony-2017.pl mega-polis.biz.ua -me-ke.com -menetie.ru -mesbuta.info -minecraft-rus.org -mirobuvi.com.ua -misswell.net -mmofreegames.online -mmog-play.ru -mnimmigrantrights.net +metaxalonevstizanidine.blogspot.com +methodsmarketing.com +microsearch.ru +mikozstop.com +millionare.com +miniads.ca +mir-limuzinov.ru +mnogabukaff.net mobsfun.net -moi-glazki.ru -mojaocena.com -mojowhois.com -monclerjacketsoutlet.pw +modnie-futbolki.net +moivestiy.biz +mojpregled.com +monarchfind-a.akamaihd.net +monarhs.info monsterdivx.com -moviemail-online.co.uk -movies-in-theaters.net -mrbojikobi4.biz -mrwhite.biz -multgo.ru -mybinaryoptionsrobot.com -myxdate.info -nakozhe.com -narosty.com -naturtreenspicerx.pw -naval.jislaaik.com -netpics.org -nevansk.ru -newhairstylesformen2014.com -newsperuse.com -news-speaker.com +motherboard.vice.com +moz.com +muznachas-service.ru +muztops.ru +mydirtyhobby.com +mydoctorok.ru +namenectar.com +nastydollars.com +national-today-winning-winner.club +naturalbreakthroughsresearch.com +naturalpharm.com.ua +needtosellmyhousefast.com +nelc.edu.eg +new-apps.ru +news-readers.ru +newstaffadsshop.club +nextrent-crimea.ru +nfljerseyscheapchinabiz.com +niroo.info +nlfjjunb5.ru +noclegonline.info +no-fuel.org nootrino.com -northfacestore.online -noumeda.com -nsatc.net -ntic.fr -oakleyglassesonline.us -oballergiya.ru -obnal.org -ohmyrings.com +nosecret.com.ua +obiavo.net +obuv-kupit.ru +odesproperty.com +offf.info +officedocuments.net okayimage.com -oktube.ru -olvanto.ru -oneclickfiles.com -onlainbesplatno.ru -opinionreelle.com -orenburg-gsm.ru -owohho.com -pagesense.com -paidonlinesites.com -partnerworkroom.men -pekori.to -perl.dp.ua -perosan.com -phelissota.xyz -picsfair.com -pipki.r.acdnpro.com -pisanieprac.info -pixelrz.com -pizdopletka.club -pizza-imperia.com -pkr1hand.com -poems.com.ua -pogosh.com -popup.matchmaker.com -porno-transsexuals.ru -pornzone.tv -portadd.men -predominant-invent.tk -preventheadacheguide.info -printie.com -priscilarodrigues.com.br -privatbank46.ru -privatefx.all4invest.info -privat-girl.net -prizesbook.online -producm.ru -proekt-mos.ru -pronorm.fr -pufip.com -pulseonclick.com -q-moto.ru -quickbuck.com +okel.co +okuos.com +onclickpredictiv.com +onlywoman.org +ontargetseo.us +openlibrary.org +openstat.com +ops.picscout.com +orion-v.com +ororodnik.goodbb.ru +osoznanie-narkotikam.net +ostrovtaxi.ru +outpersonals.com +outrageousdeal-a.akamaihd.net +panchro.co.uk +paradontozanet.ru +park.above.com +p-business.ru +pc4download.co +perfectpracticeweb.com +pflexads.com +photo.houseofgaga.ru +photokitchendesign.com +picscout.com +pictures-and-images.net +piski.top +plastgran.com +plugingeorgia.com +pod-muzyku.club +popander.mobi +popugauka.ru +popup-hgd.xyz +porndroids.com +pornhive.org +pornhubforum.tk +pornhub-ru.com +pornonik.com +pornophoto.xyz +pornoreino.com +porno.simple-image.com.ua +pospr.waw.pl +prefersurvey.net +pregnant.guru +pricheski-video.com +privacylocationforloc.com +privetsochi.ru +progonrumarket.ru +projefrio.com.br +promalp-universal.ru +prostitutki-almata.org +prostitutki-kharkova.org +prostitutki-tyumeni.org +py100.ru +qpypcx.com +quickcashlimited.com quickchange.cc +quit-smoking.ga raavidesigns.com +rada.ru +raisedseo.com randki-sex.com -rankscanner.com -razorweb-a.akamaihd.net -realresultslist.com -real-time-analytics.com -redirlock.com -referencemoi.com +razamicroelectronics.com +realmonte.net +redirectme.net +refererx.com reklama-i-rabota.ru -resellerclub.com -responsinator.com -roznica.com.ua -ruscoininvest.company -rutor.vip -salutmontreal.com -sankt-peterburg.nodup.ru -saugatuck.com -saveriopiazza.it -sax-sex.com -sbwealthsolutions.ca -scanner-don.top -scanner-marwin.top -scanner-walt.top -scansafe.net -seadragonherbery.com -searchinquire.com -semaltmedia.com -seonetwizard.com -servethis.com -serving.adbetclickin.pink -sex-foto.pw -sexsearch.com -sex-sex-sex5.com -sexspornotub.com -sex-tracker.com -sextracker.de -shakhtar-doneck.ru -sharebutton.org -shariki-zuma-lines.ru -shmetall.com.ua -shoppytoolmac.com -signal03.ru -signx.info -siha.de -similardeals.net -site-speed-checker.site -sitevaluation.org -sky-mine.ru -slimcdn.com -snowplanes.com -socialmadesimple.com -socialmonkee.com -solicita.info -songoo.wz.cz -songplanet.ru -space4update.pw -speeddream.xyz -spidtest.org -spymac.net -stairliftsarea.com -stairliftstrue.com -steebook.com -stjamesschool.info -stpicks.com -stpolice.com -streetfooduncovered.com -stroymonolit.su -stroy-portal22.ru -strv.se -studworks.org +remont-komputerov-notebook.ru +resant.ru +restaurantlescampi.com +resultshub-a.akamaihd.net +retreatia.com +rn-to-bsn.com +robot-forex.biz +runstocks.com +sad-torg.com.ua +safe-app.net +sarf3omlat.com +scanmyphones.com +scanner-fred.top +scanner-john.top +scripted.com +sdi-pme.com +seoboxes.com +seocheckupx.com +seogadget.ru +sexsaoy.com +sexy.babes.frontend-stack.top +shama-rc.net +sharebutton.net +shikiso.info +shinikiev.com.ua +shoesonlinebuy.cn +shoplvlv.us +signoredom.com +siteheart.net +sivs.ru +sizeplus.work +slavkokacunko.de +slutloadlive.com +smallseotools.com +social-search.me +sockshare.net +solinf.co +solitaire-game.ru +spasswelt.net +spasswelt.xyz +spb-plitka.ru +spitfiremusic.com +spl63.fr +sport7777.net +stathat.com +statustroll.com +sterva.cc +stopnarco.ru +streetfire.net +stromerrealty.com +strongsignal-a.akamaihd.net +stroyhelp-dv.ru +students-cheapskate.ml +studiofaca.com +suhanpacktech.com +sukarame.net superinterstitial.com -supermama.top -svarkagid.com -sweepstakes.rewardit.com -tamada69.com -taxi-v-eisk.ru -tecspb.ru -telefonsexi.com -thefds.net +supervesti.ru +svbur.ru +swingerseiten.com +sygraem.com +taihouse.ru +takethatad.com +taqywu51.soup.io +template-kid.com +thecounter.com +thegalerie.eu thegioixekhach.com -thegolfclub.info -thesmartsearch.net -tizanidineandgabapentin.blogspot.com -tizanidinecost.blogspot.com -tizanidineduringpregnancy.blogspot.com -tizanidinehydrochloride4mgstreetvalue.blogspot.com -tk-assortiment.ru -tnctrx.com -topappspro.com +thfox.com +timdreby.com +tizanidineinteractions.blogspot.com +tizanidinepills.blogspot.com +tizanidinewithdrawal.blogspot.com +tizanidinewithdrawalsymptoms.blogspot.com +tkanorganizma.ru +toolsky.com topcar-krasnodar.ru topclickguru.com -topquality.cf -torrentdownloadhub.com -toursmaps.com -tracxn.com -traffic2cash.org -traffpartners.com -trainoffend.ml -truck-land.life -trumpetedextremes.com -trustedmaccleaner.com +topsiteminecraft.com +top-study.work +torrent-newgames.com +torrentred.games +torrents-tracker.com +tradedeals.biz +trafficfactory.biz +trafficgenius.xyz +trafficmp.com +tramadolandtizanidine.blogspot.com +trudogolik.net tube8.com -tula.mdverey.ru -tvnewsclips.info -typimga.pw -u17795.netangels.ru -uchetunet.su -um-razum.ru -unacittaconte.org -urzedowski.eu -us-america.ru -uzungil.com -venta-prom.ru -video-chat.love +tvory.predmety.in.ua +tytoona.com +ua-company.ru +uroffer.link +usswrite.com +vabasa.inwtrade.com +varikozdok.ru +vashsvet.com +vasileostrovsky-rayon.ru +vclicks.net +vesnatehno.ru videochaty.ru videonsk.com -videooko.weebly.com -vozbujdenie.com -vtc.pw +videos-for-your-business.com +videtubs.pl +vinylvault.co.uk +vksaver-all.ru +voloomoney.com +voloslove.ru +voluumtracker1.com +vsdshnik.com vucms.com -vvpg.ru -wallpapersist.com -warningzscaler.heraeus.com -watchmyfb.pl -webjam.com -website-analyzer.info +web.cvut.cz +webinstantservice.com +webpromotion.ae website-speed-checker.site -wesharepics.com -whatistizanidine2mg.blogspot.com +westen-v.life +whipme.yopoint.in +wholesalejerseyscheapjerseys.us.com wholesalejerseysgaa.com -wieseversa.no -winterclassichockeyjerseys.com -wleuaprpxuvr.ga -writingservices17.blogspot.ru -wstroika.ru -www.ehscloud.cn -www.kabbalah-red-bracelets.com -xaylapdiendanang.com -x-mix.info -xn----7sbaaabaei0cc8aj5bj0bncejx.xn--p1ai -xn----7sbbagbq7bd5aheftfllo4m.xn--p1ai -xn----7sbbahaq9bb5afgiqfliv4m.xn--p1ai -xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai -xn--80aeahghtf8ac5i.xn--p1ai -xn--80aebbcbcdemfkhba4byaehoejh8dza3v.xn--p1ai -xn----8sbhefaln6acifdaon5c6f4axh.xn--p1ai -xn----dtbndd4ae7eub.top -x-stars.ru -xvideospornoru.com +whos.amung.us +wovis.site +wrona.it +x69ty.ru +xbaboon.com +xchat26.myfreecams.com +xdoza.com +xerox-douglas.cf +xn--24-glceagatoq7c2a6ioc.xn--p1ai +xn----7sbaphztdjeboffeiof6c.xn--p1ai +xn--80aikhbrhr.net +xn--90acenikpebbdd4f6d.xn--p1ai +xn----ctbigni3aj4h.xn--p1ai +xn--e1aggki3c.xn--80adxhks +xnxx699.com +x-porno.video +xsion.net +xtube.com +xvideosporn.biz +xxxrus.org +y8games-free.com +yoopsie.com youdesigner.kz +yougotanewdomain.com youjizz.com -yourporngay.com -youtubedownload.org -zebradudka.com -zeroredirect2.com -zigarettenonl.canalblog.com -zigarettenonlinekaufen2.bloog.pl -zirondelli.it -z-master.ru -zolotoy-lis.ru -zoodrawings.com -zoolubimets.ru -zscaler.net -zvooq.eu +yourmovies.pl +zarajbuilders.com +zerocash.msk.ru +zigarettenonlinekaufen2.drupalgardens.com +zoogdisany.com +zvezdagedon.ru diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index 2c430c41d..3a2abeec6 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -00go.com -1pamm.ru -2000k.ru +1000n1.ru +11pikachu.ru +1flag.co.za +202ch.com +214jaluwobapef.cf 23kw.ru -51.la -5i2.net -8kisses.eu -aa625d84f1587749c1ab011d6f269f7d64.com -acheterviagrafr24.com -adultfullhd.com -adultmeetonline.info -akita.kz -akusherok.ru -alienwheels.de -all4invest.ru +2pxg8bcf.top +4kplayer.pl +51unlim.ru +91abcw.com +aa81bf391151884adfa3dd677e41f94be1.com +addbin.men +admitad.com +adverster.com +alfabot.xyz +alldezservice.kz allornamenti.com -alpinism.ru -anaksma.info -ananumous.ru -angel1777.kz -appfastplay.com -applyneedy.xyz -app-ready.xyz -axbocz.net -b2b-lounge.com -basedecor.ru -beget.tech -berdasovivan.ru -beyan.host.sk -bif-ru.info -bluesman.nu -bmsco.co -brothers-smaller.ru -bscodecs.com -burn-fat.ga -buyhoverboard.com -cablecar.us -cartierreplica.win -cartiertrinity.xyz +amateurgalls.com +amoi.tn +andpolice.com +anjalika.co.in +anonymizeme.pro +antons-transporte.de +apelsinnik.website +apessay.com +apifasterlightin-a.akamaihd.net +apper.de +appfixing.space +aussie-prizes.com +backiee.com +balkanfarma.ru +balook.com +bcmp.org +beachpics.com +berrymall.ru +bigcities.org +blacktwhite.com +blogig.org +bluesalt.co +bmw-ark.ru +bumble.cheapwebsitehoster.com +buyessay3.blogspot.ru +buynorxx.com +camel-beach.com +candcstuccoandstone.com +carstrends2015.com cartujano-pre.de -cdncash.com -cementaresearch.se -centrumcoachingu.com +cayado.snn.gr +cfsrating.sonicwall.com +cheapestjerseysonwholesale.com cheapmoncler.xyz -check-host.net -cialis-samples.com -clickpapa.com -clicksor.net -codysbbq.com -customcollegeessays.net -danashop.ru -dellalimov.com -delta-line.men -demian.kz -dentalpearls.com.au -dichvuvesinhhanoi.com -dikx.gdn -dipstar.org -docs4all.com -dogbreedspicture.net -dostavka-v-krym.com -dotnetdotcom.org -draniki.org -drugs-no-rx.info -dumuelave.xyz -duplicashapp.com -dverimegapolis.ru -e-buyeasy.com +chocoslim.pro +cienum.fr +cindymatches.com +classiquebijoux.ru +compliance-elena.top +compliance-walter.top +coolgramgoods.com +cracksplanet.com +dailyrank.net +date-withme.com +dekorkeramik.ru +descargar-musica-gratis.net +detoxmed24.ru +dev.citetick.com +dev-seo.blog +dogoimage.com +dokfilms.net +dostavimvdom.ru +e2click.com e-collantes.com -ege-essay.ru -elektroniksigaraankara.info -enpolis.ru -evening-dating-club.info -expdom.com -extads.net -ezigarettekaufen2.yolasite.com -fable.in.ua -facebook-mobile.xyz -fashion-stickers.ru -fefo.gdn -feorina.ru -fettefrauen.net -fikasound.tk -find1friend.com -fix-website-errors.com -flash4fun.com -ge0ip.net -ggiaro.com -ghostvisitor.com -gidonline.one -given2.com -go2jump.org -golmau.host.sk -gotwebsite1.com -gratis-sexkontakte.com -gribokstop.com -gsasearchenginerankerdiscount.com -handicapbathtubarea.com -hchha.com -hdapp1008-a.akamaihd.net -hollywoodactress.info -idc.com.ua -idealtits.net -ikearugs.xyz +elektrischezigarette2.onsugar.com +elitedollars.com +empis.magix.net +enhand.se +escort-russian.com +f1nder.org +facecup.top +falopicm.pw +fba-mexico.com +fickkontaktehobbyhuren.com +filmfanatic.com +forodvd.com +freeuploader.com +fres-news.com +gabeshop.ru +gagrasector.ru +gay-file.com +gazoblok.net.ua +getlamborghini.ga +getyourimage.club +goldpanningtools.com +guiadeserraazul.com +harmonyglen.us +healthcarestore.info +hermesbelts.xyz +hexpilot.com +histats.com +historichometeam.com +hothor.se +i4track.net +ikritikimou.gr imagine-ex.co -imediadesk.com -infoupdate.org -invivo.hu -islamtoday.co.za -isotoner.com -itbc.kiev.ua -jerseyschinabizwholesale.us -katadhin.co -kladrus.ru -kongoultry.net -konkursov.net -konteiner24.com -koversite.info -kovrenok.ru -laapp.com -lanasshop.ru -law-check-seven.xyz -laxob.com -leadn.pl -leftofcentrist.com -lincolntheatre.com -liquimondo.com -low-format.ru -ltvperf.com -marcoislandvacations.net -masserect.com -meddesk.ru -medicines-choice.com -megahdporno.net -metallosajding.ru +imgarit.pw +increasewwwtraffic.info +infazavr.ru +iqoption.com +ispaniya-costa-blanca.ru +james13prix.info +javstock.com +jillepille.com +kamagragelusa.net +kambasoft.com +kerwinandcariza.com +keywesthideaways.co +kinocccp.net +kinomaniatv.pl +knowyournextmove.com +kovesszucs.atw.hu +krasivoe-hd.com +krasnodar.zrus.org +kredit-pod-zalog-krasnodar.ru +larger.io +ldrtrack.com +letolove.ru +letslowbefast.site +liverpool.gsofootball.com +lulea-auktionsverk.se +macotool.com +makemoneyonline.com +members.ghanaweb.com methodsmarketing.com -mockupui.com +mhtr.be +mitrasound.ru +mmofreegames.online +mojaocena.com +mompussy.net moneyviking-a.akamaihd.net -moscow-clining.ru -mustwineblog.com -muz-baza.net -mykings.pw -mymobilemoneypages.com -narkologiya-penza.ru -neks.info -netvouz.com -niroo.info -novodigs.com -oakridgemo.com -oddamzadarmo.eu -odoratus.net -of-ireland.info -oil-td.ru -oops-cinema.ru -orgasmus-virtual.com -paintingplanet.ru -paleohub.info -panouri-solare-acoperis.com -par-fallen.ga -photo.houseofgaga.ru -phrcialiled.com -picturesify.com -pills24h.com -pizda.lol -pons-presse.com -popserve.adscpm.net -profitwithalex.info +moonci.ru +muabancantho.info +muz-shoes.ru +mydeathspace.com +myxdate.info +narkologiya-orel.ru +naturtreenspicerx.pw +nextbackgroundcheck.gq +novgorod.xrus.org +nudepatch.net +nuup.info +ofanda.com +oktube.ru +onclickpredictiv.com +onlinadverts.com +online.ktc45.ru +online-sbank.ru +onlywoman.org +opinionreelle.com +oralsexfilme.net +orel-reshka.net +osoznanie-narkotikam.net +ownshop.cf +paradontozanet.ru +pizdopletka.club +play-movie.pl +playtap.us +poptool.net +pornofeuer.com +pornozhara.com +pretty-mart.com +priscilarodrigues.com.br +prodess.ru +promgirldresses.xyz +pronorm.fr +prospekt-st.ru prostitutki-yaroslavlya.org -push-ad.com -puteshestvennik.com -realnye-otzyvy.info -reelheroes.net +pskcijdc.bloger.index.hr +pulseonclick.com +rarbg.to +reeyanaturopathy.com +relayblog.com remedyotc.com -rocketchange.ru -ruspdd.com -rustic-quiver.win -sale-japan.com -sa-live.com -santasgift.ml -sarafangel.ru -scanner-mary.top -scanner-walt.top -seeingmeerkat.com -segol.tv -seoheap.com -sexgalleries.top -sexsaoy.com -shillyourcoins.com -shoppingmiracles.co.uk -shtaketniki.kz -smailik.org -smokewithrabbits.com -social-s-ggg.xyz -social-s-hhh.xyz -space4updating.win -spymac.net -spywarenuker.com -stanthonyscatholicchurch.org -startwp.org -staticfs.host -steebook.com +running-line.ru +saletool.ru +scanner-alex.top +scanner-donald.top +scrapy.org +scripted.com +secret.xn--oogle-wmc.com +seo-traffic-ranking.info +sexkontakte-seite.com +sexpornotales.net +shop-electron.ru +sibdevice.ru +signoredom.com +simul.co +siteexpress.co.il +site.ru +skanninge.se +sluganarodu.ru +social-search.me +sockshare.net +sotechco.co +spravka130.ru +stairliftsarea.com +stal-rulon.ru +stathat.com stickers-market.ru -students-cheapskate.ml -suggest-keywords.com -suture.co -svtrd.com -tamada69.com -td-33.ru -tizanidineandcipro.blogspot.com -tizanidineduringpregnancy.blogspot.com -tizanidineshowupondrugtest.blogspot.com -tobeyouday.win -toonfamilies.net -top-study.work -tozup.com -tracfone.com -tradedeals.biz -trafficstars.com -traffmonster.info -traxdom.ru -ukrtextbook.com -uptime-delta.net -urlopener.blogspot.com.au -ussearche.cf -uvozdeckych.info -velen.io -videtubs.pl -vigrx-original.ru -vilingstore.net -watchmygf.net -weblo.com -website-datenbank.de -weburok.com -weclipart.com -while.cheapwebsitehoster.com -wikes.20fr.com -woman-tampon.ru -wstroika.ru -wufak.com -xmlinde.com -xmronta.com -xn--80aaagvmjabrs1aoc9luc.xn--p1ai -xn--80aaajbdbddwj2alwjieei2afr3v.xn--p1ai -xn--80ajjbdhgmudixfjc8c5a9df8b.xn--p1ai -xn--hxazdsfy.blogspot.com -yugo-star.ru -zatjmuzu.info -zeikopay.com -z-master.ru -zrizvtrnpale.tk +stjamesschool.info +stroiminsk.com +sugarlyflex.pw +superinterstitial.com +supermama.top +swsociety.se +tasteidea.com +techart24.com +terrafootwear.us +thetardistimes.ovh +tiandeural.ru +tizanidine4mgstreetvalue.blogspot.com +tizanidinehcl2mgsideeffects.blogspot.com +tksn.ru +torrent-newgames.com +tpu.ru +trafficjunky.net +truck-addzilla.life +tubo360.com +uhdtv.website +unimodemhalfduplefw.pen.io +unitygame3d.com +vacances-voyages.info +vesnatehno.com +viagroid.ru +voloo.ru +vseuznaem.com +vtc.pw +webtherapy.ru +wechatdownload10.com +wetgames.ru +whatistizanidinehclusedfor.blogspot.com +whatsappdownload10.com +win-spy.com +wpsecurity.website +xn--90acenikpebbdd4f6d.xn--p1ai +xn--e1afanlbnfckd7c3d.xn--p1ai +xxxsiterips.xyz +yellowfootprints.com +yunque.pluto.ro +zakazvzloma.com +zoodrawings.com +zoolubimets.ru diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index 36ec3f9ef..f0474a154 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -1,24 +1,3 @@ - -
This is merely used for testing !!!
- - - -This is merely used for testing !!!
- \ No newline at end of file diff --git a/.dev-tools/test_units/whitelist-ips.conf b/.dev-tools/test_units/whitelist-ips.conf index d99c8cd8c..bb50a4a64 100644 --- a/.dev-tools/test_units/whitelist-ips.conf +++ b/.dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.122.208.80 0; +34.122.173.211 0; 127.0.0.1 0; diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index 207bc014e..9580e5253 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index 2f2b8fb8f..8ddeb37c7 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 491c8ef88..5763937d2 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.02.2275 +#### Version: V4.2021.02.2276 #### Bad Referrers Blocked: [6932](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [608](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [607](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index c4161ad77..13b259164 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.02.2275 +#### Version: V4.2021.02.2276 #### Bad Referrers Blocked: [6932](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [608](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [607](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index 9aa6954dd..127f4dafa 100755 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.02.2275 +#### Version: V4.2021.02.2276 #### Bad Referrers Blocked: [6932](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [608](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [607](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 93e90b7b7..452907af9 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.02.2275 +#### Version: V4.2021.02.2276 #### Bad Referrers Blocked: [6932](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [608](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [607](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 29888c55a..94946308c 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.02.2275 -### Updated: Wed Feb 10 01:52:43 SAST 2021 +### Version: V4.2021.02.2276 +### Updated: Wed Feb 10 06:06:23 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index ba5790d93..f49141aa0 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.02.2275 -### Updated: Wed Feb 10 01:52:44 SAST 2021 +### Version: V3.2021.02.2276 +### Updated: Wed Feb 10 06:06:25 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 29888c55a..94946308c 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.02.2275 -### Updated: Wed Feb 10 01:52:43 SAST 2021 +### Version: V4.2021.02.2276 +### Updated: Wed Feb 10 06:06:23 SAST 2021 ### Bad Referrer Count: 6932 -### Bad Bot Count: 608 +### Bad Bot Count: 607 ################################################### ### VERSION INFORMATION ## @@ -485,7 +485,6 @@ map $http_user_agent $bad_bot { "~*(?:\b)Nimbostratus(?:\b)" 3; "~*(?:\b)Ninja(?:\b)" 3; "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)node-fetch(?:\b)" 3; "~*(?:\b)NPbot(?:\b)" 3; "~*(?:\b)Nuclei(?:\b)" 3; "~*(?:\b)Nutch(?:\b)" 3; diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 3a471c956..31a805aad 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2021.02.2275 -### Updated: Wed Feb 10 01:52:45 SAST 2021 -### Bad Bot Count: 608 +### Version: V4.2021.02.2276 +### Updated: Wed Feb 10 06:06:26 SAST 2021 +### Bad Bot Count: 607 ################################################### ### Version Information ## @@ -691,8 +691,6 @@ User-agent: Ninja Disallow:/ User-agent: Nmap Disallow:/ -User-agent: node-fetch -Disallow:/ User-agent: NPbot Disallow:/ User-agent: Nuclei