diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md
index 96cc8d25e..e9e5bd65c 100755
--- a/AUTO-CONFIGURATION.md
+++ b/AUTO-CONFIGURATION.md
@@ -4,9 +4,9 @@
### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation:
_______________
-#### Version: V4.2021.10.2661
+#### Version: V4.2021.10.2662
#### Bad Referrers Blocked: [7078](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [623](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Bad User-Agents (Bots) Blocked: [624](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
#### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
____________________
diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md
index 3c9e76915..dfa88cdd6 100755
--- a/MANUAL-CONFIGURATION.md
+++ b/MANUAL-CONFIGURATION.md
@@ -4,9 +4,9 @@
### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation:
_______________
-#### Version: V4.2021.10.2661
+#### Version: V4.2021.10.2662
#### Bad Referrers Blocked: [7078](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [623](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Bad User-Agents (Bots) Blocked: [624](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
#### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
____________________
diff --git a/README.md b/README.md
index 12e1a64bc..f9d041f02 100755
--- a/README.md
+++ b/README.md
@@ -11,9 +11,9 @@
##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots)
_______________
-#### Version: V4.2021.10.2661
+#### Version: V4.2021.10.2662
#### Bad Referrers Blocked: [7078](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [623](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Bad User-Agents (Bots) Blocked: [624](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
#### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
____________________
diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list
index ac16f56fb..6da549abe 100755
--- a/_generator_lists/bad-user-agents.list
+++ b/_generator_lists/bad-user-agents.list
@@ -1,5 +1,3 @@
-DnBCrawler-Analytics
-dataforseo.com
01h4x.com
360Spider
404checker
@@ -114,6 +112,7 @@ Discobot
Discoverybot
Dispatch
DittoSpyder
+DnBCrawler-Analytics
DnyzBot
DomCopBot
DomainAppender
diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md
index 70b4eba16..8f17807c1 100644
--- a/_google_analytics_ghost_spam/README.md
+++ b/_google_analytics_ghost_spam/README.md
@@ -3,9 +3,9 @@
# EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM
_______________
-#### Version: V4.2021.10.2661
+#### Version: V4.2021.10.2662
#### Bad Referrers Blocked: [7078](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [623](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Bad User-Agents (Bots) Blocked: [624](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
#### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
____________________
diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf
index 91d48bee7..76402f5c4 100755
--- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf
+++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.10.2661
-### Updated: Tue Oct 19 22:02:58 UTC 2021
+### Version: V4.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 625
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf
index d66b46396..3116f1292 100644
--- a/conf.d/globalblacklist-testing-version.conf
+++ b/conf.d/globalblacklist-testing-version.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2021.10.2661
-### Updated: Tue Oct 19 22:02:59 UTC 2021
+### Version: V3.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf
index 91d48bee7..76402f5c4 100644
--- a/conf.d/globalblacklist.conf
+++ b/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.10.2661
-### Updated: Tue Oct 19 22:02:58 UTC 2021
+### Version: V4.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 625
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt
index b3750bd3a..a5d11eb88 100644
--- a/dev-tools/_robots_input/robots-input.txt
+++ b/dev-tools/_robots_input/robots-input.txt
@@ -112,6 +112,7 @@ Discobot
Discoverybot
Dispatch
DittoSpyder
+DnBCrawler-Analytics
DnyzBot
DomCopBot
DomainAppender
diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf
index d66b46396..3116f1292 100644
--- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf
+++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2021.10.2661
-### Updated: Tue Oct 19 22:02:59 UTC 2021
+### Version: V3.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 5caad0012..8528be6c6 100644
--- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 1;
+10.1.0.56 1;
127.0.0.1 1;
diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index 19f6f3310..ce3618377 100644
--- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 0;
+10.1.0.56 0;
127.0.0.1 0;
diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index d66b46396..3116f1292 100644
--- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2021.10.2661
-### Updated: Tue Oct 19 22:02:59 UTC 2021
+### Version: V3.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 5caad0012..8528be6c6 100644
--- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 1;
+10.1.0.56 1;
127.0.0.1 1;
diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index 19f6f3310..ce3618377 100644
--- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 0;
+10.1.0.56 0;
127.0.0.1 0;
diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
index d66b46396..3116f1292 100644
--- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2021.10.2661
-### Updated: Tue Oct 19 22:02:59 UTC 2021
+### Version: V3.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
index 5caad0012..8528be6c6 100644
--- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 1;
+10.1.0.56 1;
127.0.0.1 1;
diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf
index 7a5f576ef..f4c0a8be6 100644
--- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf
+++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf
@@ -173,6 +173,7 @@
"~*(?:\b)Discoverybot(?:\b|)" 0;
"~*(?:\b)Dispatch(?:\b|)" 0;
"~*(?:\b)DittoSpyder(?:\b|)" 0;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b|)" 0;
"~*(?:\b)DnyzBot(?:\b|)" 0;
"~*(?:\b)DomCopBot(?:\b|)" 0;
"~*(?:\b)DomainAppender(?:\b|)" 0;
diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
index 19f6f3310..ce3618377 100644
--- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 0;
+10.1.0.56 0;
127.0.0.1 0;
diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
index d66b46396..3116f1292 100644
--- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2021.10.2661
-### Updated: Tue Oct 19 22:02:59 UTC 2021
+### Version: V3.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber
index 71150e880..58d32dc6b 100644
--- a/dev-tools/buildnumber
+++ b/dev-tools/buildnumber
@@ -1 +1 @@
-2661
+2662
diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template
index d66b46396..3116f1292 100644
--- a/dev-tools/globalblacklist-testing.template
+++ b/dev-tools/globalblacklist-testing.template
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2021.10.2661
-### Updated: Tue Oct 19 22:02:59 UTC 2021
+### Version: V3.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template
index 91d48bee7..76402f5c4 100755
--- a/dev-tools/globalblacklist.template
+++ b/dev-tools/globalblacklist.template
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.10.2661
-### Updated: Tue Oct 19 22:02:58 UTC 2021
+### Version: V4.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 625
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
index c7cd7160b..7aef4fbff 100644
--- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
+++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
@@ -45,4 +45,4 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-20.69.94.157 0;
+13.92.117.253 0;
diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf
index 91d48bee7..76402f5c4 100644
--- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf
+++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.10.2661
-### Updated: Tue Oct 19 22:02:58 UTC 2021
+### Version: V4.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 625
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 5caad0012..8528be6c6 100644
--- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 1;
+10.1.0.56 1;
127.0.0.1 1;
diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index 19f6f3310..ce3618377 100644
--- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 0;
+10.1.0.56 0;
127.0.0.1 0;
diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index 91d48bee7..76402f5c4 100644
--- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.10.2661
-### Updated: Tue Oct 19 22:02:58 UTC 2021
+### Version: V4.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 625
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 5caad0012..8528be6c6 100644
--- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 1;
+10.1.0.56 1;
127.0.0.1 1;
diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index 19f6f3310..ce3618377 100644
--- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 0;
+10.1.0.56 0;
127.0.0.1 0;
diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
index 91d48bee7..76402f5c4 100644
--- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.10.2661
-### Updated: Tue Oct 19 22:02:58 UTC 2021
+### Version: V4.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 625
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
index 5caad0012..8528be6c6 100644
--- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 1;
+10.1.0.56 1;
127.0.0.1 1;
diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf
index 7a5f576ef..f4c0a8be6 100644
--- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf
+++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf
@@ -173,6 +173,7 @@
"~*(?:\b)Discoverybot(?:\b|)" 0;
"~*(?:\b)Dispatch(?:\b|)" 0;
"~*(?:\b)DittoSpyder(?:\b|)" 0;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b|)" 0;
"~*(?:\b)DnyzBot(?:\b|)" 0;
"~*(?:\b)DomCopBot(?:\b|)" 0;
"~*(?:\b)DomainAppender(?:\b|)" 0;
diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
index 19f6f3310..ce3618377 100644
--- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 0;
+10.1.0.56 0;
127.0.0.1 0;
diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
index 91d48bee7..76402f5c4 100644
--- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.10.2661
-### Updated: Tue Oct 19 22:02:58 UTC 2021
+### Version: V4.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 625
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
index c7cd7160b..7aef4fbff 100644
--- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
+++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
@@ -45,4 +45,4 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-20.69.94.157 0;
+13.92.117.253 0;
diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf
index 65f3e32dc..4e65cb86c 100644
--- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf
+++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.10.2661
-### Updated: Tue Oct 19 22:02:58 UTC 2021
+### Version: V4.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 625
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
index c7cd7160b..7aef4fbff 100644
--- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
+++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
@@ -45,4 +45,4 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-20.69.94.157 0;
+13.92.117.253 0;
diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf
index ba617d990..87be5ea34 100644
--- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf
+++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.10.2661
-### Updated: Tue Oct 19 22:02:58 UTC 2021
+### Version: V4.2021.10.2662
+### Updated: Wed Oct 20 05:20:24 UTC 2021
### Bad Referrer Count: 7078
-### Bad Bot Count: 623
+### Bad Bot Count: 625
###################################################
### VERSION INFORMATION ##
@@ -261,6 +261,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)Discoverybot(?:\b)" 3;
"~*(?:\b)Dispatch(?:\b)" 3;
"~*(?:\b)DittoSpyder(?:\b)" 3;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b)" 3;
"~*(?:\b)DnyzBot(?:\b)" 3;
"~*(?:\b)DomCopBot(?:\b)" 3;
"~*(?:\b)DomainAppender(?:\b)" 3;
diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf
index 5caad0012..8528be6c6 100644
--- a/dev-tools/test_units/blacklist-ips.conf
+++ b/dev-tools/test_units/blacklist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 1;
+10.1.0.56 1;
127.0.0.1 1;
diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf
index 7a5f576ef..f4c0a8be6 100644
--- a/dev-tools/test_units/blacklist-user-agents.conf
+++ b/dev-tools/test_units/blacklist-user-agents.conf
@@ -173,6 +173,7 @@
"~*(?:\b)Discoverybot(?:\b|)" 0;
"~*(?:\b)Dispatch(?:\b|)" 0;
"~*(?:\b)DittoSpyder(?:\b|)" 0;
+ "~*(?:\b)DnBCrawler-Analytics(?:\b|)" 0;
"~*(?:\b)DnyzBot(?:\b|)" 0;
"~*(?:\b)DomCopBot(?:\b|)" 0;
"~*(?:\b)DomainAppender(?:\b|)" 0;
diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list
index 258ab5ef3..d47ecd4d0 100644
--- a/dev-tools/test_units/random-bots-for-test-quick.list
+++ b/dev-tools/test_units/random-bots-for-test-quick.list
@@ -1,10 +1,10 @@
-AiHitBot
-Devil
-Gotit
-LeechGet
-LexiBot
-Niki-bot
-ProWebWalker
-Titan
-WinHTTrack
-netEstate NE Crawler
+Download Wonder
+GT::WWW
+Magnet
+Nessus
+Openfind
+RankingBot2
+RebelMouse
+SeobilityBot
+Tracemyfile
+Upflow
diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list
index 001742690..fec095237 100644
--- a/dev-tools/test_units/random-bots-for-test.list
+++ b/dev-tools/test_units/random-bots-for-test.list
@@ -1,250 +1,250 @@
-404checker
+01h4x.com
+360Spider
+404enemy
+80legs
AIBOT
-ASPSeek
+ALittle Client
Abonti
Aboundexbot
+Acunetix
AhrefsBot
-AiHitBot
+Aipbot
+Alexibot
+AllSubmitter
Anarchie
-Anthill
-Apexoo
-Asterias
+Anarchy
+Anarchy99
+Aspiegel
+AwarioRssBot
BBBike
-Backlink-Ceck
-Badass
-Barkrowler
-BatchFTP
-Battleztar Bazinga
+BDFetch
+BackWeb
+BacklinkCrawler
+Bandit
+BetaBot
Bigfoot
Bitacle
-BlackWidow
Blackboard
BlowFish
Boardreader
-Brandprotect
-BuiltBotTough
+Bolt
+Brandwatch
+Buddy
Bullseye
-Cegbfeieh
-CensysInspect
-CheTeam
+CATExplorador
+Calculon
CherryPicker
-ChinaClaw
-Cliqzbot
+Chlooe
+Claritybot
Cloud mapping
Cocolyzebot
-CopyRightCheck
-Crawling at Home Project
+Cogentbot
+Collector
+Copier
+Craftbot
CrazyWebCrawler
CrunchBot
-DBLBot
+CyotekWebCopy
DIIbot
DSearch
DataCha0s
+DatabaseDriverMysqli
Deusu
-Dirbuster
+Discobot
DittoSpyder
-DomCopBot
-Domains Project
-Download Wonder
+DomainAppender
+DomainStatsBot
Drip
+EMail Siphon
EMail Wolf
-EasyDL
-Ebingbong
-Ecxi
EirGrabber
-EroCrawler
Evil
Exabot
ExtLinksBot
-Extractor
-Extreme Picture Finder
-Ezooms
-FDM
-Fimap
-Firefox/7.0
-Flunky
+EyeNetIE
+FHscan
+FemtosearchBot
Foobot
-FyberSpider
-Fyrebot
-GT::WWW
+FrontPage
Genieo
GermCrawler
-GetRight
-Getintent
+Go!Zilla
Go-Ahead-Got-It
-GrabNet
-Grafula
-GrapeFX
+Grabber
GridBot
-HEADMasterSEO
+HTTP::Lite
+HTTrack
Haansoft
HaosouSpider
-Harvest
-Havij
-Heritrix
HybridBot
-IDBTE4M
+IDBot
Iblog
+IlseBot
+Image Fetch
IndeedBot
+Indy Library
Intelliseek
-Internet Ninja
+InterGET
Iria
-Iskanie
JamesBOT
-JikeSpider
Keyword Density
-Kozmosbot
+LNSpiderguy
Leap
-LexiBot
+LibWeb
+Libwhisker
Likse
+LinkScan
LinkWalker
Linkbot
+Linkdexbot
+LinksManager
LinqiaScrapeBot
+Lipperhey Spider
Lmspider
-Ltx71
-MQQBrowser
+MFC_Tear_Sample
+MJ12bot
+MSFrontPage
MSIECrawler
-Magnet
-Mail.RU_Bot
Majestic SEO
+Majestic12
MarkMonitor
-Masscan
-Mb2345Browser
+Mass Downloader
+Mata Hari
+MauiBot
+MeanPath Bot
Mediatoolkitbot
MegaIndex.ru
Metauri
-Mozlila
-NPbot
-Nameprotect
-Needle
-NetMechanic
+MicroMessenger
+Mister PiX
+Moblie Safari
+Mojeek
+Mojolicious
+Mr.4x3
+Name Intelligence
+NetAnts
NetZIP
-Nettrack
+Netcraft
+Netvibes
NextGenSearchBot
-Nibbler
+Niki-bot
Nikto
-Nmap
+Ninja
+Octopus
Offline Navigator
OnCrawl
-OutclicksBot
-PECL::HTTP
+OpenVAS
+OrangeSpider
+PHPCrawl
+POE-Component-Client-HTTP
Page Analyzer
-PageScorer
-PageThing.com
+PageGrabber
Pandalytics
-Petalbot
+PeoplePal
Pi-Monster
-PictureFinder
-Psbot
-Pump
+Pixray
PyCurl
-RankActiveLinkBot
-Re-re
+QueryN Metasearch
+RSSingBot
+RankFlex
+RankingBot2
+Rankivabot
ReGet
RealDownload
Reaper
-Recorder
RedesScrapy
RepoMonkey
Ripper
+Rogerbot
+SEOkicks-Robot
SEOstats
Scanbot
-Scrapy
-ScreenerBot
-Searchestate
-SeoSiteCheckup
Seomoz
-Site Sucker
-SiteCheckerBotCrawler
+Siphon
+SiteExplorer
SiteSnagger
SiteSucker
-Sitevigil
+Sitebeam
+Siteimprove
+SmartDownload
Snake
-Snapbot
-Sogou web spider
+Snoopy
+Sosospider
+Sottopop
SpaceBison
Spammen
-SpankBot
+Spinn3r
+SputnikBot
+Sqlworm
+Sqworm
Steeler
-Stripper
-SuperHTTP
-Surfbot
-Swiftbot
-Telesphorep
+Telesoft
+The Intraformant
+Thumbor
Toweyabot
-Tracemyfile
-Trendictionbot
-True_Robot
-TurnitinBot
+Turingos
TwengaBot
Twice
-URLy Warning
+Typhoeus
URLy.Warning
-UnisterBot
-Upflow
-V-BOT
-VCI
+VB Project
+Vagabondo
VidibleScraper
-VoidEYE
+Virusdie
+Voltron
WASALive-Bot
-WBSearchBot
-WISENutbot
-WPScan
-Wallpapers
-WallpapersHD
-Web Collage
+WWW-Mechanize
+WWW::Mechanize
Web Pix
-Web Sauger
-Web Sucker
WebBandit
-WebCollage
+WebCopier
WebEnhancer
+WebFetch
+WebFuck
+WebGo IS
+WebImageCollector
WebLeacher
-WebWhacker
-WebZIP
-Webalta
-WebmasterWorldForumBot
+WebPix
+WebReaper
WebsiteExtractor
-WebsiteQuester
+Webster
+Whack
Whatweb
Widow
-WinHTTrack
WiseGuys Robot
-Xaldon WebSpider
-Xenu
-Zade
+Wprecon
+Xaldon_WebSpider
+Zauba
+Zermelo
+Zeus
ZmEu
-ZoomBot
+ZoominfoBot
ZumBot
+ZyBorg
archive.org_bot
arquivo.pt
autoemailspider
check1.exe
-clark-crawler
com.plumanalytics
-crawler.feedback
-domainsproject.org
+crawl.sogou.com
+dataforseo.com
evc-batch
gopher
-ips-agent
-lwp-request
+instabid
meanpathbot
-muhstik-scan
-pcBrowser
-plumanalytics
-polaris version
-probe-image-size
+oBot
+page scorer
ripz
-satoristudio.net
scalaj-http
-scan.lol
-seoscanners
-seostar
-sitechecker.pro
-siteripz
+seobility
+serpstatbot
+sp_auditbot
spyfu
-trendiction.com
+sysscan
trendiction.de
ubermetrics-technologies.com
-x22Mozilla
+voyagerx.com
+webpros.com
+webprosbot
xpymep1.exe
+zgrab
diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list
index 424eb0064..957e7c14f 100644
--- a/dev-tools/test_units/random-bots-for-whitelist-test.list
+++ b/dev-tools/test_units/random-bots-for-whitelist-test.list
@@ -1,250 +1,250 @@
-01h4x.com
+360Spider
+404enemy
+80legs
+ADmantX
ALittle Client
ASPSeek
-Acunetix
AiHitBot
+Aipbot
+Alexibot
+AllSubmitter
+Alligator
Anarchie
-Anarchy
+Ankit
Anthill
-Aspiegel
-BBBike
-BDFetch
-BackDoorBot
-BackWeb
-Bandit
-Barkrowler
+AwarioSmartBot
+BDCbot
+BackStreet
+BacklinkCrawler
+BatchFTP
Battleztar Bazinga
-Bigfoot
-Bitacle
+BetaBot
Black Hole
Blackboard
-Blow
-Brandprotect
-Buck
-BunnySlippers
+Boardreader
+Bolt
+BotALot
+Brandwatch
+BuiltBotTough
CATExplorador
-Cegbfeieh
+CSHttp
+CensysInspect
CheeseBot
-CherryPicker
-Cliqzbot
Cocolyzebot
-Cogentbot
Collector
-Copier
-Copyscape
Cosmos
+Craftbot
Crawling at Home Project
-Crescent
-DSearch
-DatabaseDriverMysqli
+Curious
+Custo
+CyotekWebCopy
Demon
Dirbuster
-Discobot
-Dispatch
-DnyzBot
-DomainAppender
-Dotbot
+Discoverybot
+DomainSigmaCrawler
+Domains Project
+Dragonfly
Drip
-ECCP/1.0
-EMail Wolf
EasyDL
-EroCrawler
-Evil
-Extractor
-EyeNetIE
-FDM
-FemtosearchBot
-Firefox/7.0
-FlashGet
+EirGrabber
+Exabot
+ExtractorPro
+Fimap
Flunky
Foobot
Fuzz
-Fyrebot
-GermCrawler
-GetWeb
+FyberSpider
+Genieo
+GetRight
Getintent
Gigabot
-GoZilla
+Go!Zilla
+Gotit
+GrabNet
+Grabber
Grafula
GrapeFX
-HMView
-Haansoft
-Harvest
+GrapeshotCrawler
+HEADMasterSEO
+HTMLparser
+HTTP::Lite
Havij
-Hloader
-IDBot
+Heritrix
+Humanlinks
+HybridBot
+IDBTE4M
IRLbot
-Iblog
-Id-search
-IlseBot
Image Fetch
-Internet Ninja
+Indy Library
+IstellaBot
JOC Web Spider
-JamesBOT
Jbrofuzz
JennyBot
-JetCar
-Jetty
+JikeSpider
Joomla
-Kenjin Spider
-LWP::Simple
+JustView
+Jyxobot
+Kozmosbot
Lanshanbot
-Larbin
-LeechFTP
-LexiBot
+Leap
+Lftp
LieBaoFast
-Likse
-LinkScan
LinkWalker
+Linkbot
LinksManager
-LinqiaScrapeBot
-Lipperhey
-Lmspider
-MIDown tool
+Ltx71
+MFC_Tear_Sample
+MSFrontPage
MSIECrawler
+MTRobot
+Mag-Net
Magnet
-Majestic-SEO
-Masscan
+MarkWatch
Mata Hari
+MauiBot
+Mb2345Browser
MeanPath Bot
-MegaIndex.ru
-Microsoft Data Access
+Mediatoolkitbot
Mister PiX
+Moblie Safari
Morfeus Fucking Scanner
-Mr.4x3
Msrabot
Musobot
-Nameprotect
-Navroad
+NearSite
+Net Vampire
NetAnts
-NetLyzer
-NextGenSearchBot
-Nibbler
-Niki-bot
-Nikto
+NetMechanic
+Netvibes
Nimbostratus
-Ninja
-Nuclei
-Offline Explorer
-OrangeBot
-PECL::HTTP
+Octopus
+Openfind
+Openvas
+OutfoxBot
+POE-Component-Client-HTTP
Page Analyzer
-PageAnalyzer
-PageThing.com
-Pandalytics
+PageGrabber
+PageScorer
+Panscient
Papa Foto
+Petalbot
Pi-Monster
+Picscout
PictureFinder
Pixray
-PleaseCrawl
-Pockey
-Probethenet
-PyCurl
+PxBroker
+QueryN Metasearch
+Quick-Crawler
RankActive
RankActiveLinkBot
-RankurBot
+RankFlex
+RankingBot
+RankingBot2
+Rankivabot
Re-re
+ReGet
RealDownload
-Ripper
-Rogerbot
+Reaper
+RebelMouse
SEOkicks-Robot
-SEOlyticsCrawler
-SEOstats
-Scanbot
-ScrepyBot
-SemrushBot
-SentiBot
-Seomoz
-Shodan
-SiteCheckerBotCrawler
-SiteLockSpider
+SEOprofiler
+SISTRIX
+ScanAlert
+ScoutJet
+ScreenerBot
+SearchmetricsBot
+SeobilityBot
+Siphon
+Site Sucker
SiteSnagger
-SlySearch
-SmartDownload
+Sitebeam
+Siteimprove
Snoopy
Sociscraper
+Sogou web spider
SpaceBison
Spammen
SpankBot
-SputnikBot
Sqworm
-Stripper
SuperBot
-Surfbot
-SurveyBot
+SuperHTTP
+Swiftbot
Szukacz
+T0PHackTeam
+Teleport
+TeleportPro
Telesoft
+Telesphoreo
Thumbor
Titan
+Tracemyfile
Trendiction
Trendictionbot
+Turnitin
TwengaBot
Twice
+Typhoeus
+URLy.Warning
UnisterBot
-V-BOT
+Upflow
VB Project
VCI
-Virusdie
-VoidEYE
+VeriCiteCrawler
+VidibleScraper
Voil
-WASALive-Bot
-WEBDAV
WPScan
WWW-Mechanize
Wallpapers/3.0
-WallpapersHD
-Web Auto
Web Collage
Web Enhancer
+Web Fetch
Web Pix
Web Sauger
-Web Sucker
+WebBandit
WebCopier
-WebEnhancer
-WebPix
+WebFetch
+WebStripper
WebWhacker
-WebZIP
-Webalta
-WebmasterWorldForumBot
Webshag
+WebsiteQuester
Whack
-Whacker
+Whatweb
+Who.is Bot
+Widow
WinHTTrack
-Wonderbot
-Xaldon WebSpider
+Wotbox
+Xaldon_WebSpider
Xenu
-YoudaoBot
Zade
-Zauba
Zermelo
Zeus
+Zitebot
ZmEu
-ZoomBot
+ZoominfoBot
ZyBorg
archive.org_bot
-arquivo-web-crawler
-arquivo.pt
autoemailspider
-check1.exe
+cah.io.community
coccocbot-web
+cognitiveseo
crawl.sogou.com
-eCatch
+crawler.feedback
+demandbase-bot
+facebookscraper
+heritrix
instabid
-ips-agent
+isitwp.com
+lwp-request
lwp-trivial
-magpie-crawler
+meanpathbot
mediawords
-page scorer
-polaris version
-s1z.ru
+satoristudio.net
scalaj-http
-scan.lol
-seoscanners
-sexsearcher
sitechecker.pro
-siteripz
-sogouspider
-sp_auditbot
-tAkeOut
+spyfu
+sysscan
trendiction.de
+voyagerx.com
webpros.com
+webprosbot
+x22Mozilla
+zauba.io
diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list
index ab688f223..cf16ddc82 100644
--- a/dev-tools/test_units/random-referrers-for-test-quick.list
+++ b/dev-tools/test_units/random-referrers-for-test-quick.list
@@ -1,10 +1,10 @@
-1j7740kd.website
-aramaicmedia.org
-bookmark4you.com
-filmi-onlain.info
-free-stock-illustration.com
-investyb.com
-saugatuck.com
-school-diplomat.ru
-vezuviy.su
-westen-v.life
+alienwheels.de
+focalink.com
+lost-alpha.ru
+mamasuper.prom.ua
+myyour.eu
+news-speaker.com
+picmoonco.pw
+r-e-f-e-r-e-r.com
+uptime-gamma.net
+visitcambridge.org
diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list
index 358895265..ee104edea 100644
--- a/dev-tools/test_units/random-referrers-for-test.list
+++ b/dev-tools/test_units/random-referrers-for-test.list
@@ -1,500 +1,500 @@
-00webcams.com
-0912701309f8ce.com
-1001desert.com
-1adult.com
-2020iscoming.info
-2x2fan.ru
-4rent.ru
-76brighton.co.uk
-9icmzvn6.website
+1008.su
+123cha.com
+12masterov.com
+1bet.com
+1xbet4you.com
+2728fb936f0.com
+5i2.net
+7ooo.ru
+888.com
+91abcw.com
a1security.com.ua
-aasoldes.fr
-above.com
-addtoadd.men
-adexprts.com
-adsland.men
-adviceforum.info
-afora.ru
-aksonural.ru
-akusherok.ru
-alert-fjg.xyz
-algarveglobal.com
-all4wap.ru
-alloysteel.ru
-allpdfmags.net
-alvtank.se
-anime.dougasouko.com
-anti-virus-removal.info
-ap.senai.br
+aac52006ec82a24e08b665f4db2b5013f7.com
+acads.net
+ad-words.ru
+adcash.com
+adprotect.net
+ads.gold
+advancedcleaner.com
+advertex.info
+advokateg.ru
+agadelha.com.br
+aktobe.xkaz.org
+akuhni.by
+alborzan.com
+alfa-medosmotr.ru
+allwidewallpapers.com
+amazon-seo-service.com
+ample-awards-today.us
+ananumous.ru
+apiallgeniusinfo-a.akamaihd.net
apimegabrowsebiz-a.akamaihd.net
-apiwoodensealcom-a.akamaihd.net
-appsaurus.com
-apptester.tk
-arius.tech
-arvut.org
-audiofree.ru
-australia-opening-times.com
-autoblogger24.info
-automatic-seo.com
-autoplate.info
-autovideobroadcast.com
-avon-severozapad.ru
-ayanaspa.com
-azartmsl.com
-azlex.uz
-back.dog
-bambi.ck.ua
-bamo.xsl.pt
-bbtec.net
-best-seo-software.xyz
-bestchoice.cf
-beztuberkuleza.ru
-bigcities.org
-bildsuche.ru
-biocn.dx.am
-blacktwhite.com
-bluesalt.co
-bmwhighperformers.com
-bongacams.com
-bonvillan.com
-boobsimge.com
-boom-celebs.com
-brains2.biz
-bristolhostel.com
-buigas.00it.com
-buypuppies.ca
-buytizanidineonlinenoprescription.blogspot.com
-buyviagraa.blogspot.com
-buzzzg.men
-c-english.ru
-cactussoft.biz
-candiceloves.us
-canrole.com
-canyougethighofftizanidine.blogspot.com
-cbcseward.com
-cbozhe.com
-cfacarrosserie74.com
-charmstroy.info
-chatroulette.life
-cheapmarmot.online
+apispringsmartne-a.akamaihd.net
+appfastplay.com
+appfixing.space
+apple.com-cleaner.systems
+asociatia-tipografilor-transilvania.ro
+auto.rusvile.lt
+autodo.info
+autonew.biz
+aviav.co
+aviav.ru.com
+avtointeres.ru
+axbocz.net
+babosas.com
+backlink4u.net
+bannerpower.com
+bard-real.com.ua
+beauby.ru
+beauty-things.com
+berdasovivan.ru
+bestwaystogetpaid.us
+bharatdefencekavach.com
+bigames.online
+bigpictures.club
+bimatoprost-careprost.com
+bingoporno.com
+binomo.kz
+bird1.ru
+bitcoin-ua.top
+bizfly.info
+blackbabesporn.com
+blagovest-med.ru
+blockety.co
+blog.axant.it
+bloglag.com
+bluerobot.info
+bnt-team.com
+boltalko.xyz
+brainzpod.win
+brandbucket.com
+breget74.ru
+brendbutik.ru
+buckspar.gq
+burger-tycoon.com
+buy-loft.ru
+buy-meds24.com
+buybest1.biz
+buyclomidonlaine.com
+buyparajumpers.online
+cacheimages.com
+cah.io.community
+cartierbracelet.xyz
+cartierlovebracelet.xyz
+cartiertrinity.xyz
+cartujano-pre.de
+cementaresearch.se
+cenokos.ru
cheapmoncler.pw
-checkhit.com
-chee-by.biz
-chelnytruck.ru
+cheatcode-lita12.rhcloud.com
+check-host.net
+choice-credit.ru
chuckguilford.com
-ci.ua
-cialisovercounteratwalmartusa.com
-classicakuhni.ru
-clayvasedesigns.tk
-cleanallvirus.com
-clickbank.net
-clickprotects.com
-clippingphotoindia.com
-com-supportcenter.website
-compliance-alex.xyz
-compliance-george.top
-compliance-jane.top
-conboy.us
-constantaservice.net
-countercrazy.com
-cs-passion.pl
-customchocolate.business-for-home.com
-cyse.tk
-czat.wp.pl
-d0t.ru
-dalavia.ru
-daretodonate.co
-dawlenie.com
-dbmkfhqk.bloger.index.hr
-ddpills.com
-dealitright.click
-dear-diary.ru
-dekoration.us
-depositfiles-porn.ga
-detalizaciya-tut.biz
-dichvuvesinhhanoi.com
-digitalfaq.com
+citizenclsdriveri7.pen.io
+clarithromycin500mg.com
+cocolyze.com
+codysbbq.com
+cognitiveseo.com
+compliance-ivan.top
+coolingoods.com
+counter.bloke.com
+cowblog.fr
+cracksplanet.com
+crest-poloski.ru
+custodieva.ru
+darkbooks.org
+ddsoldes.fr
+demon-tweeks.com
+deniven.1bb.ru
+deshevo-nedorogo.ru
+diegolopezcastan.com
+digital-video-processing.com
dignitasdata.se
-directivepub.com
-dirtpics.pw
-discovertreasure-a.akamaihd.net
-djonwatch.ru
-dogclothing.store
+div.as
+dkvorota.ru
+doctormakarova.ru
+documentsite.net
+dodlive.mil
+domaincheck.io
+domaincrawler.com
+domashneeruporno.com
+dood.live
+dostavka-v-krym.com
drhomes.biz
-drunkenstepfather.com
-duawitchrarato.tk
-duplicashapp.com
-dynainbox.com
-e-kwiaciarz.pl
-eachdayisagift.review
-earl-brown.info
-educontest.net
+drivotracker.com
+drugs-no-rx.info
+dtm-spain.com
+dustyorate.com
+dvervmoskvu.ru
+e-c.al
+earncash.com.ua
+edge.sharethis.com
+ekaterinburg.xrus.org
elektrozigarette2.webs.com
-elektrozigarette2.wordpressy.pl
+elektrozigaretten1.postbit.com
+elexies.info
elidelcream.weebly.com
-es5.com
-eshop4u.jp
-essay-writing.work
+eloxal.ru
+elvenmachine.com
+emediate.eu
+englate.com
estelight.ru
-estibot.com
-everytalk.tv
-exonline.info
-exportshop.us
-extremez.net
-f-loaded.de
-f00kclan.de
-f1nder.org
-family1st.ca
-fbdownloader.com
-fearcrow.com
-ferrotodo.com
-ff30236ddef1465f88547e760973d70a.com
-filesclub.net
-firstdrugmall.ru
-fiuxy.com
-fix-website-errors.com
-floridamhca.org
-fortwosmartcar.pw
-forum.tvmir.org
-foto-telok.net
-fotzen-ficken.com
-foxtechfpv.com
-free-gluten.ru
-game-mmorpg.net
-gameonasia.com
-getmyads24.com
-gobongo.info
-golfresa.lucania.se
-googlepositions.com
-googleseo.com.tr
-gotwebsite1.com
-greenzaim.ru
-guildebzh.info
-gwebtools.com.br
-gz2.bbsoldes.fr
-hamilton.ca
-handicapbathtubarea.com
-hasshe.com
-hauleddes.com
-haveinc.xyz
-hawaiisurf.com
-hccoder.info
-hchha.com
-hdimagegallery.net
-hdseriale.pl
-hledejvshopech.cz
-home-task.com
-hotgirlhdwallpaper.com
-hspline.com
-humanelydrew.com
-iboss.com
-ideashome.id
-idegenvezeto.eu
-igru-xbox.net
+etm-consult.de
+euromasterclass.ru
+europages.com.ru
+europeanwatches.ru
+evogarage.com
+eyelike.com.ua
+ezigarettekaufen2.yolasite.com
+falllow.gq
+familyholiday.ml
+familyphysician.ru
+fashion.stellaconstance.co
+fast-torrent.ru
+fatmaelgarny.com
+femdom.twiclub.in
+feorina.ru
+ferieboligkbh.dk
+fickkontaktehobbyhuren.com
+filesdatabase.net
+finejewelryshop.ru
+flash4fun.com
+forexunion.net
+formseo.com
+forodvd.com
+foto-weinberger.at
+free-deals.faith
+free-fbook-traffic.com
+freejabs.com
+frustrated-favorable.gq
+fullfileaccess.com
+funnypica.com
+galaxyflowers.ru
+game300.ru
+gamedayhouse.com
+gay-site.store
+generic-pills-online.com
+genericviagrasildenafiled.net
+gentamicineyedrops.blogspot.com
+get-free-social-traffic.com
+getoutofdebtfree.org
+given2.com
+goodly.pro
+goodprotein.ru
+gorodservis.ru
+gpms.org.my
+grand-chlen.ru
+greenidesign.co
+growshop.es
+gsasearchengineranker.site
+gsmlab.pl
+guardlink.org
+guitar-master.org
+hdfreeporno.net
+headpress.ru
+hentai-manga.porn
+hermesbelts.xyz
+hermesbracelets.xyz
+hermesreplica.pw
+himazin.info
+hiwibyh.bugs3.com
+hjaoopoa.top
+hobild.net
+hop.clickbank.net
+hoporno.com
+hoverboard360.es
+https-legalrc.biz
+hully.altervista.org
+ideoworld.org
ihc2015.info
-iminent.com
-infokonkurs.ru
-inmate-locator.us
-insta-add.pro
-investpamm.ru
-iusstf.org
-ix20.ru
-jabimgo.pw
-jackwolfskinoutlet.online
-japfm.com
-jav-fetish.site
+imfamous.info
+indetiske.ya.ru
+infoupdate.org
+internetartfair.com
+invitefashion.com
+iradiology.ru
+istripper.com
+itag.pw
+ivanovo.zrus.org
+izhevsk.zrus.org
+izoll.ru
+javcoast.com
javlibrary.cc
-javspace.net
-je7.us
-jerseychinabizwholesale.com
+javrip.net
+juliaworld.net
+junketjuice.blogspot.com
jurajskie.info
-justdating.online
-kamalsinha.com
-kanimage.com
-karadene.com
-kazan.xrus.org
-keki.info
-kellyonline.xyz
-kevblog.top
-keywordbasket.com
-keywordsuggest.org
-keywordteam.net
-kimcurlrvsms.com
-kino-ecran.ru
-kndxbkdx.bloger.index.hr
-kolotiloff.ru
-konoplisemena.com
-kosova.de
-kraljeva-sutjeska.com
-kustanay.kz
-ladov.ru
-ladsup.com
-laptoper.net
-lennatin.info
-letto.by
-liky.co.ua
-liquimondo.com
-live-cam6.info
-livejasmin.com
-loadingpages.me
-locksmith.jp
-lomb.co
-lovasszovetseg.hu
-macnewtech.com
-mail.allnews24.in
-makedo.ru
-map028.com
-marmitaco.cat
-marsgatan.com
-maslenka.kz
-maximpartnerspr.com
-meccadumps.net
-medicineseasybuy.com
-mejoresfotos.eu
-metaxalonevstizanidine.blogspot.com
-michaelkorsoutlet.store
-microsearch.ru
-minyetki.ru
-missis.top
-mncrftpcs.com
-mnogolok.info
-monarchfind-a.akamaihd.net
-monclerboots.xyz
-monetizer.com-01.site
-money-every-day.com
-moneyviking-a.akamaihd.net
-monitorwebsitespeed.com
-morf.snn.gr
-morlat.altervista.org
-motherboard.vice.com
-muabancantho.info
-music.utrolive.ru
+jyrxd.com
+karachev-city.ru
+kerwinandcariza.com
+keywordhut.com
+keywordsuggests.com
+kino-fun.ru
+kirov.zrus.org
+kleine-titten.biz
+kol-energo.ru
+komp-pomosch.ru
+koopilka.com
+krasivye-devushki.net
+kskjco.club
+kung-fu-ru.com
+kupit-adenu.ru
+la-fa.ru
+lacloop.info
+laserpen.club
+laulini.soclog.se
+law-check-nine.xyz
+laxob.com
+letolove.ru
+lexaprogeneric.link
+lightinghomes.net
+lineavabit.it
+lineshops.biz
+linkbolic.com
+linkbux.ru
+linkdebrideur.xyz
+locimge.pw
+lombia.co
+lonely-mature.com
+lottospring.com
+lovelycraftyhome.com
+low-format.ru
+lunamedia.co
+m1media.net
+m3gadownload.pl
+m4ever.net
+mactechinfo.info
+magento-crew.net
+manifestyourmillion.com
+manualterap.roleforum.ru
+mapquestz.us
+marwer.info
+masserect.com
+masthopehomes.com
+meds-online24.com
+meendo-free-traffic.ga
+meteostate.com
+methodsmarketing.com
+minecraft-rus.org
+mmofreegames.online
+mobioffertrck.com
+mobplayer.ru
+modenamebel.ru
+mojpreskumanie.com
+money-for-placing-articles.com
+moneymaster.ru
+morpicert.pw
+moyaterapiya.ru
+mozello.ru
+mttwtrack.com
+musezone.ru
musicspire.online
-muzaporn.com
-mydirtystuff.com
-mydownloadengine.com
-myiptest.com
-mzdish.site
-nac-bearings.ru
-narkologiya-belgorod.ru
-narkologiya-voronezh.ru
-narutonaruto.ru
-national-today-winning-winner.club
-naturtreenspicerx.pw
-naval.jislaaik.com
-nelc.edu.eg
-net-profits.xyz
-newyorkhotelsmotels.info
-nippon-bearings.ru
-nofreezingmac.work
-nordvpn.com
-nowtorrents.com
-npoet.ru
-oballergiya.ru
-obesidadealgarve.com
-ofanda.com
-officedocuments.net
-offside2.5v.pl
-okeinfo.online
-one.net.in
-onetravelguides.com
-online7777.com
-onlineporno.site
-osagonline.ru
-paphoselectricianandplumber.com
-partnerworkroom.men
+mydeathspace.com
+myftpupload.com
+mygameplus.ru
+myindospace.com
+mypets.by
+naperehresti.info
+narosty.com
+newstraveller.ru
+nfljerseyscheapbiz.us
+nhl17coins.exblog.jp
+niroo.info
+njpalletremoval.com
+nootrino.com
+norththeface.store
+nuit-artisanale.com
+nw-servis.ru
+nyfinance.ml
+o-o-8-o-o.com
+obuv-kupit.ru
+oddamzadarmo.eu
+oil-td.ru
+online-zaymy.ru
+openfrost.com
+ops.picscout.com
+oralsexfilme.net
+ossmalta.com
+ovirus.ru
+pammik.ru
+parajumpersstore.online
pathwhelp.org
-pattersonsweb.com
-pdns.download
-pensplan4u.com
-picturesmania.com
-pigrafix.at
-piratecams.com
-pirateday.ru
-pl.aasoldes.fr
-platinumdeals.gr
-podshipniki-ntn.ru
-pogosh.com
-pogruztehnik.ru
-polska-poezja.com
-pony-business.com
-poptool.net
-popugauka.ru
-popunder.ru
+pcgroup.com.uy
+pepperstyle.ru
+photostudiolightings.com
+pictures-and-images.com
+pijoto.net
+pixgood.com
+plusnetwork.com
+pochemychka.net
+pogodnyyeavarii.gq
+pokemon-go-play.online
+polcin.de
+pomoc-drogowa.cba.pl
+poquoson.org
+porno-home365.com
porno.simple-image.com.ua
+pornodojd.ru
+pornophoto.xyz
+pornoplen.com
+pornosok.ru
pornozhara.com
-potolokelekor.ru
-pravoholding.ru
-priscilarodrigues.com.br
-privatov-zapisi.ru
-productarium.com
-produkto.net
-promoheads.com
-puteshestvennik.com
-rabotaetvse.ru
-rankexperience.com
-ranksonic.info
-razamicroelectronics.com
-razyboard.com
-real-time-analytics.com
-registrydomainservices.com
-rentalcarnavi.info
-replica-watch.ru
-reward-survey.net
-rff-cfal.info
-rfserial.net
+porto.abuilder.net
+portside.cc
+predominant-invent.tk
+privhosting.com
+pro-tec.kz
+profit-opportunity.com
+proprostatit.com
+prostitutki-almata.org
+prostitutki-tolyatti.org
+qualitymarketzone.com
+quickloanbank.com
+rankia.com
+redirectingat.com
+relax.ru
+remote-dba.de
+resultshub-a.akamaihd.net
+retailwith.com
ribieiendom.no
-ric.info
-robertefuller.com
-rockprogblog.com
-rostov.xrus.org
-royalads.net
-rrutw.com
-rusoft-zone.ru
-russkoe-zdorovie.ru
-sack.net
-sankt-peterburg.nodup.ru
-sanyuprojects.com
-saugatuck.com
-scanner-andrew.top
-scanner-don.top
-scrapy.org
-sdsjweb.com
-security60-e.com
-seeingmeerkat.com
-semprofile.com
-semrush.com
-seocdvig.ru
-seolab.top
-seorankinglinks.xyz
-servethis.com
-service.adtech.us
-serving.adbetclickin.pink
-sex-watch.com
-sexkrasivo.net
-sexphoto.site
-sexstream.pl
-shell-pmr.ru
-shikiso.info
+rimedia.org
+ring4rhino.com
+rniaeba.ga
+rospromtest.ru
+royal-investments.net
+rp9.ru
+rusenvironmental.net
+s-forum.biz
+sarf3omlat.com
+satoristudio.net
+scanner-alexa.top
+scanner-nelson.top
+semalt.com
+seokicks.de
+seorussian.ru
+serpstat.com
+service-core.ru
+sex-sex-sex5.com
+sexad.net
+sexcamamateurchat.com
+sexgalleries.top
+sexkvartal.com
+shama-rc.net
+share-buttons-for-free.com
+shariki-zuma-lines.ru
+sharpchallenge.com
+shillyourcoins.com
+ship-marvel.co.ua
+shoesonlinebuy.xyz
+shoppingmiracles.co.uk
+sierraapps.com
signx.info
-sildenafilcitratemed.com
-sitebeam.net
-skuteczna-dieta.co.pl
-slowmac.tech
-small-game.com
-snomer1.ru
-snow.nvr163.com
-sobecjvuwa.com.ru
-social-search.me
-socialtrade.biz
-sodexo.com
-soheavyblog.com
-sohoindia.net
-sonnikforme.ru
-spammen.de
-sptslmtrafms.com
-spywarenuker.com
-spywarespy.com
-ssconstruction.co
-stackthatbucks.com
-stard.shop
+simplesite.com
+site3.free-share-buttons.com
+social-button.xyz
+soda.media
+soft1.ru
+softxaker.ru
+soserfis.com
+spravka-medosmotr.ru
+sps-shop.com
+star61.de
stathat.com
-stats-public.grammarly.io
-staynplay.net
-stop-gepatit.te.ua
-stpicks.com
-strangeduckfilms.com
-streha-metalko.si
-stretchmate.net
+steamoff.net
+stickers-market.ru
+stoki.ru
+studiofmp.com
+sumo.com
superstarfloraluk.com
-syvertsen-da.no
-tacbibirfa.tk
-taihouse.ru
-taqplayer.info
-tasteidea.com
-tattooreligion.ru
-td-33.ru
-td-l-market.ru
-tengohydar.tk
+support.nopeas.sk
+svetlotorg.ru
+svolze.com
+swinginwithme.ru
+tecnoteakviareggio.it
+telemetryverification.net
+template-kid.com
+texbaza.by
tgtclick.com
-theautoprofit.ml
-thedownloadfromwarez.blogspot.com
-tiandeural.ru
-timecrimea.ru
-timhost.ru
-tizanidine4mgprice.blogspot.com
-tizanidine4mgstreetvalue.blogspot.com
-tmm-kurs.ru
-tobeyouday.win
-torrents-tracker.com
-torture.ml
-trafficshaper.com
-traffmonster.info
-trumpetedextremes.com
-tubeoffline.com
-tubo360.com
-tuckpointingmasonrysystems.com
-turkeyreport.tk
-tvteleport.ru
-txxx.com
-uac.net.au
-uptime-as.net
-userequip.com
-usiad.net
-viagraneggrx.com
-video-chat.cn
-video-chat.love
-videochat.cafe
-videochat.mx
-videochat.tv.br
-videogamesecrets.com
-videos-for-your-business.com
-vip-parfumeria.ru
-vipsexfinders.com
-visionwell.com.cn
-vkontarkte.com
-voditeltrezviy.ru
-vseigry.fun
-waterefficiency.co
-watsonrealtycorp.com
-weburok.com
-westermarkanjou.se
-wheelchairliftsarea.com
-whiteelephantwellington.com
-wholesalenfljerseys.us.com
-wholinkstome.com
-wildcattube.com
+thepokertimer.com
+thesmartsearch.net
+titan-ads.life
+tizanidine4mgtablets.blogspot.com
+tizanidine6mg.blogspot.com
+tizanidineformigraines.blogspot.com
+tizanidinehcl2mgtablet.blogspot.com
+tizanidinemusclerelaxant.blogspot.com
+tizanidinestreetprice.blogspot.com
+tnctrx.com
+token-lab.org
+toondinsey.com
+topbestgames.com
+topkarkas.com
+torrentgamer.net
+torrents.cd
+torrents.life
+torrnada.ru
+tour-line.net
+tracklead.net
+trahvid.com
+tramadolandtizanidine.blogspot.com
+twitlinks.com
+uchetunet.su
+ukkelberg.no
+undergroundcityphoto.com
+uptimebot.net
+us-america.ru
+utiblog.fr
+uzporno.mobi
+vchulkah.net
+vegan-foods.us
+viddyoze.com
+video-camer.com
+video-hollywood.ru
+videochat.bz
+vinsit.ru
+vkak.ru
+vostoktrade.info
+vpnhowto.info
+vseuznaem.com
+w-journal.ru
+waycash.net
+web.cvut.cz
+webinstantservice.com
+website-speed-up.top
+welck.octopis.com
+welcomeauto.ru
+while.cheapwebsitehoster.com
win-spy.com
+winner7777.net
wleuaprpxuvr.ga
-word-vorlagen.xyz
-woweb.com.ua
-wttavern.com
-ww2awards.info
-www.solartek.ru
-x-stars.ru
-xbaboon.com
-xchangetrak.com
-xmnb.net
-xn----8sblgmbj1a1bk8l.xn----161-4vemb6cjl7anbaea3afninj.xn--p1ai
+wordkeyhelper.com
+works.if.ua
+worldhistory.biz
+wovis.site
+wufak.com
+wyniki-lista.pl
+xkaz.org
+xlolitka.com
+xn-----7kcabaipgeakzcss7bjdqdwpfnhv.xn--p1ai
xn----btbdvdh4aafrfciljm6k.xn--p1ai
-xn----itbeirbjbi7bc6bh2d.xn--p1ai
-xn--80aaajbdbddwj2alwjieei2afr3v.xn--p1ai
-xn--80aebbcbcdemfkhba4byaehoejh8dza3v.xn--p1ai
+xn--1-8sbcpb0bdm8k6a.xn--p1ai
xn--d1aifoe0a9a.top
-xn--oogle-wmc.com
-xrus.org
-xxlargepop.com
-yellowstonevisitortours.com
-youtubedownload.org
-youtuhe.com
-ypmuseum.ru
+xn--h1ahbi.com.ua
+xpresscare.ru
+xxxhdvideo.site
+xzlive.com
+yginekologa.com
+yoluxuryevents.com
+yoursite.com
yunque.pluto.ro
-zarabotat-na-sajte.ru
-zatjmuzu.info
-zeroredirect1.com
-zeroredirect10.com
-zeroredirect6.com
-zhuravlev.info
-zs2vm.top
+zebradudka.com
+zed21.net
+zeikopay.com
+zeroredirect12.com
+zigarettenonlinekaufen.tumblr.com
+zolotoy-lis.ru
+zona-aqua.ru
+zoodrawings.com
+zoolubimets.ru
+zootoplist.com
zscalerone.net
-zvuker.net
diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list
index eb4a5a1dd..bea2f8c67 100644
--- a/dev-tools/test_units/random-referrers-for-whitelist-test.list
+++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list
@@ -1,250 +1,250 @@
-14b.info
-178evakuator178.ru
-28n2gl3wfyb0.ru
+007angels.com
+101flag.ru
+101lesbian.xyz
+123cha.com
+1qingdao.com
+24-ak.ru
+4kpics.rocks
4kplayer.pl
-55wmz.ru
-6hopping.com
-7a2rnuey1tw9ar.ru
-883zy.com
-8kisses.eu
-aanapa.ru
-adloads.net
-adtiger.tk
-adventureparkcostarica.com
-akita.kz
-alf-img.com
-alfa9.com
-all-number.com
-allstatesugarbowl.org
-aludecor.info
-alyeskaresort.com
-animal-drawings.com
-apishops.ru
-arendas.net
-asiavirtualsolutions.com
-asrvrep-a.akamaihd.net
-autonew.biz
-avtovolop.ru
-azadnegar.com
-bagcionderlab.com
-baksman.com
-bear.gotcher.us
-bestmarriages.com
-beta.hotkeys.com
-bezsporno.ru
-binomo.com
-blog.remote-computer.de
-bloggers.nl
-bocoarchives.org
-bonky.biz
-bonvillan.com
-brothers-smaller.ru
-bscodecs.com
-buqyxa.rincian.info
-butstrap.space
-buyfriend.ru
-calcularpagerank.com
-callawaygolfoutlet.online
-cartierreplica.win
-cartierreplica.xyz
-chatroulette.si
-chatroulette.world
-cheapestjerseys-wholesale.com
-chidporn.com
-chocoslim.pro
-coderstate.com
-com-secure.download
-cubs-jerseys.us
-czeshop.info
-d-black.bz
-darkbooks.org
-day-news.info
+7ooo.ru
+8lufu.com
+albuteroli.com
+alekseevec.ru
+alert-hgd.xyz
+alive-ua.com
+apps-analytics.net
+bala.getenjoyment.net
+balkanfarma.org
+bannerspace.com
+basedecor.ru
+bcmp.org
+beachdriveblog.com
+bestcurs.org
+bharatdefencekavach.com
+biketank.ga
+bioscorp.ru
+biznesluxe.ru
+blockety.co
+bluesalt.co
+bracketsmackdown.com
+callawaygolfstore.online
+capacitacionyaprendizaje.com
+cashkitten-a.akamaihd.net
+cashonads.com
+cbozhe.com
+cercacamion.it
+cfacarrosserie74.com
+chcu.net
+com-onlinesupport.host
+compiko.info
+concealthyself.com
+converse.ddsoldes.fr
+copblock.org
+cupday.com
+customchocolate.business-for-home.com
+daydream-studio.ru
+de.zapmeta.com
dealitright.click
-delitime.info
-e-avon.ru
-elmifarhangi.com
-excitacao.com
-exonline.info
-extlinks.com
-failingmarriege.blogspot.com
-fatmaelgarny.com
-filesmonster.porn
-finemanteam.com
-finteks.ru
-funnymama.com
-gambarxkata.co
-gay-site.store
-gaygalls.net
-generousdeal-a.akamaihd.net
+demian.kz
+domini.cat
+downloadmefiranaratb1972.xpg.com.br
+drillsaw.ru
+duawitchrarato.tk
+dwomlink.info
+dyt.net
+e-buyeasy.com
+ec-file.info
+edudocs.net
+elmacho.xyz
+energydiet24.ru
+enginebay.ru
+ereko.ru
+exdocsfiles.com
+eywords-monitoring-your-success.com
+fearcrow.com
+filesvine.com
+flashbannernow.com
+fodelsedagspresenter.nu
+forpostlock.ru
+foto-sisek.porngalleries.top
+freefoto.ca
+frequiry.com
+freshmac.space
+fugarif.ga
+gallerylisting.com
+gamedayassist.com
+ge0ip.com
+ge0ip.org
gentamicineyedrops.blogspot.com
-gerhardhealer.com
-go2mike.ru
-gpms.org.my
-gsasearchenginerankerdiscount.com
-gtopstats.com
-hdpixion.com
+germetiki.com.ua
+gettpromos.com
+getyourimage.club
+glcomputers.ru
+go2album.com
+goforexvps.com
+goodnightjournal.com
+gsasearchengineranker.top
+hdpixent.com
+hdwalls.xyz
+headpress.ru
+healgastro.com
healmytrauma.info
-histats.com
-hitsbox.info
-hotkeys.com
-hotsocialz.com
-housekuba.org
-hoverboard360.se
-ilmen.net
-imagerydatabase.com
-increasewwwtraffic.info
-infospot.pt
-intellego.info
-istizanidinelikexanax.blogspot.com
-iwantedmoney.com
-iwantmyfreecash.com
-janerikholst.se
+hit-kino.ru
+hmmm.cz
+holidaypics.org
+hollywoodactress.info
+homeinns.com
+hoste.octopis.com
+hostingclub.lk
+hotcore.info
+hotshoppymac.com
+hoverboard360.nl
+humanelydrew.com
+i-midias.net.br
+id-forex.com
+ilmexico.com
+imitex-plus.ru
+import-sales.com
+incep.imagine-ex.co
+iradiology.ru
+iyasimasennka.com
+jeremyeaton.co
+jewelryandfiligree.com
+jwcialislrt.com
karaganda.xkaz.org
-kazan.zrus.org
-kino2018.club
-kinohit1.ru
-krasivye-devushki.net
-kretpicf.pw
-kskjco.club
-kunstaktien.info
-la-fa.ru
-landinez.co
-lenvred.org
-lflash.ru
-limtu.ifmo.ru
-lisque.batcave.net
-livefixer.com
-lombardfinder.ru
-lonerangergames.com
-longadventure.com
-louboutinshoes.xyz
-magnetpress.sk
-mebel-vstroika.ru
-meble-bogart.info
-meduza-consult.ru
-megavolt.net.ua
-members.ghanaweb.com
-misandesign.se
-mobifunapp.weebly.com
-morefastermac.trade
-morepoweronmac.trade
-morpicert.pw
-moscow-region.ru
-mrpornogratis.xxx
-mysexpics.ru
-naj-filmy24.pl
-nardulan.com
-next-dentists.tk
-niki-mlt.ru
-nlfjjunb5.ru
-obnal.org
-offtime.ru
-oklogistic.ru
-onefilms.net
-onlinefilmz.net
-onlinemeetingnow.com
-openlibrary.org
-openmultipleurl.com
-ordernorxx.com
-ourville.info
-outclicks.net
-p-business.ru
-panamaforbeginners.com
-pandroid.co
-parfusale.se
-partner-trustworthy.men
-pateaswing.com
-pdamods.ru
-pdns.cz
-pendelprognos.se
-picscout.com
-picsforkeywordsuggestion.com
-popads.net
-popup.matchmaker.com
-positive2b.ru
-praisong.net
-preventheadacheguide.info
-proftests.net
-prohoster.info
-pronekut.com
-pshare.biz
-qexyfu.bugs3.com
-qpypcx.com
-quelle.ru
-qwesa.ru
-rankexperience.com
-rebelmouse.com
-registry-clean-up.net
-registrydomainservices.com
-remont-mobile-phones.ru
-rique.host.sk
-roleforum.ru
-royalads.net
-rusoft-zone.ru
-sanjosestartups.com
-sbetodiodnye-lampy.ru
-scanner-olga.top
-searchlock.com
-seo-2-0.com
-seo-tools-optimizing.com
-seocheckupx.com
-serving.adbetclickin.pink
-shakhtar-doneck.ru
-ship-marvel.co.ua
-simpoed.ufop.br
-sinel.info
-sisiynas.ru
-siteexpress.co.il
-sklad-24.ru
-skuteczna-dieta.co.pl
-slayerlife.com
-small-game.com
-songplanet.ru
-spidtest.space
-sport7777.net
-spylog.com
-staff.prairiesouth.ca
-star61.de
-stromerrealty.com
-studentguide.ru
-studiofaca.com
-surcentro.com
-surflinksmedical.com
-surgut.zrus.org
-swinger-mobil.net
-syvertsen-da.no
-taqplayer.info
-tehngr.ru
-tgtclick.com
+kidd.reunionwatch.com
+kino-filmi.com
+kino-fun.ru
+kino2018.cc
+kinofak.net
+krassh.ru
+law-check-nine.xyz
+legionalpha.com
+lernur.net
+libertybilisim.com
+luchshie-topcasino.ru
+magiadinamibia.blogspot.com
+matrixalchemy.com
+maxxtor.eu
+mediaoffers.click
+megakino.net
+minyetki.ru
+mixed-wrestling.ru
+mobioffertrck.com
+mojpreskumanie.com
+mokrayakiska.com
+moncleroutletonline.pw
+mosrif.ru
+musflashtv.com
+musicdaddy.net
+muznachas-service.ru
+myra.top
+nastroyke.net
+news-speaker.com
+nhl17coins.exblog.jp
+o-dachnik.ru
+oecnhs.info
+okout.ru
+onlineporno.site
+partners-ship.pro
+partybunny.ru
+parvezmia.xyz
+pcimforum.com
+pflexads.com
+pills24h.com
+pinwallpaper.top
+pony-business.com
+portadd.men
+pr-ten.de
+purchasepillsnorx.com
+pussyfleet.com
+putitin.me
+rebrand.ly
+reklama-i-rabota.ru
+remote-dba.de
+rent2spb.ru
+rewardit.com
+robot-forex.biz
+rockingclicks.com
+rogervivierforsale.com
+rrutw.com
+rubanners.com
+rusexy.xyz
+sabizonline.com
+saneyes.com
+scanner-alex.top
+scmor.ilxc.cc
+scopich.com
+se.bnt-team.com
+sexyteens.hol.es
+shinikiev.com.ua
+signoredom.com
+simple-image.com.ua
+siteripz.net
+skachat-besplatno-obrazcy.ru
+slonechka.ru
+slowmacfaster.trade
+soblaznu.net
+sodexo.com
+soft-program.com
+sortthemesitesby.com
+spasswelt.net
+speeddream.xyz
+spl63.fr
+stair.registrydomainservices.com
+stanthonyscatholicchurch.org
+stop-gepatit.te.ua
+strangeduckfilms.com
+studiofmp.com
+studiokamyk.com.pl
+sugarlyflex.pw
+symphonyintegratedhealthcare.com
+tandvardshuset.net
+tech4master.com
+teencastingporn.com
+telefonsex-ohne0900.net
+the-trader.net
thetoiletpaper.com
-tizanidinehclsideeffects.blogspot.com
-toursmaps.com
-traffmonster.info
-twinderbella.com
-umaseh.com
-urlopener.blogspot.com.au
-vadimkravtcov.ru
-veerotech.com
-viddyoze.com
-video-woman.com
-videochat.cafe
-videojam.tv
-viteonlusarezzo.it
-wallpaperaccess.com
-watsonrealtycorp.com
-webcamtalk.net
-webmasterhome.cn
-webpromotion.ae
-westum.se
-wormix-cheats.ru
-writingservices17.blogspot.ru
-x-diesel.biz
-x-diesel.info
-x-mix.info
-x69ty.ru
-xgftnlrt.bloger.index.hr
-xn--lifehacer-1rb.com
-youdesigner.kz
-yuweng.info
-zaobao.com.sg
-zetmaster.ru
+threecolumnblogger.com
+tjkckpytpnje.com
+tmearegion26.com
+toolsky.com
+topanasex.com
+torrents.cd
+trafficshaper.com
+trafficstars.com
+tuberkuleznik.ru
+tubo360.com
+unece.org
+univerfiles.com
+unmaroll.ya.ru
+uptime-delta.net
+url-extractor.xyz
+usadacha.net
+vacuumscleaner.com
+vapomnoncri.tk
+viagroid.ru
+watchmyfb.pl
+westen-z.life
+wholesalecheapjerseysfree.com
+wholesalejerseysgaa.com
+williamrobsonproperty.com
+wowcasinoonline.ooo
+x5market.ru
+xn----7sbaphztdjeboffeiof6c.xn--p1ai
+xn--hxazdsfy.blogspot.com
+xn--oogle-wmc.com
+xnxx-n.com
+xnxx699.com
+xsion.net
+xxxguitars.com
+yougetsignal.com
+zakonobosago.ru
+zapiszto.pl
+zebramart.ru
+zhuravlev.info
+zigzog.ru
+zionstar.net
+zolotoy-lis.ru
+zoomovies.org
+zzlgxh.com
diff --git a/dev-tools/test_units/ratelimittest-beta.txt b/dev-tools/test_units/ratelimittest-beta.txt
index 3656720f2..f0474a154 100644
--- a/dev-tools/test_units/ratelimittest-beta.txt
+++ b/dev-tools/test_units/ratelimittest-beta.txt
@@ -581,14 +581,4 @@
Welcome to the NginX Ultimate Bad Bot Blocker
This is merely used for testing !!!
-