diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md
index 8be8377c0..cf0e7571c 100755
--- a/AUTO-CONFIGURATION.md
+++ b/AUTO-CONFIGURATION.md
@@ -4,9 +4,9 @@
### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation:
_______________
-#### Version: V4.2021.08.2555
+#### Version: V4.2021.08.2556
#### Bad Referrers Blocked: [7079](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [623](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Bad User-Agents (Bots) Blocked: [624](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
#### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
____________________
diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md
index 1dbcf4d05..d2e274550 100755
--- a/MANUAL-CONFIGURATION.md
+++ b/MANUAL-CONFIGURATION.md
@@ -4,9 +4,9 @@
### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation:
_______________
-#### Version: V4.2021.08.2555
+#### Version: V4.2021.08.2556
#### Bad Referrers Blocked: [7079](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [623](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Bad User-Agents (Bots) Blocked: [624](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
#### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
____________________
diff --git a/README.md b/README.md
index e1ca7af6a..3655e968f 100755
--- a/README.md
+++ b/README.md
@@ -11,9 +11,9 @@
##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots)
_______________
-#### Version: V4.2021.08.2555
+#### Version: V4.2021.08.2556
#### Bad Referrers Blocked: [7079](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [623](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Bad User-Agents (Bots) Blocked: [624](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
#### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
____________________
diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list
index a4c14eee7..ba924e862 100755
--- a/_generator_lists/bad-user-agents.list
+++ b/_generator_lists/bad-user-agents.list
@@ -1,4 +1,3 @@
-dataforseo.com
01h4x.com
360Spider
404checker
@@ -569,6 +568,7 @@ com.plumanalytics
crawl.sogou.com
crawler.feedback
crawler4j
+dataforseo.com
demandbase-bot
domainsproject.org
eCatch
diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md
index ec2d5054d..237954880 100644
--- a/_google_analytics_ghost_spam/README.md
+++ b/_google_analytics_ghost_spam/README.md
@@ -3,9 +3,9 @@
# EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM
_______________
-#### Version: V4.2021.08.2555
+#### Version: V4.2021.08.2556
#### Bad Referrers Blocked: [7079](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [623](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Bad User-Agents (Bots) Blocked: [624](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
#### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
____________________
diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf
index 78f84ba59..2f518b8c9 100755
--- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf
+++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V4.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf
index cf9080665..6170d7d9a 100644
--- a/conf.d/globalblacklist-testing-version.conf
+++ b/conf.d/globalblacklist-testing-version.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V3.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf
index 78f84ba59..2f518b8c9 100644
--- a/conf.d/globalblacklist.conf
+++ b/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V4.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt
index f5c58df12..e8ac9da3a 100644
--- a/dev-tools/_robots_input/robots-input.txt
+++ b/dev-tools/_robots_input/robots-input.txt
@@ -568,6 +568,7 @@ com.plumanalytics
crawl.sogou.com
crawler.feedback
crawler4j
+dataforseo.com
demandbase-bot
domainsproject.org
eCatch
diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf
index cf9080665..6170d7d9a 100644
--- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf
+++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V3.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 5caad0012..c3c56f8fb 100644
--- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 1;
+10.1.0.170 1;
127.0.0.1 1;
diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index 19f6f3310..29d4125ca 100644
--- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 0;
+10.1.0.170 0;
127.0.0.1 0;
diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index cf9080665..6170d7d9a 100644
--- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V3.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 5caad0012..c3c56f8fb 100644
--- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 1;
+10.1.0.170 1;
127.0.0.1 1;
diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index 19f6f3310..29d4125ca 100644
--- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 0;
+10.1.0.170 0;
127.0.0.1 0;
diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
index cf9080665..6170d7d9a 100644
--- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V3.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
index 5caad0012..c3c56f8fb 100644
--- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 1;
+10.1.0.170 1;
127.0.0.1 1;
diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf
index e6add592d..f6413bea8 100644
--- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf
+++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf
@@ -629,6 +629,7 @@
"~*(?:\b)crawl.sogou.com(?:\b|)" 0;
"~*(?:\b)crawler.feedback(?:\b|)" 0;
"~*(?:\b)crawler4j(?:\b|)" 0;
+ "~*(?:\b)dataforseo.com(?:\b|)" 0;
"~*(?:\b)demandbase-bot(?:\b|)" 0;
"~*(?:\b)domainsproject.org(?:\b|)" 0;
"~*(?:\b)eCatch(?:\b|)" 0;
diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
index 19f6f3310..29d4125ca 100644
--- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 0;
+10.1.0.170 0;
127.0.0.1 0;
diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
index cf9080665..6170d7d9a 100644
--- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V3.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber
index 5475b1683..d765d59d6 100644
--- a/dev-tools/buildnumber
+++ b/dev-tools/buildnumber
@@ -1 +1 @@
-2555
+2556
diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template
index cf9080665..6170d7d9a 100644
--- a/dev-tools/globalblacklist-testing.template
+++ b/dev-tools/globalblacklist-testing.template
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V3.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template
index 78f84ba59..2f518b8c9 100755
--- a/dev-tools/globalblacklist.template
+++ b/dev-tools/globalblacklist.template
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V4.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
index 84df96d52..fe9b40a62 100644
--- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
+++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
@@ -45,4 +45,4 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-20.106.167.166 0;
+52.156.71.125 0;
diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf
index 78f84ba59..2f518b8c9 100644
--- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf
+++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V4.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 5caad0012..c3c56f8fb 100644
--- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 1;
+10.1.0.170 1;
127.0.0.1 1;
diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index 19f6f3310..29d4125ca 100644
--- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 0;
+10.1.0.170 0;
127.0.0.1 0;
diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index 78f84ba59..2f518b8c9 100644
--- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V4.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 5caad0012..c3c56f8fb 100644
--- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 1;
+10.1.0.170 1;
127.0.0.1 1;
diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index 19f6f3310..29d4125ca 100644
--- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 0;
+10.1.0.170 0;
127.0.0.1 0;
diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
index 78f84ba59..2f518b8c9 100644
--- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V4.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
index 5caad0012..c3c56f8fb 100644
--- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 1;
+10.1.0.170 1;
127.0.0.1 1;
diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf
index e6add592d..f6413bea8 100644
--- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf
+++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf
@@ -629,6 +629,7 @@
"~*(?:\b)crawl.sogou.com(?:\b|)" 0;
"~*(?:\b)crawler.feedback(?:\b|)" 0;
"~*(?:\b)crawler4j(?:\b|)" 0;
+ "~*(?:\b)dataforseo.com(?:\b|)" 0;
"~*(?:\b)demandbase-bot(?:\b|)" 0;
"~*(?:\b)domainsproject.org(?:\b|)" 0;
"~*(?:\b)eCatch(?:\b|)" 0;
diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
index 19f6f3310..29d4125ca 100644
--- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 0;
+10.1.0.170 0;
127.0.0.1 0;
diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
index 78f84ba59..2f518b8c9 100644
--- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V4.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
index 84df96d52..fe9b40a62 100644
--- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
+++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
@@ -45,4 +45,4 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-20.106.167.166 0;
+52.156.71.125 0;
diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf
index 6edbcd855..e949bed83 100644
--- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf
+++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V4.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
index 84df96d52..fe9b40a62 100644
--- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
+++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
@@ -45,4 +45,4 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-20.106.167.166 0;
+52.156.71.125 0;
diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf
index d930d8184..316a8e027 100644
--- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf
+++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf
@@ -4,10 +4,10 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2021.08.2555
-### Updated: Sat Aug 28 22:01:29 UTC 2021
+### Version: V4.2021.08.2556
+### Updated: Sun Aug 29 07:43:25 UTC 2021
### Bad Referrer Count: 7079
-### Bad Bot Count: 623
+### Bad Bot Count: 624
###################################################
### VERSION INFORMATION ##
@@ -717,6 +717,7 @@ map $http_user_agent $bad_bot {
"~*(?:\b)crawl.sogou.com(?:\b)" 3;
"~*(?:\b)crawler.feedback(?:\b)" 3;
"~*(?:\b)crawler4j(?:\b)" 3;
+ "~*(?:\b)dataforseo.com(?:\b)" 3;
"~*(?:\b)demandbase-bot(?:\b)" 3;
"~*(?:\b)domainsproject.org(?:\b)" 3;
"~*(?:\b)eCatch(?:\b)" 3;
diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf
index 5caad0012..c3c56f8fb 100644
--- a/dev-tools/test_units/blacklist-ips.conf
+++ b/dev-tools/test_units/blacklist-ips.conf
@@ -1,2 +1,2 @@
-10.1.0.129 1;
+10.1.0.170 1;
127.0.0.1 1;
diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf
index e6add592d..f6413bea8 100644
--- a/dev-tools/test_units/blacklist-user-agents.conf
+++ b/dev-tools/test_units/blacklist-user-agents.conf
@@ -629,6 +629,7 @@
"~*(?:\b)crawl.sogou.com(?:\b|)" 0;
"~*(?:\b)crawler.feedback(?:\b|)" 0;
"~*(?:\b)crawler4j(?:\b|)" 0;
+ "~*(?:\b)dataforseo.com(?:\b|)" 0;
"~*(?:\b)demandbase-bot(?:\b|)" 0;
"~*(?:\b)domainsproject.org(?:\b|)" 0;
"~*(?:\b)eCatch(?:\b|)" 0;
diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list
index 9b0b58582..802960046 100644
--- a/dev-tools/test_units/random-bots-for-test-quick.list
+++ b/dev-tools/test_units/random-bots-for-test-quick.list
@@ -1,10 +1,10 @@
-Aboundexbot
-Brandprotect
-Ezooms
-InfoNaviRobot
-MSFrontPage
-MSIECrawler
-Semrush
-Zermelo
-check1.exe
-instabid
+Abonti
+BackStreet
+Brandwatch
+Ebingbong
+Getintent
+IlseBot
+LNSpiderguy
+Lmspider
+Psbot
+Vacuum
diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list
index c2e998336..ec39694a5 100644
--- a/dev-tools/test_units/random-bots-for-test.list
+++ b/dev-tools/test_units/random-bots-for-test.list
@@ -1,250 +1,250 @@
01h4x.com
-360Spider
+404enemy
+80legs
AIBOT
ALittle Client
+ASPSeek
Abonti
Aboundex
-AllSubmitter
+AhrefsBot
+AiHitBot
+Aipbot
+Alexibot
+AlphaBot
Anarchie
Anarchy99
+Anthill
+Apexoo
+Asterias
Attach
-BBBike
-BDFetch
-BackDoorBot
+AwarioSmartBot
+BDCbot
BackWeb
-BacklinkCrawler
-Badass
-Barkrowler
-Battleztar Bazinga
+Bandit
+BetaBot
+Bigfoot
+Bitacle
+Black Hole
BlackWidow
Blackboard
+Blow
BlowFish
-BotALot
-BuiltWith
-Bullseye
+Brandprotect
+Buck
+Buddy
+BuzzSumo
CATExplorador
+CCBot
CSHttp
+CazoodleBot
CensysInspect
-CheeseBot
-Claritybot
+Cliqzbot
+Cloud mapping
+Cogentbot
CopyRightCheck
-Cosmos
+Copyscape
+Craftbot
+Crawling at Home Project
Crescent
-Custo
-DBLBot
+Curious
+CyotekWebCopy
+DTS Agent
+DataCha0s
DatabaseDriverMysqli
-Demon
+Deusu
Devil
+Digincore
Disco
Discobot
+Dispatch
DnyzBot
-DomCopBot
+DomainAppender
DomainStatsBot
Domains Project
-EasyDL
+Dotbot
+Download Wonder
+Dragonfly
+Drip
+ECCP/1.0
+EMail Wolf
Ecxi
+Evil
Exabot
-ExtractorPro
-Extreme Picture Finder
+Express WebPictures
+ExtLinksBot
EyeNetIE
+FDM
FHscan
FemtosearchBot
Fimap
+Firefox/7.0
Flunky
-FrontPage
-Fuzz
+Freeuploader
FyberSpider
-GT::WWW
-GetRight
-GetWeb
+Fyrebot
+GermCrawler
Getintent
+Gigabot
+Go!Zilla
Go-Ahead-Got-It
GoZilla
-Gotit
Grabber
GrapeFX
-HEADMasterSEO
-HTMLparser
+GrapeshotCrawler
+GridBot
HTTP::Lite
HTTrack
-Havij
-Hloader
-HybridBot
-IDBot
-IlseBot
-InfoNaviRobot
-InterGET
+Harvest
+Humanlinks
+Indy Library
+Intelliseek
Internet Ninja
InternetSeer
-IstellaBot
JamesBOT
JennyBot
JetCar
-Jetty
Joomla
-Jorgee
-JustView
Kenjin Spider
-Keyword Density
-LNSpiderguy
-LWP::Simple
Lanshanbot
-Larbin
-LeechFTP
LeechGet
-Libwhisker
-LinkWalker
+LexiBot
+LibWeb
+Likse
Linkdexbot
-LinkextractorPro
-LinkpadBot
LinksManager
+LinqiaMetadataDownloaderBot
LinqiaRSSBot
+Lipperhey
+Lipperhey Spider
Lmspider
-MQQBrowser
-MSFrontPage
-MTRobot
+Ltx71
+MJ12bot
Mag-Net
Magnet
-Mail.RU_Bot
+MarkWatch
+Mass Downloader
+Mb2345Browser
MeanPath Bot
-MegaIndex.ru
-MicroMessenger
-Microsoft URL Control
-Moblie Safari
+Meanpathbot
Mojolicious
-Mozlila
Msrabot
Musobot
+NICErsPRO
+NPbot
Needle
Nessus
Net Vampire
-NetAnts
-NetMechanic
-Nikto
Nimbostratus
-Offline Explorer
-OnCrawl
+Offline Navigator
+OpenVAS
Openvas
+OrangeBot
OutclicksBot
-PECL::HTTP
-POE-Component-Client-HTTP
-PageAnalyzer
-PageScorer
+OutfoxBot
+PHPCrawl
+Page Analyzer
PageThing.com
-Pandalytics
-PeoplePal
-Petalbot
-PictureFinder
-ProWebWalker
+Panscient
+Piepmatz
+Pixray
Probethenet
-Pu_iN
+QueryN Metasearch
+Quick-Crawler
RankActiveLinkBot
-RankingBot
-RankingBot2
+RankFlex
+ReGet
RealDownload
-RepoMonkey
-Ripper
+RebelMouse
+Recorder
SBIder
SEOstats
-SISTRIX
-SalesIntelligent
-ScanAlert
-ScoutJet
+SMTBot
Screaming
-ScreenerBot
-ScrepyBot
Searchestate
-SearchmetricsBot
-SentiBot
-SeoSiteCheckup
-Siphon
+Shodan
+Site Sucker
SiteCheckerBotCrawler
SiteExplorer
-SiteSnagger
+Sitebeam
Siteimprove
-SlySearch
-SmartDownload
-Snake
+Sitevigil
Snapbot
-SocialRankIOBot
Sociscraper
Sogou web spider
-Sosospider
-Sottopop
-SpaceBison
Spammen
-Spinn3r
+Sqlmap
+Sqlworm
+Steeler
+Sucuri
+SuperBot
+SuperHTTP
SurveyBot
-T8Abot
-Teleport
-TeleportPro
+Suzuran
Telesoft
Telesphoreo
Telesphorep
-The Intraformant
TheNomad
-Toweyabot
-True_Robot
+Titan
+Toata
+Tracemyfile
+Trendictionbot
+Turingos
Turnitin
-TurnitinBot
-TwengaBot
Twice
URLy Warning
-Upflow
+UnisterBot
VB Project
-Vagabondo
-VidibleScraper
+VeriCiteCrawler
Virusdie
-VoidEYE
-Voil
Voltron
-WEBDAV
-WPScan
-WWW-Collector-E
+WASALive-Bot
WWW-Mechanize
-Wallpapers
+WWW::Mechanize
Wallpapers/3.0
-WallpapersHD
-WeSEE
+Web Collage
Web Enhancer
-Web Fetch
Web Fuck
+Web Pix
Web Sauger
-WebBandit
-WebGo IS
-WebReaper
+WebCopier
+WebStripper
WebWhacker
Webalta
-WebmasterWorldForumBot
+Webshag
Website Quester
-WinHTTrack
+Webster
+Whack
+Whatweb
+Widow
WiseGuys Robot
-Wotbox
-Wprecon
-Xenu
-Zade
+Wonderbot
+Woobot
+Xaldon WebSpider
+Xaldon_WebSpider
+YoudaoBot
+Zauba
+Zermelo
Zeus
-ZmEu
ZoomBot
-archive.org_bot
autoemailspider
-backlink-check
clark-crawler
-crawl.sogou.com
crawler4j
-demandbase-bot
evc-batch
-facebookscraper
gopher
-meanpathbot
+lwp-request
+lwp-trivial
mediawords
-muhstik-scan
-oBot
+page scorer
pcBrowser
probe-image-size
-scan.lol
+scalaj-http
+seobility
serpstatbot
-siteripz
-sp_auditbot
-webprosbot
-x22Mozilla
+sexsearcher
+sysscan
+trendiction.de
+voyagerx.com
+webmeup-crawler
+webpros.com
+x09Mozilla
+zgrab
diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list
index 563a4ce19..b8e37f814 100644
--- a/dev-tools/test_units/random-bots-for-whitelist-test.list
+++ b/dev-tools/test_units/random-bots-for-whitelist-test.list
@@ -1,250 +1,250 @@
-01h4x.com
+404checker
+404enemy
+80legs
+AIBOT
+ASPSeek
+Abonti
Acunetix
-AllSubmitter
-Anarchy
-Ankit
-Anthill
+Alexibot
+Alligator
Apexoo
Asterias
Attach
-AwarioSmartBot
-BBBike
+BDFetch
BackWeb
Badass
-Bandit
-BatchFTP
+Barkrowler
Battleztar Bazinga
-Bigfoot
-Bitacle
-Black Hole
Blow
-Bolt
+BlowFish
+Boardreader
+Brandprotect
+Brandwatch
Buddy
-Bullseye
-CSHttp
+BuiltBotTough
+BuiltWith
+BunnySlippers
CazoodleBot
CensysInspect
+CheeseBot
ChinaClaw
-Claritybot
+Cliqzbot
+Cloud mapping
Collector
-Craftbot
-Crawling at Home Project
-CrazyWebCrawler
-Curious
+Copier
+Cosmos
Custo
-Deusu
-Devil
-Dirbuster
+DataCha0s
+Digincore
+Disco
Dispatch
+DittoSpyder
DnyzBot
-DomainSigmaCrawler
+DomCopBot
+DomainCrawler
+DomainStatsBot
Domains Project
Dotbot
-Download Wonder
-Dragonfly
+Drip
+ECCP/1.0
EMail Siphon
+EMail Wolf
Ebingbong
-EroCrawler
-Express WebPictures
-Extreme Picture Finder
-EyeNetIE
+Ecxi
+EirGrabber
+Evil
Ezooms
-FDM
-FemtosearchBot
-Fimap
-Flunky
-Freeuploader
-FrontPage
+Fuzz
+Fyrebot
GT::WWW
+GalaxyBot
Genieo
+GermCrawler
GetRight
-GetWeb
-Go!Zilla
-GoZilla
-Gotit
+Gigablast
+Go-Ahead-Got-It
GrapeFX
-GridBot
-HEADMasterSEO
+GrapeshotCrawler
HMView
+HTMLparser
+Haansoft
HaosouSpider
Havij
-Humanlinks
IDBTE4M
IRLbot
-Iblog
Id-search
-IlseBot
Image Fetch
-InfoNaviRobot
+Indy Library
InfoTekies
Internet Ninja
-JamesBOT
-Jetty
-Joomla
+InternetSeer
+Iria
+Iskanie
+JennyBot
+JetCar
+Jorgee
JustView
Keyword Density
Kinza
LNSpiderguy
-Leap
-LexiBot
-Lftp
+Lanshanbot
+LeechGet
LibWeb
-Libwhisker
-Likse
+Lightspeedsystems
+LinkWalker
Linkbot
-LinkpadBot
-LinksManager
-LinqiaMetadataDownloaderBot
+LinkextractorPro
+LinqiaRSSBot
LinqiaScrapeBot
+Lipperhey
+Ltx71
MJ12bot
-MSFrontPage
MSIECrawler
-Mag-Net
-Magnet
-Mail.RU_Bot
Majestic SEO
-Majestic-SEO
MarkMonitor
-MeanPath Bot
-MegaIndex.ru
-Mister PiX
+MarkWatch
+Masscan
+Mata Hari
+Meanpathbot
+Microsoft Data Access
+Microsoft URL Control
+Moblie Safari
Mozlila
Mr.4x3
-NICErsPRO
-Name Intelligence
Nameprotect
+Navroad
NearSite
+Net Vampire
+NetAnts
+NetLyzer
NetZIP
-Netcraft
-Nettrack
-NextGenSearchBot
-Nibbler
+Netvibes
NimbleCrawler
-Nimbostratus
-Ninja
Nmap
-Nuclei
Nutch
+Octopus
OpenLinkProfiler
Openfind
Openvas
-OrangeBot
-OrangeSpider
-PECL::HTTP
-Page Analyzer
-PageScorer
+OutclicksBot
+OutfoxBot
PageThing.com
-Pandalytics
-Panscient
-Picsearch
+Pavuk
+Petalbot
+Pi-Monster
+PictureFinder
+Piepmatz
+Pimonster
PleaseCrawl
Pockey
-ProWebWalker
-Psbot
-Pu_iN
+ProPowerBot
PxBroker
QueryN Metasearch
-Quick-Crawler
+RankActive
+RankActiveLinkBot
+RankFlex
RankingBot
RankingBot2
+RankurBot
Re-re
-RealDownload
-Reaper
-Recorder
+ReGet
RedesScrapy
-RepoMonkey
-Rogerbot
-SEOkicks
+Ripper
+RocketCrawler
+SBIder
+SEOkicks-Robot
+SEOlyticsCrawler
SEOprofiler
-SMTBot
+SEOstats
+ScanAlert
Scrapy
-SearchmetricsBot
+ScreenerBot
+ScrepyBot
Semrush
-SemrushBot
-SeoSiteCheckup
-Site Sucker
-SiteCheckerBotCrawler
-SiteLockSpider
-SiteSnagger
+SeobilityBot
SiteSucker
Sitebeam
-Sitevigil
-SocialRankIOBot
-Sottopop
-SpaceBison
+Snake
+Sosospider
Spammen
-SpankBot
-Spbot
-Steeler
-Sucuri
-SuperBot
-SuperHTTP
-Surfbot
-Szukacz
+Spanner
+SputnikBot
+Sqworm
+Stripper
+Sucker
+Swiftbot
T0PHackTeam
T8Abot
-Telesoft
Telesphoreo
Telesphorep
The Intraformant
TheNomad
-Tracemyfile
+TightTwatBot
+Toata
+Trendiction
Trendictionbot
True_Robot
-Turingos
Turnitin
Typhoeus
-URLy.Warning
-V-BOT
+UnisterBot
+Upflow
+VCI
+VeriCiteCrawler
VidibleScraper
Virusdie
-Voil
Voltron
-WASALive-Bot
-WBSearchBot
+WEBDAV
WPScan
+WWW-Collector-E
+WWWOFFLE
+Wallpapers
Wallpapers/3.0
-WeSEE
+WallpapersHD
+Web Auto
Web Collage
+Web Enhancer
Web Pix
-WebAuto
-WebCopier
-WebEnhancer
-WebLeacher
-WebPix
-WebReaper
-WebSauger
-Webalta
-Webshag
+Web Sauger
+Web Sucker
+WebGo IS
+WebStripper
+WebWhacker
+WebZIP
+WebmasterWorldForumBot
+WebsiteExtractor
WebsiteQuester
-Webster
Whack
-WiseGuys Robot
-Wonderbot
-Woobot
+Whacker
+Who.is Bot
+Wotbox
Wprecon
-Xenu
Zauba
-Zitebot
-ZoomBot
+ZmEu
+ZoominfoBot
+ZyBorg
archive.org_bot
arquivo.pt
autoemailspider
+backlink-check
check1.exe
-crawl.sogou.com
-demandbase-bot
-eCatch
+coccocbot-web
+com.plumanalytics
+evc-batch
gopher
-magpie-crawler
-muhstik-scan
-netEstate NE Crawler
-ripz
-scalaj-http
+instabid
+internetVista monitor
+ips-agent
+lwp-request
+lwp-trivial
+meanpathbot
+pcBrowser
+polaris version
+probe-image-size
seoscanners
+serpstatbot
sitechecker.pro
+sogouspider
spyfu
-sysscan
-trendiction.com
webpros.com
-zauba.io
+x22Mozilla
zgrab
diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list
index 9fa6931b0..32ed851e8 100644
--- a/dev-tools/test_units/random-referrers-for-test-quick.list
+++ b/dev-tools/test_units/random-referrers-for-test-quick.list
@@ -1,10 +1,10 @@
-adimmix.com
-boom-celebs.com
-cheapmoncler.pw
-fermersovet.ru
-haveinc.xyz
-oops-cinema.ru
-paintingplanet.ru
-rusvideos.su
-vote-up.ru
-wedding-salon.net
+4ureyesonly.com
+autogrep.ru
+avon-ukraine.com
+danashop.ru
+gigablast.com
+milblueprint.com
+nac-bearings.ru
+popserve.adscpm.net
+popup-jdh.xyz
+ranksonic.info
diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list
index 2cc2fdc09..6c4258a1e 100644
--- a/dev-tools/test_units/random-referrers-for-test.list
+++ b/dev-tools/test_units/random-referrers-for-test.list
@@ -1,500 +1,500 @@
-0daymusic.org
-1-free-share-buttons.com
-1001desert.com
-2nt.ru
-3xforum.ro
-8lufu.com
-a.frcls.fr
-aa9bd78f328a6a41279d0fad0a88df1901.com
-adultnet.in
-advokateg.ru
-agahinameh.com
+1001watch.com.ua
+100dollars-seo.com
+101flag.ru
+1pamm.ru
+1webmaster.ml
+2ads.co.uk
+2girls1cup-free.com
+2itech.ru
+2rich4bitches.com
+5kstore.com
+7search.com
+98oi.ru
+991mostfm.co.id
+9med.net
+actulite.com
+adserver-e7.com
+adtech.de
+adultmeetonline.info
+advocatemsk.ru
+affiliate-fr.com
+afslankpillen2017nl.eu
agecheckadult.com
-ahrefs.com
-alphahoverboards.com
-alphavisions.net
-am15.net
-amateurgalls.com
-amoi.tn
-apiallgeniusinfo-a.akamaihd.net
-apiappenableinfo-a.akamaihd.net
-appiq.mobi
-apple.com-webbrowsing-security.science
-artel116.ru
-atmovs.com
-autoloans.com
-autotrends.today
-aviav.eu
-back.dog
-baikaleminer.com
-baladur.ru
-bang-hotties.com
-battlecarnival.su
-bavsac.com
-bbsoldes.fr
-bcmp.org
-bedcapdealers.com
-bestdooz.com
-bg6s0.com
+ahhjf.com
+aksonural.ru
+alfa9.com
+all4wap.ru
+allbizne.co.ua
+alldubai.biz
+allesohnegirls.net
+alltheviews.com
+allwidewallpapers.com
+am-se.com
+amanda-porn.ga
+amyfoxfitness.com
+angigreene.com
+animalia-life.club
+anlimebel.kiev.ua
+apptester.tk
+arewater.com
+arvut.org
+asseenontv.ru
+asseenontvonline.ru
+ausmepa.org.au
+aviav.org
+aviav.ru
+avtocredit-legko.ru
+azbuka-mo.ru
+b-styles.xyz
+b2b-lounge.com
+backiee.com
+balkanfarma.org
+bankiem.pl
+bankmib.ru
+bankofthewext.com
+bannerads.de
+bashtime.ru
+bazaronline24.ru
+beatifullandscape.co
+beauty-clinic.ru
+bestimagecoollibrary.com
+besttorrentknifta.weebly.com
+betterhdporn.com
+bget.ru
bidr.trellian.com
-billiard-classic.com.ua
-bimatoprost-careprost.com
-blackplanet.com
-blacktwhite.com
-blog.axant.it
-blogs.rediff.com
-bmw.afora.ru
-boanonihaca.tk
-boazpower.com
-boleznikogi.com
-bombla.org
+bif-ru.info
+big-boards.info
+bitcoin-ua.top
+bitporno.sx
+bizzliving.com
+blogqpot.com
+bmwhighperformers.com
+bodybuilding-shop.biz
+boobsimge.com
bookhome.info
-bosefux.esy.es
+boole.onlinewebshop.net
+bosman.pluto.ro
+brandbucket.com
+browsepulse-a.akamaihd.net
+brus.city
+buckspar.gq
+buigas.00it.com
+bum.com.ru
+bumble.cheapwebsitehoster.com
bumskontakte.org
-businesn.men
-buy-cheap-online.info
+buttons-for-website.com
buy-forum.ru
-buybest1.biz
-buypillsorderonline.com
-buzzsumo.com
-candypeople.se
-canrioloadm.gq
-cardul.ru
-cartiertrinity.xyz
-centrdebut.ru
-cheapestjerseys-wholesale.com
-chocolateslim-u-srbiji.com
-cialisndbrx.com
-ciekawinki.pl
-cinemaflix.website
-cityadspix.com
+buyfriend.ru
+buyhoverboard.com
+buzzurl.jp
+cacheimages.com
+callawaygolfoutlet.online
+candiceloves.us
+capture-room.com
+carina-sy.de
+cars-modification.net
+carsplate.com
+casinorewards.info
+cementaresearch.se
+cenokos.ru
+cete.ru
+chcu.net
+choice-credit.ru
+choosecuisine.com
+christianlouboutinsaleonline.us
+cleanmypc.com
+clicksor.net
+cmd.kz
cognitiveseo.com
-colehaanoutlet.store
+com-onlinesupport.site
+communisave.co.za
compliance-checker-7.info
+compliance-don.top
compliance-don.xyz
-compliance-elena.xyz
-conboy.us
-contentsexpress.com
+compliance-george.xyz
+compliance-jess.top
+compliance-jessica.top
+compliance-julianna.top
+compliance-nelson.top
+comsysnet.com
+concealthyself.com
+constantaservice.net
+copypaste.traderzplanet.in
costablue.xyz
-cottageofgrace.com
-cougfan.info
-covid-schutzmasken.de
-creditnation.ru
-customerguru.in
-cwetochki.ru
-dawlenie.com
-decorazilla.com
-dekoration.us
-delitime.info
-design-ideas.info
-detmebel.su
-detskie-zabavi.ru
-deutschehobbyhuren.net
-digital-video-processing.com
-directivepub.com
-discovertreasure-a.akamaihd.net
-dokumentalkino.net
-domainanalyzing.xyz
-domini.cat
-dominterior.org
-download-of-the-warez.blogspot.com
-download-walpaperhd.blogspot.com
-downloaddy.net
-downloadeer.net
-downvids.net
-drugspowerstore.com
-e-collantes.com
-e-commerce-seo1.com
-e-kwiaciarz.pl
-e705.net
-eandsgallery.com
-easyukraine.com
-edudocs.net
+counter.bloke.com
+crackzplanet.com
+credit.co.ua
+crynet.cc
+cubs-jerseys.us
+customergrowthsystems.com
+cv.wallhade.co
+dandiyabeats.in
+daneshetabiat.com
+daretodonate.co
+deda-moroza-zakaz.ru
+dev33.dioniqlabb.se
+diamond-necklace.info
+diffbot.com
+dipstar.org
+documentbase.net
+dolohen.com
+dominateforex.ml
+dotnetdotcom.org
+doxysexy.com
+drivotracker.com
+drpornogratisx.xxx
+duplicashapp.com
+dustyorate.com
+e-biznes.info
+e-poker-2005.com
+edelstahlschornstein-123.de
+ekaterinburg.xrus.org
ekobata.ru
-erectile.bid
-eropho.com
-europeanwatches.ru
-everypony.ru
-exct.net
-executehosting.com
-extener.org
-f07.de
-face.hostingx.eu
-facialporntube.com
-faithe.top
-falcon-images.blogspot.com
+elektrischezigarette1.onsugar.com
+elektrischezigaretten1.blogse.nl
+eluxer.net
+enginebay.ru
+enthuse.computernetworksonline.com
+eralph.tk
+eugenevaultstorage.com
+euroskat.ru
+exhibitionplus.eu
+expdom.com
+eyes-on-you.ga
+ezigarettekaufen2.mpbloggar.se
+ezigarettekaufen2.yolasite.com
familienzahnaerzte.com
-fashionindeed.ml
-fefo.gdn
-fellowshipoftheminds.com
-fighrofacciufreesig.ga
-fikasound.tk
-firma-legion.ru
-fodelsedagspresenter.nu
-foodcrafts.website
-forum20.smailik.org
-fotoxxxru.com
-fr-bearings.ru
-frbizlist.com
-free-social-buttons.com
-freshberry.com.ua
-fusoradio.info
-fz139.ttk.ru
-gambarxkata.co
-gamebackyard.com
-gameprimary.com
-gamesprite.me
-gay.adultgalls.com
-gaygalls.net
+fastfixing.tech
+favornews.com
+fdzone.org
+fermersovet.ru
+fetlifeblog.com
+fidelityfunding.com
+film-one.ru
+finemanteam.com
+fireads.men
+fiuxy.com
+flashlarevista.com
+flavors.me
+flipper.top
+floridahuntingfishingadventures.com
+formseo.com
+forsex.info
+foto-sisek.porngalleries.top
+freecamdollars.com
+frvo.alptandem.ru
+funnymama.com
+funponsel.com
+g.starmoe.xyz
+gallerily.com
+gallerylisting.com
+gate5.co.za
+gavuer.ru
gaypornmovie.net
-gcup.ru
gdzkurokam.ru
-germetiki.com.ua
-get-your-social-buttons.info
-getmyads24.com
-gigapeta.com
-global-ics.co.za
-go2mike.ru
-gonextmedia.com
-goodhousekeeping.com
-goodnightjournal.com
-goodwriterssales.com
-gopro-online.info
-goroda-vsego-mira.ru
-gox.com.ua
-gpms.org.my
-greatzip.com
-growboxbl.ru
-growmyfunds.ca
-guardlink.com
-hanwei.us
-hatedriveapart.com
+getlamborghini.ga
+gigablast.com
+girls-ufa.ru
+gktt.ru
+goo.ne.jp
+gsbs.com.ua
+gsou.cf
+guardlink.org
+guigyverpo.cf
+gwebtools.com
+handicapvantoday.com
+hanink.biz.ly
hausfrauensex18.com
-hccoder.info
+hdfreeporno.net
hdhc.ru
-heatpower.ru
+headpress.ru
+healgastro.com
helicalpile.us
-helvetia.com.ua
-hermesbracelets.xyz
-hidefiles.org
-home-task.com
-homeandhealth.ru
-homedo.fabpage.com
-host-tracker.com
-hotel-mkad.ru
-howtoclean.club
-https-legalrc.biz
-humanelydrew.com
-ie.57883.net
-iideaidekonkatu.info
-imagez.co
-immobiliaremassaro.com
-impotentik.com
-inmate-locator.us
-intellego.info
-intimshop-fantasy.ru
-invivo.hu
-isotoner.com
-ispaniya-costa-blanca.ru
-istizanidinelikexanax.blogspot.com
-iwanttodeliver.com
-izoll.ru
+himazin.info
+homedecorpicture.us
+hoverboard360.at
+ibb.com.ua
+id-forex.com
+images-free.net
+incep.imagine-ex.co
+infodocsportal.com
+insider.pro
+iphantom.com
+iptool.xyz
+iradiology.ru
jabimgo.pw
-janavibekken.no
-japfm.com
-javidol.site
-juliadiets.com
-justkillingti.me
-kaidalibor.de
+jamiembrown.com
+jav-idol.com
+jaxcube.info
+jerseysbizwholesalecheap.com
kalandranis.gr
-karadene.com
-kazinogames.lv
-kinofak.net
-knowsitall.info
-kollesa.ru
-konpax.com
-kopihijau.info
-kvartiry-remont.ucoz.ru
-lacapilla.info
-lasvegaslockandsafe.com
-lechenie-gemorroya.com
+karusel-market.ru
+katjimej.blog.fc2.com
+kino-fun.ru
+kinobaks.com
+kinomaniatv.pl
+kiskinhouse.com
+komp-pomosch.ru
+kostenlos-sexvideos.com
+kriokomora.info
+kustanay.kz
+lawyersinfo.org
+ledpolice.ru
+legalrc.biz
+lemon-ade.site
+lennatin.info
lenpipet.ru
-lenvred.org
-linkredirect.biz
-lion.snapmap.us
+lexiangwan.com
+lida-ru.com
+liquimondo.com
+liran-locks.com
littleberry.ru
-lockerz.com
-lonerangergames.com
-long-beach-air-conditioning.com
-lookingglassemb.com
-lyrster.com
-m3gadownload.pl
-m5home.ru
-machicon-akihabara.info
-magda-gadalka.ru
-magiadinamibia.blogspot.com
+locimge.pw
+lodki-pvh.dp.ua
+lombia.com
+losangeles-ads.com
+m-google.xyz
+mail.allnews24.in
maju.bluesalt.co
-managerpak204.weebly.com
-marmitaco.cat
-masserect.com
-matb3aa.com
-matematikus.info
matpre.top
-mature.free-websites.com
-mazda-roadsters.com
-mebel-alait.ru
-meteocast.net
-miniads.ca
-minyetki.ru
+microsearch.ru
+microstatic.pl
+mirmedinfo.ru
+missis.top
+misterjtbarbers.com
mitrasound.ru
-mixx.com
-mjchamonix.org
-mlf.hordo.win
-mockupui.com
-mokrayakiska.com
-mompussy.net
-movies-in-theaters.net
+mixed-wrestling.ru
+mmofreegames.online
+mncrftpcs.com
+mobicover.com.ua
+monetizer.com-01.site
+moneyzzz.ru
+montazhnic.ru
+morefastermac.trade
+mosrif.ru
+mrbojikobi4.biz
+mrcsa.com.au
+muabancantho.info
musflashtv.com
-myanyone.net
-mydeathspace.com
-mydirtyhobby.com
-myra.top
-nac-bearings.ru
-nakozhe.com
-nakrutka.cc
-narco24.me
-narkologiya-peterburg.ru
-nastroyke.net
-naughtyconnect.com
-needtosellmyhousefast.com
-neodownload.webcam
-net-profits.xyz
-news-readers.ru
-nhl09.ru
-nhl17coins.exblog.jp
-nhl17coinsforps3.gratisblog.biz
-nicovideo.jp
-nofreezingmac.click
-nufaq.com
-nuker.com
-office2web.com
-okeinfo.online
-okout.ru
-onemantrip.com
-onko-24.com
-online-templatestore.com
-onload.pw
-oralsexfilme.net
-otbelivanie-zubov.com
-ourtherapy.ru
-outpersonals.com
-outrageousdeal-a.akamaihd.net
-paidonlinesites.com
-paparazzistudios.com.au
-pastaleads.com
-pc-services.ru
-pendelprognos.se
+musicktab.com
+muztops.ru
+mxgetcode.com
+my-aladin.com
+my-big-family.com
+mydoctorok.ru
+mymercy.info
+myonlinepayday.co
+mypets.by
+namecrumilchlet.tk
+naphukete.ru
+newfilmsonline.ru
+newsperuse.com
+nfljerseyscheapbiz.us
+nibbler.silktide.com
+norththeface.store
+november-lax.com
+nullrefer.com
+oil-td.ru
+okmedia.sk
+okmusic.jp
+one.net.in
+onlinebay.ru
+onlywoman.org
+ooomeru.ru
+ops.picscout.com
+orgasmus-virtual.com
+ostrovtaxi.ru
+oxotl.com
+pammik.ru
+paperwritingservice17.blogspot.ru
+pdn-4.com
+pekori.to
+pelfind.me
pestomou.info
-pflexads.com
-picswe.com
-picture-group.com
+photokitchendesign.com
+picsearch.com
+pictures-and-images.com
+picurams.pw
pinstake.com
-pinup-casino1.ru
-pirateday.ru
-pisanieprac.info
-pistonclasico.com
-pk-pomosch.ru
-pl-vouchers.com
-plastweb.ru
-playfortuna-play.ru
-pobeiranie.pl
-poem-paying.gq
-pokerniydom.ru
-popads.net
-popander.mobi
-porn-w.org
-porndl.org
-pornoblood.com
-pornoklad.net
-pornokorol.com
-porodasobak.net
-portadd.men
-portside.cc
-pospr.waw.pl
-privatov-zapisi.ru
-prlog.ru
-progress-upakovka.ru
-pronekut.com
+piski.top
+play-mp3.com
+plugingeorgia.com
+polybuild.ru
+popunder.ru
+popup.matchmaker.com
+pornobrazzers.biz
+pornogig.com
+pozvonim.com
+pregnant.guru
+pricheskaonline.ru
+printie.com
+privatbank46.ru
+pro-poly.ru
+professionaldieselcare.com
+prohoster.info
+projectforte.ru
+prom23.ru
+promoforum.ru
+promover.org
+pronorm.fr
propranolol40mg.blogspot.com
-prosperent.com
-pskcijdc.bloger.index.hr
-puzo2arbuza.ru
-quickcashlimited.com
-r-control.ru
-rabotaetvse.ru
-ragecash.com
-real-time-analytics.com
-realitykings.com
-remont-fridge-tv.ru
-rfserial.net
-rialp.getenjoyment.net
-rocketchange.ru
-roleforum.ru
-rucrypt.com
-rus-teh.narod.ru
+prostitutki-tolyatti.org
+puserving.com
+puzzleweb.ru
+quick-seeker.com
+rabot.host.sk
+rainbowice.ru
+ranking2017.ga
+razvratnoe.org
+redirect.trafficreceiver.club
+retailwith.com
+rethinkwasteni.info
+rewardit.com
+ric.info
+rightenergysolutions.com.au
+rollercoin.com
+rubbed.us
ruscoininvest.company
ruspdd.com
-sady-urala.ru
-saitevpatorii.com
-sampleletters.net
-sarahmilne.top
-scanner-alexa.top
-scanner-andrew.top
-scanner-ivan.top
-scanner-jack.top
-scanner-mark.top
-schlampen-treffen.com
-search.xtconnect.com
-secret.xn--oogle-wmc.com
-security60-e.com
-seminarygeorgia59.ga
-seo-2-0.com
-seo-tools-optimizing.com
-seobility.net
-seorankinglinks.us
-seorankinglinks.xyz
+rusprostitute.com
+rvtv.ru
+s-forum.biz
+sa-live.com
+saltspray.ru
+samara.rosfirm.ru
+sammyweaver.com
+scanner-alex.top
+scanner-jessica.top
+search-error.com
+search.1and1.com
+semp.net
+semrush.com
+seorank.info
seotoolsagency.com
-sergiorossistore.online
-sex-dating.co
-sex-sex-sex5.com
-sexspornotub.com
+sex-foto.pw
+sexblog.pw
+sexflirtbook.com
+sexgalleries.top
+sexkontakte-seite.com
+sexyteens.hol.es
sfd-chess.ru
-shikiso.info
-shopfishing.com.ua
-shtaketniki.kz
-signx.info
-sims-sims.ru
-sinel.info
-site5.com
-sitopreferito.it
-sledstvie-veli.net
-small-game.com
-smstraf.ru
+shijian.ac.cn
+sisi-go.ru
+sispe.com.br
+skypasss.com
+slavkokacunko.de
+smailik.org
+smart-balancewheel.com
snworks.com
-socialseet.ru
-softxaker.ru
-sohoindia.net
+snyatie-lomki-v-stacionare.ru
+social-s-ggg.xyz
+social-s-iii.xyz
+socialmonkee.com
+socialtrade.biz
+sofit-dmd.ru
+softonicads.com
+softtor.com
space-worry.ml
-spb.ru
-spravka130.ru
-sprttrack.com
-sr-rekneskap.no
-srecorder.com
-ssconstruction.co
-stariy-baku.com
-statistici.ro
-studworks.org
-surffoundation.nl
-svetlotorg.ru
-tabakur77.com
-takethatad.com
-tattomedia.com
-tecnoteakviareggio.it
-teenbbw.yopoint.in
-telefonsexkostenlos.tk
-thaismartloan.com
-the-torrent-tracker.blogspot.com
-thepornsex.org
-tizanidine4mg.blogspot.com
-tizanidine4mgstreetprice.blogspot.com
-tizanidine6mg.blogspot.com
-tizanidinefibromyalgia.blogspot.com
-tizanidineingredients.blogspot.com
+spin2016.cf
+spitfiremusic.com
+stathat.com
+store-rx.com
+stpolice.com
+stroiminsk.com
+stroyhelp-dv.ru
+sugarlyflex.pw
+suhanpacktech.com
+sukirgenk.dvrlists.com
+sundrugstore.com
+susanholtphotography.com
+svarbit.com
+sweepstakes.rewardit.com
+tabletkinaodchudzanie.com.pl
+tampabaywatch.org
+taqplayer.info
+techart24.com
+telefonsexi.com
+thecoral.com.br
+thelottosecrets.com
+ticketsys.inetwd.com
+timecrimea.ru
+tizanidinehcl4mgtab.blogspot.com
+tizanidinemusclerelaxant.blogspot.com
tizanidinerecreationaluse.blogspot.com
-tizanidinevssoma.blogspot.com
-tmm-kurs.ru
-tnctrx.com
-torrent-newgames.com
-toursmaps.com
-tovaroboom.vast.ru
-tracksurf.daooda.com
-trafficmania.com
-trichizobswiv.agddns.net
+tizanidinesleepaid.blogspot.com
+tizanidinestreetvalue.blogspot.com
+tmtrck.com
+toon-families.com
+top-deal.com.pl
+topanasex.com
+topbestgames.com
+topsiteminecraft.com
+torrent-to-magnet.com
+torrentdownloadhub.com
+torture.ml
+tracksz.co
truck-addzilla.life
-trustaffs.com
-turist-strani.ru
-tytoona.com
-u555u.info
-ucoz.ru
-uggsale.online
-uhod-za-sobakoj.ru
-umekana.ru
-universals.com.ua
-url2image.com
-uzporno.mobi
-vabasa.inwtrade.com
-velen.io
-vesnatehno.com
-videochat.bz
-videochat.ph
-videonsk.com
-vielporno.net
-vip-file.com
-vip-parfumeria.ru
-vip2ch.com
+tupper-shop.ru
+turbodsp.com
+uchetunet.su
+uggbootsoutletsale.us
+ugguk.online
+uhdtv.website
+uptimebot.net
+urlopener.com
+uroki.net
+urzedowski.eu
+usiad.net
+validdomain.xyz
+valueclick.com
+vashsvet.com
+vegascosmetics.ru
+vekzdorov.ru
+velpanex.ru
+veselokloun.ru
+viagra-soft.ru
+videochat.cafe
+visionwell.com.cn
+visitcambridge.org
+vita.com.hr
vitanail.ru
vksaver-all.ru
vodkoved.ru
-voluumtracker1.com
-vriel.batcave.net
-vrotike.ru
-vseigru.one
-vzlomfb.com
-walkme.com
-webads.co.nz
-webnode.me
-webshoppermac.com
-webstatsdomain.org
-wedding-salon.net
-weddingdresses.xyz
-westermarkanjou.se
-westum.se
-whatsupinfoley.com
-wholesalenfljerseys.us.com
-wildcattube.com
-win-spy.com
-wingsofrefuge.net
-winner7777.net
-winx-play.ru
-woman-h.ru
-wordkeyhelper.com
-worldis.me
-wrona.it
+voronezh.xrus.org
+vucms.com
+wait3sec.org
+wallpaperaccess.com
+wallpapersdesk.info
+wallpapersist.com
+watchmygf.net
+webextract.profound.net
+weboptimizes.com
+welcomeauto.ru
+wesharepics.site
+whiteproduct.com
+wholinkstome.com
+wingsoffury2.com
+woman-orgasm.ru
+wonderfulflowers.biz
+wttavern.com
ww1943.ru
-wyniki-lista.pl
-x-diesel.biz
-xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai
-xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai
-xn----7sbaphztdjeboffeiof6c.xn--p1ai
-xn----8sblgmbj1a1bk8l.xn----161-4vemb6cjl7anbaea3afninj.xn--p1ai
-xn----ctbbcjd3dbsehgi.xn--p1ai
-xn----itbkqkfiq.xn--p1ai
-xxxtubesafari.com
-xzlive.com
-yaminecraft.ru
-yatrk.xyz
-yellowproxy.net
-youbrainboost.asia
-youdesigner.kz
-yoursearch.me
-yoursite.com
+ww2awards.info
+www.solartek.ru
+www.timer4web.com
+xfire.com
+xtube.com
+yebocasino.co.za
+yogamatsexpert.com
+youtube-downloader.savetubevideo.com
yuarra.pluto.ro
-yunque.pluto.ro
-zaimite.ru
-zebramart.ru
-zero1.it
-zigarettenonlinekaufen.tumblr.com
-zmoda.hostreo.com
-zojirushi-products.ru
-zvuker.net
-zx6.ru
+yurgorod.ru
+zaapplesales.blogspot.com
+zakon-ob-obrazovanii.ru
+zarabiaj-dzis.pl
+zawyna.ua
+zdesoboi.com
+zigarettenonl.canalblog.com
+zigzog.ru
+zirondelli.it
+zlatnajesen.com
+znakomstvaonlain.ru
diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list
index b56f2f3f3..12a50790a 100644
--- a/dev-tools/test_units/random-referrers-for-whitelist-test.list
+++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list
@@ -1,250 +1,250 @@
-00it.com
-10bet.com
-6128786.com
-7fon.ru
-abbanreddy.com
-absugars.com
-actices.com
-actulite.com
-ad-words.ru
-addblueoff.com.ua
-adman.se
-allornamenti.com
-animaltoplist.com
-ap.senai.br
-apptester.tk
-auto-zapchasti.org
-autolombard-krasnodar.ru
-avon-severozapad.ru
-becuo.com
-bestcurs.org
-beyan.host.sk
-bidr.trellian.com
-biocn.dx.am
-biographiya.com
-bjetjt.com
-blackwitchcraft.ru
-blockety.co
-blogos.kz
-blogqpot.com
-bobba.dzaba.com
-braindaily.xyz
-burger-tycoon.com
-bus-offer.com
-buytizanidineonlinenoprescription.blogspot.com
-buyviagraa.blogspot.com
-camdenmemorials.com
-cementaresearch.se
-christianlouboutinsaleonline.us
-citizenclsdriveri7.pen.io
-claytransformations.info
-clothesforcash.com
-com-onlinesupport.host
-coolgramgoods.com
-crackzplanet.com
-crazyprotocol.com
-cwetochki.ru
-damedingel.ya.ru
-db.speedup.gdn
-deanmoore.ie
-deda-moroza-zakaz.ru
-deutschland123.de
-docs4all.com
-docsportal.net
-domaincrawler.com
-drunkenstepfather.com
-dtnlyss.com
-econom.co
-elektroniksigaraankara.info
-eluxer.net
-emediate.eu
-ero-advertising.com
-escortslet.net
-everytalk.tv
-ezaz.info
-ezigarettekaufen.myblog.de
-ezigarettekaufen2.blox.pl
-ezigaretteshop2.mywapblog.com
-f07.de
-f0815.de
-falopicm.pw
-fast-wordpress-start.com
-fastcrawl.com
-femmesdenudees.com
-fighrofacciufreesig.ga
-franch.info
-free-laptop-reward.com
-free-share-buttons.top
-free-video-tool.com
+0912701309f8ce.com
+108shot.com
+3rbseyes.com
+4kplayer.pl
+4rent.ru
+7ooo.ru
+a.pr-cy.ru
+aa14ab57a3339c4064bd9ae6fad7495b5f.com
+adelly.bg
+adf.ly
+advancedcleaner.com
+aihelen.net
+aktobe.xkaz.org
+alibestsale.com
+allknow.info
+allkrim.com
+allsilver925.co.il
+amazon-seo-service.com
+animenime.ru
+apccargo.com
+apiholdingmypage-a.akamaihd.net
+apiluckyleapnet-a.akamaihd.net
+appiq.mobi
+arate.ru
+artel116.ru
+aruplighting.com
+asociatia-tipografilor-transilvania.ro
+asophoto.com
+astrochicks.com
+aversis.net
+balkanfarma.org
+basisches-wasser.net
+battleforupdating.site
+beachtoday.ru
+bestempresas.es
+bg6s0.com
+big-cash.net
+bioinnovate.co
+bitcoins-live.ru
+bitnote.co
+bizlist.com.de
+board.f00d.de
+bonzybuddy.com
+boole.onlinewebshop.net
+californianews.cf
+candlehandmade.com
+canzoni.ru
+carloans.com
+cejewelry.xyz
+cfjump.com
+chocolateslim-original.com
+cialgenisrx.com
+cl.s7.exct.net
+clarithromycin500mg.com
+clubfashionista.com
+coaching-netz.info
+comissionka.net
+companies-catalog.com
+compliance-elena.top
+compliance-fred.top
+compliance-jessica.top
+compliance-julianna.top
+comsysnet.com
+creditnation.ru
+cupday.com
+cv.wallhade.co
+daneshetabiat.com
+darmebel.com.ua
+detoxmed24.ru
+digest-project.ru
+digitalfaq.com
+div.as
+driving.kiev.ua
+drunkmoms.net
+easycommerce.cf
+ed-shop01.ru
+electro-prom.com
+emailaccountlogin.co
+enbersoft.com
+enthuse.computernetworksonline.com
+erotische-geschichten-xxl.com
+eyemagination.com
+ezigarettenkaufen1.deviantart.com
+faithe.top
+favornews.com
+fetroshok.ru
+findacheaplawyers.com
+findsexguide.com
+firesub.pl
freenom.link
-frequiry.com
-front.ru
-g7m.pl
-gambarkatabaru.com
-geile-lelly.eu
-genericlowlatencyasiodriverhq.aircus.com
-golden-praga.ru
-gombita.info
-goodly.pro
-greatgrace.ru
-gstatey.net
-handsandlegs.ru
+frivgame250.com
+fun-mobi.pl
+funcrushgames.com
+gaypornmovie.net
+get-your-social-buttons.info
+gfaq.ru
+girlgamerdaily.com
+globalscam.ga
+gloverid.site
+goldandcard.ru
+greenidesign.co
+guod.me
healmytrauma.info
-hitcpm.com
-hoholikik.club
-hoztorg-opt.ru
-htmlcorner.com
-hyhj.info
-i-midias.net.br
-ictizanidinehcl4mg.blogspot.com
-ideibiznesa2015.ru
-imagefinder.site
+healthcarestore.info
+hunthillfarmtrust.org
+hyiphunter.org
+i-service.kz
+iklysha.ml
+imagerydatabase.com
investyb.com
-javxxx18.com
+iopeninghours.co.uk
+istanbulit.com
+itbc.kiev.ua
+iusstf.org
+ivanovo.zrus.org
+ivearchenceinflu.cf
+iwanttodeliver.com
+jimmychoosale.online
johnrobertsoninc.com
-kambasoft.com
-karachev-city.ru
-kazan.zrus.org
-kellyonline.xyz
-kino-key.info
-kiwi237au.tk
-kliksaya.com
-kraljeva-sutjeska.com
-kunstaktien.info
-laapp.com
-law-check-seven.xyz
-lescinq.com
+journeydownthescale.info
+jpcycles.com
+karpun-iris.ru
+kasino-money.pw
+kenaba.su
+keywordblocks.com
+kinohall.ru
+kotaku.com
+krasivoe-hd.com
+krasula.pp.ua
+kretpicf.pw
+landmania.ru
+legalrc.biz
letmacworkfaster.site
-linkpulse.com
-liver-chintai.org
-loadingpages.me
-lostaruban.com
-macdamaged.tech
-machicon-ueno.info
-manipulyator-peterburg.ru
-marcogrup.com
-med-zdorovie.com.ua
-metaxalonevstizanidine.blogspot.com
-microsoftportal.net
-middlerush-a.akamaihd.net
-mielec.pl
-mojowhois.com
+letsrepair.in
+littlesunraiser.com
+loadopia.com
+luciddiagnostics.in
+mackeeper-center.club
+magazintiande.ru
+magicplayer-s.acestream.net
+maglid.ru
+massage-info.nl
+masthopehomes.com
+mcar.in.ua
+medtherapy.ru
+meet-flirt-dating.com
+megaindex.ru
+meget.co.za
+mensandals.xyz
+mesto-x.com
+mini.7zap.com
+mockupui.com
+modabutik.ru
+moetomnenie.com
morocco-nomad-excursions.com
-mosdverka.ru
-my-bc.ru
-myblogregistercm.tk
-mypets.by
-nalogovyy-kodeks.ru
-natali-forex.com
-nationalbreakdown.com
-neks.info
-nero-us.com
-netoil.no
-next-dentists.tk
-nonameread45.live
-nuker.com
-o-o-6-o-o.ru
-office-windows.ru
-olgacvetmet.com
-orion-v.com
-outpersonals.com
-outrageousdeal-a.akamaihd.net
-ownshop.win
-paparazzistudios.com.au
-paradontozanet.ru
-paydayloanslocal.com
-photo.houseofgaga.ru
-picture-group.com
-pinapchik.com
-pk-pomosch.ru
-pornmania.pl
-porno-chaman.info
-pornodojd.ru
-pornokajf.com
-pornosemki.info
-positive2b.ru
-pregnant.guru
-privhosting.com
-profitwithalex.info
-pukaporn.com
-qwarckoine.com
-ragecash.com
-rapevideosmovies.com
-restaurantlescampi.com
-rethinkwasteni.info
-rialp.getenjoyment.net
-ringtonepartner.com
-risparmiocasa.bz.it
-sa-live.com
-sabaapress.com
-salutmontreal.com
-sanjosestartups.com
-sbetodiodnye-lampy.ru
-scanner-donald.top
-scanner-viktor.top
-sdi-pme.com
-semxiu.com
-seocheki.net
-seoexperimenty.ru
-serdcenebolit.com
-servicecenter.co.ua
-sibtest.ru
-simpoed.ufop.br
-sirpornogratis.xxx
-slavia.info
-social-fun.ru
-social-s-hhh.xyz
-solicita.info
-spabali.org
-sport7777.net
-spy-sts.com
+most-kerch.org
+musicspire.online
+mvpicton.co.uk
+mydownloadengine.com
+myiptest.com
+nac-bearings.ru
+narosty.com
+niki-mlt.ru
+nisuturnetdgu.tk
+nlfjjunb5.ru
+no-rx.info
+oconto.ru
+oneclickfiles.com
+oops-cinema.ru
+picturesify.com
+plastjulgranar.se
+playfortuna-play.ru
+pliks.pl
+porno.simple-image.com.ua
+pornoinn.com
+postclass.com
+pozdrawleniya.ru
+presleycollectibles.com
+privetsochi.ru
+prod2016.com
+puserving.com
+pussysaga.com
+rapidgator-porn.ga
+russintv.fr
+sady-urala.ru
+saitevpatorii.com
+sayyoethe.blogspot.co.za
+secretscook.ru
+seminarygeorgia59.ga
+senger.atspace.co.uk
+seogadget.ru
+serialsway.ucoz.ru
+servethis.com
+sexs-foto.top
+sexy-screen-savers.com
+sharebutton.to
+shikiso.info
+shoesonlinebuy.xyz
+sierraapps.com
+simplesite.com
+slavkokacunko.de
+smallseotools.com
+social-s-iii.xyz
+speeddream.xyz
+stop-gepatit.te.ua
subj.ukr-lit.com
-success-seo.com
-swiped.su
-tacbelarus.ru
-tarad.com
-template-kid.com
-terrafootwear.us
-thefds.net
-theplacetoupdating.pw
-tizanidineforopiatewithdrawal.blogspot.com
-tizanidinesideeffectsweightloss.blogspot.com
-tn811.us
-tnctrx.com
-tongkatmadura.info
-track112.site
+surfbuyermac.com
+telefonsex-ohne0900.net
+telegraf.by
+titangel-vietnam.com
+tizanidineformigraines.blogspot.com
+tizanidineinteractions.blogspot.com
+topappspro.com
+tour-line.net
+tracfone.com
trafaret74.ru
-trk-4.net
-trkur.com
-tuberkulezanet.ru
-tupper-shop.ru
-tvand.ru
-tversvet.ru
-twu.com.ua
-uhdtv.website
-vacances-voyages.info
-vegascosmetics.ru
-via-energy-cumpara.com
-videochat.tv.br
-vikistars.com
-vladimir.zrus.org
-voprosotvet24.ru
-vote-up.ru
-vpnhowto.info
-wallpapersdesk.info
-wildcattube.com
-williamrobsonproperty.com
-ww1943.ru
-x-diesel.biz
-xn--80ahvj9e.xn--p1ai
-xn--b1addnj3cah.xn--p1ai
-xnxxandxvideos.com
-yatrk.xyz
-ynymnwbm.bloger.index.hr
-youtuhe.com
-zazagames.org
-zhacker.net
-zigarettenonlinekaufen1.bloog.pl
-zone-kev717.info
-zoogdisany.com
-zs2vm.top
+traffmonster.info
+tvorozhnaja-zapekanka-recept.ru
+udsgame.online
+ukrtvir.com.ua
+umityangin.net
+unitygame3d.com
+unrealcommander.com
+valoresito.com
+versaut.xxx-cam.webcam
+verymes.xyz
+videochat.mx
+vip-dom.in
+vip2ch.com
+virus-schutzmasken.de
+vitalads.net
+vsesubwaysurfers.com
+website-speed-up.site
+westen-z.life
+whereverdesperate.gq
+womama.ru
+worldmusicfests.com
+wstroika.ru
+x-true.info
+xn-----8kcatubaocd1bneepefojs1h2e.xn--p1ai
+xn--24-glceagatoq7c2a6ioc.xn--p1ai
+xxxtube69.com
+yaaknaa.info
+yottos.com
+yourdesires.ru
+z-master.ru
+zaim-pod-zalog-krasnodar.ru
+zaloro.com
+zarabiaj-dzis.pl
+zastenchivosti.net
diff --git a/dev-tools/test_units/ratelimittest-master.txt b/dev-tools/test_units/ratelimittest-master.txt
index f0474a154..3656720f2 100644
--- a/dev-tools/test_units/ratelimittest-master.txt
+++ b/dev-tools/test_units/ratelimittest-master.txt
@@ -581,4 +581,14 @@
Welcome to the NginX Ultimate Bad Bot Blocker
This is merely used for testing !!!
+