diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
index 70960edbd..00808be93 100644
--- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1730
-### Updated: Tue Jul 2 11:39:49 SAST 2019
+### Version: V3.2019.07.1732
+### Updated: Tue Jul 2 12:24:32 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 966492772..4dddc08a6 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.184.96.71 1;
+35.193.14.140 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index 2f692b069..40cb4d7a8 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.184.96.71 0;
+35.193.14.140 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index 70960edbd..00808be93 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1730
-### Updated: Tue Jul 2 11:39:49 SAST 2019
+### Version: V3.2019.07.1732
+### Updated: Tue Jul 2 12:24:32 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 966492772..4dddc08a6 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.184.96.71 1;
+35.193.14.140 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index 2f692b069..40cb4d7a8 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.184.96.71 0;
+35.193.14.140 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
index 70960edbd..00808be93 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1730
-### Updated: Tue Jul 2 11:39:49 SAST 2019
+### Version: V3.2019.07.1732
+### Updated: Tue Jul 2 12:24:32 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
index 966492772..4dddc08a6 100644
--- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.184.96.71 1;
+35.193.14.140 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
index 2f692b069..40cb4d7a8 100644
--- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.184.96.71 0;
+35.193.14.140 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
index 70960edbd..00808be93 100644
--- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1730
-### Updated: Tue Jul 2 11:39:49 SAST 2019
+### Version: V3.2019.07.1732
+### Updated: Tue Jul 2 12:24:32 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/distribution_tests_missing_includes/arch-setup.log b/.dev-tools/distribution_tests_missing_includes/arch-setup.log
index 676369cce..dfb321100 100644
--- a/.dev-tools/distribution_tests_missing_includes/arch-setup.log
+++ b/.dev-tools/distribution_tests_missing_includes/arch-setup.log
@@ -8,7 +8,7 @@ Checking url: https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-ba
inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/conf.d/globalblacklist.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/nginx.conf
inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/conf.d/botblocker-nginx-settings.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/nginx.conf
-Whitelisting ip: 35.202.145.110 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/bots.d/whitelist-ips.conf
+Whitelisting ip: 35.193.14.140 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/bots.d/whitelist-ips.conf
Checking for missing includes:
diff --git a/.dev-tools/distribution_tests_missing_includes/centos-setup.log b/.dev-tools/distribution_tests_missing_includes/centos-setup.log
index 933c5507c..0b799a05e 100644
--- a/.dev-tools/distribution_tests_missing_includes/centos-setup.log
+++ b/.dev-tools/distribution_tests_missing_includes/centos-setup.log
@@ -8,7 +8,7 @@ Checking url: https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-ba
inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/centos7/etc/nginx/conf.d/globalblacklist.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/centos7/etc/nginx/nginx.conf
inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/centos7/etc/nginx/conf.d/botblocker-nginx-settings.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/centos7/etc/nginx/nginx.conf
-Whitelisting ip: 35.184.96.71 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/centos7/etc/nginx/bots.d/whitelist-ips.conf
+Whitelisting ip: 34.66.178.120 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/centos7/etc/nginx/bots.d/whitelist-ips.conf
Checking for missing includes:
diff --git a/.dev-tools/distribution_tests_missing_includes/centos7/etc/nginx/nginx.conf b/.dev-tools/distribution_tests_missing_includes/centos7/etc/nginx/nginx.conf
index d30e66573..a66af0310 100644
--- a/.dev-tools/distribution_tests_missing_includes/centos7/etc/nginx/nginx.conf
+++ b/.dev-tools/distribution_tests_missing_includes/centos7/etc/nginx/nginx.conf
@@ -8,6 +8,13 @@ error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
# Load dynamic modules. See /usr/share/nginx/README.dynamic.
+
+
+# Nginx Bad Bot Blocker Includes
+# REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/centos7/etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/centos7/etc/nginx/conf.d/globalblacklist.conf;
+
include /usr/share/nginx/modules/*.conf;
events {
@@ -74,13 +81,6 @@ http {
# # Load configuration files for the default server block.
# include /etc/nginx/default.d/*.conf;
#
-
-
-# Nginx Bad Bot Blocker Includes
-# REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/centos7/etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/centos7/etc/nginx/conf.d/globalblacklist.conf;
-
# location / {
# }
#
diff --git a/.dev-tools/distribution_tests_missing_includes/fedora-setup.log b/.dev-tools/distribution_tests_missing_includes/fedora-setup.log
index e510e586f..c24d7c193 100644
--- a/.dev-tools/distribution_tests_missing_includes/fedora-setup.log
+++ b/.dev-tools/distribution_tests_missing_includes/fedora-setup.log
@@ -8,7 +8,7 @@ Checking url: https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-ba
inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/fedora/etc/nginx/conf.d/globalblacklist.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/fedora/etc/nginx/nginx.conf
inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/fedora/etc/nginx/conf.d/botblocker-nginx-settings.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/fedora/etc/nginx/nginx.conf
-Whitelisting ip: 35.224.112.202 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/fedora/etc/nginx/bots.d/whitelist-ips.conf
+Whitelisting ip: 34.66.178.120 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/fedora/etc/nginx/bots.d/whitelist-ips.conf
Checking for missing includes:
diff --git a/.dev-tools/distribution_tests_missing_includes/fedora/etc/nginx/nginx.conf b/.dev-tools/distribution_tests_missing_includes/fedora/etc/nginx/nginx.conf
index a4d8560e7..0113a93db 100644
--- a/.dev-tools/distribution_tests_missing_includes/fedora/etc/nginx/nginx.conf
+++ b/.dev-tools/distribution_tests_missing_includes/fedora/etc/nginx/nginx.conf
@@ -8,6 +8,13 @@ error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
+
+
+# Nginx Bad Bot Blocker Includes
+# REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/fedora/etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/fedora/etc/nginx/conf.d/globalblacklist.conf;
+
include /usr/share/nginx/modules/*.conf;
events {
@@ -74,13 +81,6 @@ http {
# # Load configuration files for the default server block.
# include /etc/nginx/default.d/*.conf;
#
-
-
-# Nginx Bad Bot Blocker Includes
-# REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/fedora/etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/fedora/etc/nginx/conf.d/globalblacklist.conf;
-
# location / {
# }
#
diff --git a/.dev-tools/distribution_tests_missing_includes/opensuse-leap/etc/nginx/nginx.conf b/.dev-tools/distribution_tests_missing_includes/opensuse-leap/etc/nginx/nginx.conf
index 07871b12b..7a7bbaa37 100644
--- a/.dev-tools/distribution_tests_missing_includes/opensuse-leap/etc/nginx/nginx.conf
+++ b/.dev-tools/distribution_tests_missing_includes/opensuse-leap/etc/nginx/nginx.conf
@@ -27,6 +27,13 @@ events {
http {
+
+
+# Nginx Bad Bot Blocker Includes
+# REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/opensuse-lea/etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/opensuse-lea/etc/nginx/conf.d/globalblacklist.conf;
+
include mime.types;
default_type application/octet-stream;
@@ -133,10 +140,4 @@ http {
include vhosts.d/*.conf;
-
-
-# Nginx Bad Bot Blocker Includes
-# REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/opensuse-lea/etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/opensuse-lea/etc/nginx/conf.d/globalblacklist.conf;
}
diff --git a/.dev-tools/distribution_tests_missing_includes/opensuse-setup.log b/.dev-tools/distribution_tests_missing_includes/opensuse-setup.log
index df4d67110..1a0618a26 100644
--- a/.dev-tools/distribution_tests_missing_includes/opensuse-setup.log
+++ b/.dev-tools/distribution_tests_missing_includes/opensuse-setup.log
@@ -8,7 +8,7 @@ Checking url: https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-ba
inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/opensuse-lea/etc/nginx/conf.d/globalblacklist.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/opensuse-leap/etc/nginx/nginx.conf
inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/opensuse-lea/etc/nginx/conf.d/botblocker-nginx-settings.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/opensuse-leap/etc/nginx/nginx.conf
-Whitelisting ip: 35.202.145.110 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/opensuse-leap/etc/nginx/bots.d/whitelist-ips.conf
+Whitelisting ip: 34.68.144.114 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/opensuse-leap/etc/nginx/bots.d/whitelist-ips.conf
Checking for missing includes:
diff --git a/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/nginx.conf b/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/nginx.conf
index ad448ea3e..71e9ad8ec 100644
--- a/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/nginx.conf
+++ b/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/nginx.conf
@@ -23,6 +23,13 @@ http {
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
+
+
+# Nginx Bad Bot Blocker Includes
+# REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/conf.d/globalblacklist.conf;
+
include /etc/nginx/mime.types;
default_type application/octet-stream;
@@ -59,13 +66,6 @@ http {
##
include /etc/nginx/sites-enabled/*;
-
-
-
-# Nginx Bad Bot Blocker Includes
-# REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/conf.d/globalblacklist.conf;
}
diff --git a/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/nginx.conf b/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/nginx.conf
index db45fd1b2..e6d69c6e7 100644
--- a/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/nginx.conf
+++ b/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/nginx.conf
@@ -24,6 +24,13 @@ http {
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
+
+
+# Nginx Bad Bot Blocker Includes
+# REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/conf.d/globalblacklist.conf;
+
include /etc/nginx/mime.types;
default_type application/octet-stream;
@@ -59,13 +66,6 @@ http {
##
include /etc/nginx/sites-enabled/*;
-
-
-
-# Nginx Bad Bot Blocker Includes
-# REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/conf.d/globalblacklist.conf;
}
diff --git a/.dev-tools/distribution_tests_missing_includes/ubuntu1604-setup.log b/.dev-tools/distribution_tests_missing_includes/ubuntu1604-setup.log
index 2321e0e2e..85cc41cd2 100644
--- a/.dev-tools/distribution_tests_missing_includes/ubuntu1604-setup.log
+++ b/.dev-tools/distribution_tests_missing_includes/ubuntu1604-setup.log
@@ -8,7 +8,7 @@ Checking url: https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-ba
inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/conf.d/globalblacklist.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/nginx.conf
inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/conf.d/botblocker-nginx-settings.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/nginx.conf
-Whitelisting ip: 35.202.145.110 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/bots.d/whitelist-ips.conf
+Whitelisting ip: 35.193.14.140 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/bots.d/whitelist-ips.conf
Checking for missing includes:
diff --git a/.dev-tools/distribution_tests_missing_includes/ubuntu1804-setup.log b/.dev-tools/distribution_tests_missing_includes/ubuntu1804-setup.log
index 365e04475..478f8e52f 100644
--- a/.dev-tools/distribution_tests_missing_includes/ubuntu1804-setup.log
+++ b/.dev-tools/distribution_tests_missing_includes/ubuntu1804-setup.log
@@ -8,7 +8,7 @@ Checking url: https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-ba
inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/conf.d/globalblacklist.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/nginx.conf
inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/conf.d/botblocker-nginx-settings.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/nginx.conf
-Whitelisting ip: 35.192.85.2 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/bots.d/whitelist-ips.conf
+Whitelisting ip: 34.66.178.120 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/bots.d/whitelist-ips.conf
Checking for missing includes:
diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template
index 70960edbd..00808be93 100644
--- a/.dev-tools/globalblacklist-testing.template
+++ b/.dev-tools/globalblacklist-testing.template
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1730
-### Updated: Tue Jul 2 11:39:49 SAST 2019
+### Version: V3.2019.07.1732
+### Updated: Tue Jul 2 12:24:32 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template
index 9ff18437e..6d439042e 100755
--- a/.dev-tools/globalblacklist.template
+++ b/.dev-tools/globalblacklist.template
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1730
-### Updated: Tue Jul 2 11:39:48 SAST 2019
+### Version: V4.2019.07.1732
+### Updated: Tue Jul 2 12:24:31 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
index 798770665..6073098c1 100644
--- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
@@ -45,4 +45,4 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.188.1.99 0;
+34.66.178.120 0;
diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
index 9ff18437e..6d439042e 100644
--- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1730
-### Updated: Tue Jul 2 11:39:48 SAST 2019
+### Version: V4.2019.07.1732
+### Updated: Tue Jul 2 12:24:31 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 90fe4f9d3..eabb2e30b 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-104.154.120.187 1;
+104.198.131.58 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index bc389ee32..6bad13743 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-104.154.120.187 0;
+104.198.131.58 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index 9ff18437e..6d439042e 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1730
-### Updated: Tue Jul 2 11:39:48 SAST 2019
+### Version: V4.2019.07.1732
+### Updated: Tue Jul 2 12:24:31 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 90fe4f9d3..eabb2e30b 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-104.154.120.187 1;
+104.198.131.58 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index bc389ee32..6bad13743 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-104.154.120.187 0;
+104.198.131.58 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
index 9ff18437e..6d439042e 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1730
-### Updated: Tue Jul 2 11:39:48 SAST 2019
+### Version: V4.2019.07.1732
+### Updated: Tue Jul 2 12:24:31 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
index 90fe4f9d3..eabb2e30b 100644
--- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-104.154.120.187 1;
+104.198.131.58 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
index bc389ee32..6bad13743 100644
--- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-104.154.120.187 0;
+104.198.131.58 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
index 9ff18437e..6d439042e 100644
--- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1730
-### Updated: Tue Jul 2 11:39:48 SAST 2019
+### Version: V4.2019.07.1732
+### Updated: Tue Jul 2 12:24:31 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
index 6ff35261d..9e1609b41 100644
--- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
@@ -45,5 +45,5 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.192.85.2 0;
-35.193.7.13 0;
+35.192.187.174 0;
+104.154.113.151 0;
diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
index 7b76022d2..577b311fb 100644
--- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1730
-### Updated: Tue Jul 2 11:39:48 SAST 2019
+### Version: V4.2019.07.1732
+### Updated: Tue Jul 2 12:24:31 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
index 1a9b57786..7803202c9 100644
--- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
@@ -45,6 +45,6 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.192.85.2 0;
-104.154.120.187 0;
-35.224.112.202 0;
+34.66.178.120 0;
+34.68.144.114 0;
+35.193.14.140 0;
diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
index 9ff18437e..6d439042e 100644
--- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1730
-### Updated: Tue Jul 2 11:39:48 SAST 2019
+### Version: V4.2019.07.1732
+### Updated: Tue Jul 2 12:24:31 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
index 607f5404f..90b3681f0 100644
--- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
@@ -45,5 +45,6 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.184.226.236 0;
-35.202.145.110 0;
+104.154.113.151 0;
+35.192.136.167 0;
+104.198.131.58 0;
diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
index 9ff18437e..6d439042e 100644
--- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1730
-### Updated: Tue Jul 2 11:39:48 SAST 2019
+### Version: V4.2019.07.1732
+### Updated: Tue Jul 2 12:24:31 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf
index 966492772..4dddc08a6 100644
--- a/.dev-tools/test_units/blacklist-ips.conf
+++ b/.dev-tools/test_units/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.184.96.71 1;
+35.193.14.140 1;
127.0.0.1 1;
diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list
index ee0a3be7a..e0b0b6621 100644
--- a/.dev-tools/test_units/random-bots-for-test-quick.list
+++ b/.dev-tools/test_units/random-bots-for-test-quick.list
@@ -1,10 +1,10 @@
-Alligator
-BotALot
-IndeedBot
-instabid
-Intelliseek
-LinkpadBot
-LinqiaScrapeBot
-PHPCrawl
-trendiction.de
-WiseGuys Robot
+404enemy
+Anarchie
+Digincore
+ExtractorPro
+FemtosearchBot
+InfoNaviRobot
+MauiBot
+MeanPath Bot
+NICErsPRO
+xpymep1.exe
diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list
index f43e41ab6..af4af9325 100644
--- a/.dev-tools/test_units/random-bots-for-test.list
+++ b/.dev-tools/test_units/random-bots-for-test.list
@@ -1,250 +1,250 @@
+360Spider
+404checker
404enemy
+80legs
Abonti
Aboundex
-Aboundexbot
-Acunetix
-AIBOT
-Alexibot
-Alligator
-AllSubmitter
+AhrefsBot
+AiHitBot
+Aipbot
AlphaBot
-Apexoo
+Anarchie
archive.org_bot
Asterias
-BackDoorBot
+Backlink-Ceck
+backlink-check
+BackStreet
Badass
Bandit
-Battleztar Bazinga
-BBBike
+Barkrowler
+BatchFTP
BDCbot
-BetaBot
-Bigfoot
Blackboard
-BlackWidow
BLEXBot
-Blow
+BlowFish
Boardreader
Bolt
BuiltWith
-CazoodleBot
-CCBot
-Cegbfeieh
-CherryPicker
+Bullseye
+BunnySlippers
Chlooe
-Claritybot
+Cloud mapping
coccocbot-web
Cogentbot
-com.plumanalytics
+Collector
+Copier
CopyRightCheck
+Cosmos
+crawler4j
crawler.feedback
-crawl.sogou.com
-Crescent
+CrunchBot
DatabaseDriverMysqli
-DataCha0s
DBLBot
+DigitalPebble
+DIIbot
+Discobot
+Discoverybot
+Dispatch
DittoSpyder
-DomainAppender
+DomainCrawler
+DomainSigmaCrawler
+DomainStatsBot
+Dotbot
Download Wonder
-Dragonfly
+Drip
DSearch
-EasyDL
-Ebingbong
-Ecxi
+ECCP/1.0
+EirGrabber
+EMail Siphon
evc-batch
-ExtractorPro
-EyeNetIE
+Exabot
+facebookscraper
Fimap
+Firefox/7.0
+FlashGet
Freeuploader
-Genieo
+FrontPage
+FyberSpider
+Fyrebot
Getintent
-GetRight
-Gigablast
Gigabot
-G-i-g-a-b-o-t
-Gotit
-GoZilla
+Go-Ahead-Got-It
+Go!Zilla
GrabNet
Grafula
-GrapeFX
-GridBot
-GT::WWW
-Haansoft
+GrapeshotCrawler
+HaosouSpider
Harvest
-Heritrix
+HEADMasterSEO
HTMLparser
-Humanlinks
-HybridBot
-IDBot
-Indy Library
+HTTP::Lite
+HTTrack
+Iblog
+Id-search
+Image Fetch
+InfoNaviRobot
+InfoTekies
+InterGET
Internet Ninja
-InternetSeer
-internetVista monitor
-ips-agent
Iria
IRLbot
-Iskanie
IstellaBot
JamesBOT
-JetCar
-JOC Web Spider
+JennyBot
+Joomla
+Jorgee
JustView
Jyxobot
Kenjin Spider
+Keyword Density
Lanshanbot
Larbin
-LeechFTP
LeechGet
-LexiBot
-Lftp
+Libwhisker
Lightspeedsystems
-LinkpadBot
-LinkScan
-LinksManager
+LinkWalker
Lipperhey
Litemage_walker
-Ltx71
-lwp-request
+LNSpiderguy
+LWP::Simple
lwp-trivial
-Mag-Net
-magpie-crawler
+Magnet
Mail.RU_Bot
-Majestic SEO
-Mata Hari
+MarkMonitor
+MarkWatch
+Mass Downloader
meanpathbot
-mediawords
-Metauri
+Mediatoolkitbot
MFC_Tear_Sample
-MIDown tool
-MIIxpc
-Mister PiX
Mojeek
-Mr.4x3
+MSFrontPage
MSIECrawler
MS Web Services Client Protocol
-muhstik-scan
-Name Intelligence
Nameprotect
-Nessus
+NetAnts
Netcraft
netEstate NE Crawler
-NetLyzer
-NetSpider
-Nettrack
-Net Vampire
-NetZIP
+NetMechanic
+Netvibes
NextGenSearchBot
-NICErsPRO
+Nibbler
Niki-bot
Nikto
NimbleCrawler
+Ninja
+NPbot
+Nutch
oBot
-OutclicksBot
-OutfoxBot
+Octopus
+Offline Navigator
+OnCrawl
+OpenLinkProfiler
+OpenVAS
+OrangeBot
+OrangeSpider
+Page Analyzer
+page scorer
+PageScorer
Pandalytics
+Panscient
+Papa Foto
pcBrowser
-PHPCrawl
-Picscout
-Picsearch
-PictureFinder
-Pimonster
-Pixray
-PleaseCrawl
plumanalytics
ProWebWalker
Psbot
+PxBroker
PyCurl
+QueryN Metasearch
RankActive
-RankurBot
+RankingBot2
+Rankivabot
+RealDownload
Recorder
-RocketCrawler
-SalesIntelligent
+RedesScrapy
+RepoMonkey
+Ripper
+Rogerbot
SBIder
+Scanbot
+ScoutJet
Scrapy
Searchestate
-SearchmetricsBot
-SEOkicks-Robot
+Semrush
+SEOkicks
SEOlyticsCrawler
-SEOprofiler
-seoscanners
-SEOstats
-sexsearcher
-Shodan
+Seomoz
+SeoSiteCheckup
+Siphon
SISTRIX
-Sitebeam
-SiteExplorer
Siteimprove
-SiteSnagger
+SiteLockSpider
SiteSucker
Site Sucker
-Sitevigil
-SmartDownload
-SMTBot
Snake
+Snapbot
Snoopy
+SocialRankIOBot
Sociscraper
Sosospider
Sottopop
-SpaceBison
+Spammen
+Spanner
+sp_auditbot
SputnikBot
Sqlmap
Steeler
-Stripper
Sucuri
-SuperBot
+SuperHTTP
+Surfbot
SurveyBot
-Suzuran
Szukacz
-TeleportPro
-Telesoft
+T8Abot
+tAkeOut
Telesphoreo
-The Intraformant
-TheNomad
+Telesphorep
+Thumbor
+TightTwatBot
Titan
+Toata
+Trendictionbot
trendiction.com
-True_Robot
Turnitin
TwengaBot
Typhoeus
-Upflow
+UnisterBot
+URLy.Warning
Vacuum
-Vagabondo
-VCI
VeriCiteCrawler
VidibleScraper
-Virusdie
-Voltron
Wallpapers/3.0
-WASALive-Bot
WBSearchBot
Web Auto
+WebBandit
WebCollage
Web Collage
WebCopier
+WEBDAV
+Web Enhancer
WebFetch
-Web Fetch
WebGo IS
-WebLeacher
+WebImageCollector
WebmasterWorldForumBot
-WebPix
-WebReaper
+webmeup-crawler
+Web Pix
Web Sauger
-WebsiteExtractor
-WebsiteQuester
-Webster
+WebSucker
+WebZIP
Whack
-Whacker
-Widow
WinHTTrack
-WISENutbot
+Wonderbot
Woobot
-Wotbox
-WWW::Mechanize
-WWWOFFLE
+WPScan
+WWW-Collector-E
+x09Mozilla
Xaldon WebSpider
-Xaldon_WebSpider
+Xenu
xpymep1.exe
-Zade
-Zauba
-Zeus
+YoudaoBot
ZmEu
+ZumBot
ZyBorg
diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list
index 7ce5c8a66..f330855e3 100644
--- a/.dev-tools/test_units/random-bots-for-whitelist-test.list
+++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list
@@ -1,250 +1,250 @@
-80legs
-Abonti
-Aboundexbot
-AhrefsBot
+360Spider
+Aboundex
AiHitBot
-AllSubmitter
+Alligator
+AlphaBot
Apexoo
archive.org_bot
-ASPSeek
-Attach
-BackStreet
-BackWeb
-Bandit
-Barkrowler
+autoemailspider
+BackDoorBot
+backlink-check
+Badass
+BatchFTP
Battleztar Bazinga
-Black Hole
+BDCbot
+BDFetch
+BetaBot
+Bitacle
+Blackboard
BlackWidow
Blow
-BotALot
-Brandwatch
-Buddy
+BlowFish
+Bolt
+Brandprotect
+BuiltBotTough
BuiltWith
-Cliqzbot
-Cloud mapping
+Calculon
+CATExplorador
+CCBot
+Chlooe
+coccocbot-web
cognitiveseo
-CopyRightCheck
+Copier
Copyscape
-CrazyWebCrawler
+crawler4j
+Crescent
Curious
Custo
-DBLBot
-Demon
+Deusu
+Devil
Digincore
-DigitalPebble
DIIbot
Dirbuster
Discobot
-Dispatch
-DnyzBot
-DomainAppender
-DomainSigmaCrawler
-DSearch
-Ebingbong
+Download Wonder
+Dragonfly
+DTS Agent
ECCP/1.0
-Ecxi
-EroCrawler
-Exabot
+EMail Wolf
+Evil
+Express WebPictures
ExtLinksBot
+ExtractorPro
Extreme Picture Finder
-Ezooms
+EyeNetIE
facebookscraper
-Firefox/7.0
+FDM
+FemtosearchBot
+FHscan
+FlashGet
+Flunky
Foobot
Freeuploader
+FrontPage
GalaxyBot
-Genieo
-GermCrawler
-Getintent
+GetRight
+Gigablast
Gigabot
-GoZilla
-Grafula
-GrapeshotCrawler
-GT::WWW
+G-i-g-a-b-o-t
+Go!Zilla
+Grabber
+GrabNet
+GrapeFX
Havij
-Heritrix
HMView
-HTMLparser
-Humanlinks
-Iblog
-IDBot
+HTTrack
+HybridBot
Id-search
-IlseBot
-Image Fetch
-InfoTekies
+Indy Library
+InfoNaviRobot
instabid
-InterGET
-IRLbot
+Internet Ninja
+InternetSeer
+internetVista monitor
+ips-agent
+Iria
Iskanie
-JetCar
+JennyBot
Jetty
-JikeSpider
-Jorgee
-Kenjin Spider
+JOC Web Spider
+Jyxobot
Keyword Density
+Kozmosbot
Lanshanbot
Larbin
LibWeb
Libwhisker
-Likse
-Linkdexbot
+Lightspeedsystems
LinkpadBot
-LinksManager
LinkWalker
-LinqiaMetadataDownloaderBot
-LinqiaRSSBot
LinqiaScrapeBot
Lipperhey
-Lipperhey Spider
+Litemage_walker
Lmspider
+LNSpiderguy
lwp-request
-lwp-trivial
-Magnet
Mag-Net
-magpie-crawler
+Majestic SEO
+MarkMonitor
+MarkWatch
Masscan
-Mass Downloader
-Mata Hari
-Meanpathbot
+MauiBot
MeanPath Bot
-mediawords
-MegaIndex.ru
-Metauri
+Mediatoolkitbot
MFC_Tear_Sample
Microsoft URL Control
-MIDown tool
MIIxpc
-Morfeus Fucking Scanner
-Msrabot
+MJ12bot
+Mojeek
+Mr.4x3
MS Web Services Client Protocol
-Navroad
-NearSite
-Needle
-Nessus
+muhstik-scan
+Name Intelligence
+Nameprotect
+NetAnts
Netcraft
-NetSpider
-Netvibes
-NextGenSearchBot
+netEstate NE Crawler
+NetLyzer
+NetMechanic
+Nettrack
+Net Vampire
Nibbler
-NICErsPRO
-Niki-bot
-Nikto
+Ninja
+Nmap
NPbot
+Nutch
oBot
-Octopus
-Offline Navigator
+Offline Explorer
OnCrawl
+Openfind
+OpenLinkProfiler
+OpenVAS
OrangeSpider
-PageAnalyzer
-PageGrabber
-PageScorer
-Pandalytics
-Pavuk
-pcBrowser
+OutclicksBot
+Page Analyzer
+page scorer
+PHPCrawl
+Picscout
+Picsearch
+PictureFinder
+Pi-Monster
Pixray
+PleaseCrawl
plumanalytics
Pockey
-Probethenet
+POE-Component-Client-HTTP
ProPowerBot
+Psbot
+Pump
+PxBroker
QueryN Metasearch
-Quick-Crawler
RankActive
-RankingBot
-Rankivabot
+RankingBot2
RankurBot
-RebelMouse
-RedesScrapy
-ReGet
+RealDownload
+Reaper
+RepoMonkey
+Ripper
RocketCrawler
-Rogerbot
-s1z.ru
+SalesIntelligent
ScanAlert
Scanbot
+scan.lol
Scrapy
+ScreenerBot
Semrush
-SEOkicks-Robot
-SEOlyticsCrawler
-Seomoz
SEOprofiler
seoscanners
SeoSiteCheckup
-Shodan
+SEOstats
Sitebeam
+SiteExplorer
SiteLockSpider
-SiteSnagger
SiteSucker
-SlySearch
+Site Sucker
+SmartDownload
SMTBot
+Snake
Snapbot
Snoopy
-SocialRankIOBot
-Sociscraper
-Sogou web spider
Sosospider
+Sottopop
Spammen
-sp_auditbot
-Spbot
-Spinn3r
-SputnikBot
-Sqlmap
+SpankBot
+Sqlworm
Stripper
Sucuri
-Swiftbot
-sysscan
+SuperBot
+SuperHTTP
+Suzuran
Szukacz
-T0PHackTeam
T8Abot
Teleport
-Titan
-Trendiction
+Telesoft
+Telesphorep
+Thumbor
+Trendictionbot
trendiction.com
-TurnitinBot
TwengaBot
-Upflow
+Twice
URLy Warning
-Vacuum
+URLy.Warning
VB Project
+VCI
VeriCiteCrawler
Virusdie
Voil
-Wallpapers/3.0
-WBSearchBot
-WebAuto
-Web Auto
-WebBandit
+WallpapersHD
+Webalta
+Web Collage
WebCopier
+WEBDAV
WebEnhancer
-WebFetch
+Web Enhancer
Web Fetch
-Web Fuck
-WebGo IS
-WebImageCollector
-WebLeacher
WebmasterWorldForumBot
WebPix
Web Pix
-WebReaper
+WebSauger
Web Sauger
-WebsiteQuester
-Website Quester
+Webster
WebStripper
Web Sucker
WebWhacker
-Whack
-Whatweb
+WebZIP
+WeSEE
+Whacker
+WiseGuys Robot
WISENutbot
-WPScan
-WWW-Collector-E
+Wonderbot
WWW-Mechanize
WWW::Mechanize
-WWWOFFLE
x22Mozilla
-Xaldon WebSpider
-Xaldon_WebSpider
Xenu
xpymep1.exe
+YoudaoBot
+Zade
+zauba.io
+Zeus
zgrab
-Zitebot
ZmEu
-ZumBot
-ZyBorg
diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list
index c41a50fbd..929b3e804 100644
--- a/.dev-tools/test_units/random-referrers-for-test-quick.list
+++ b/.dev-tools/test_units/random-referrers-for-test-quick.list
@@ -1,10 +1,10 @@
-aibolita.com
-buntube.net
-elektrozigarettekaufen1.devhub.com
-middlerush-a.akamaihd.net
-mrpornogratis.xxx
-social-buttons.xyz
-stillmiracle.com
-topkarkas.com
-zigarettenonlinekaufen1.blox.pl
-zootravel.com
+apiportalmorecom-a.akamaihd.net
+customboxes4less.com
+dogsrun.net
+free-social-buttons.xyz
+greatgrace.ru
+intimshop-fantasy.ru
+myftpupload.com
+proftests.net
+tourcroatia.co.uk
+zeroredirect9.com
diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list
index 8b69049ad..0454f3800 100644
--- a/.dev-tools/test_units/random-referrers-for-test.list
+++ b/.dev-tools/test_units/random-referrers-for-test.list
@@ -1,500 +1,500 @@
-1688.com
-1bet.com
-1j7740kd.website
-4inn.ru
-5000-cotydzien.com
-55wmz.ru
-5i2.net
-883zy.com
-8kisses.eu
-a96527gi.beget.tech
-aa81bf391151884adfa3dd677e41f94be1.com
-aa8780bb28a1de4eb5bff33c28a218a930.com
-aarbur.com
-aaronabel.com
-abcdeg.xyz
-absolutelyanalog.com
-activecampaign.dreamhosters.com
-adtech.us
-adultwebhosting.info
-affrh2015.com
-agecheckadult.com
-agroeconom.kz
-albuteroli.com
-alert-fjg.xyz
-alfa-medosmotr.ru
-allblogroll.com
-alltheviews.com
-alt-servis.ru
-android4fun.org
-animaltoplist.com
-animenime.ru
-apifasterlightin-a.akamaihd.net
-apimossnetinfo-a.akamaihd.net
-apioasisspacenet-a.akamaihd.net
-apple.com-webbrowsing-security.review
-apple.com-webbrowsing-security.science
-app-ready.xyz
-approvedlocal.co.za
-arcadepages.com
-arenanews.com.ua
-arendas.net
-arkkivoltti.net
-asiavirtualsolutions.com
-asscenihotosidea.blogspot.co.za
-aussie-prizes.com
-auto4style.ru
-autobudpostach.club
-avirasecureserver.com
-avkzarabotok.info
-avon-novosib.ru
-avtochehli.by
-ayanaspa.com
-baditri.com
-bang-hotties.com
-bannerbank.ru
-bdsmgalls.net
-beachfix.co
-berrymall.ru
-bestcalovebracelet.cn
-bestcoin.cc
-bestimagecoollibrary.com
-bestssaker.com
-bet365.com
-beta.hotkeys.com
-betune.onlinewebshop.net
-bidbuy.co.kr
+101raccoon.ru
+11235813.webzdarma.cz
+3weekdiet.com
+4ureyesonly.com
+5-steps-to-start-business.com
+5tey7463.weebly.com
+72-news.com
+7minuteworkout.com
+7xc4n.com
+9icmzvn6.website
+aadroid.net
+abcguru.xyz
+academia-nsk.org
+adanih.com
+addbin.men
+ads.gold
+advancedsoftwaresupport.com
+aihelen.net
+alert.scansafe.net
+alibestsale.com
+aliexpress.com
+amazon-adsystem.com
+amigobulls.com
+andpolice.com
+apartmentbay.ru
+apessay.com
+apps-analytics.net
+arcadeplayhouse.com
+architecturebest.com
+arewater.com
+arius.tech
+asseenontv.ru
+atout-energie-69.com
+auto.rusvile.lt
+availit.weebly.com
+avon-ukraine.com
+avtointeres.ru
+awaybird.ru
+ayerbo.xhost.ro
+baersaratov.ru
+baikaleminer.com
+bankcrediti.ru
+basisches-wasser.net
+beclean-nn.ru
+bedandbreakfast.com
+beget.tech
+belinvestdom.by
+benchmarkcommunications.co.uk
+bestofupload.info
+betterhdporn.com
+bharatdefencekavach.com
+bigcareer.info
+big-cash.net
bigpenisguide.com
-bildsuche.ru
-bio-japan.net
-bio-optomarket.ru
-bkns.vn
-blogsfunky672.weebly.com
-boazpower.com
-bongiornos.info
-boo-arts.com
+bingoporno.com
+bird1.ru
+bjanshee.ru
+bodybuilding-shop.biz
+boeuklad.com
+bolitgorlo.net
+bongacams.com
bookmark4you.com
-boostmyppc.com
-brandehk.dk
-breget74.ru
-bukleteg.xyz
-burger-tycoon.com
-buy-forum.ru
-buy-loft.ru
-buyparajumpers.online
-buypillsorderonline.com
-buzzzg.men
-cacheimages.com
-callcafe.info
-camsex.xxx-cam.webcam
-canrioloadm.gq
-carsplate.com
-cartechnic.ru
-cartierbraceletsreplica.pw
-cartierjusteunclou.xyz
-cdncash.com
-cdncash.org
-celebsopics.com
-cellfun.mobi
-cenokos.ru
+bosman.pluto.ro
+brainsandeggs.blogspot.com
+brimstonehillfortress.org
+buckspar.gq
+budilneg.xyz
+burkesales.com
+business-suggested.tk
+businesxxl.com
+buyessayonline19.blogspot.ru
+buzz-porno.info
+calcularpagerank.com
+call-of-duty.info
+candycrushshop.com
+carapics.com
+casablancamanor.co.za
+cbox.ws
+cdnanalytics.xyz
cfsrating.sonicwall.com
-chee-by.biz
-chelnytruck.ru
-chinese-amezon.com
-cialgenisrx.com
-cindymatches.com
-claytransformations.info
-cleanallvirus.com
-cleanpcnow.com
-clickintext.net
-clips.ua.ac.be
-cl.s7.exct.net
-club-musics.ru
-codysbbq.com
-coinsspb.com
-collegeessay19.blogspot.ru
-collegerentals.com
+chatroulette.life
+chocolateslim-en-france.com
+choice-credit.ru
+ci.ua
+clash-clans.ru
+clicksotrk.com
+clickzzs.nl
+cntravelre.com
+coaching-netz.info
+codq.info
+cometorussia.net
compliance-checker.info
-compliance-fred.top
-compliance-george.top
-compliance-mark.top
-compliance-willy.top
-consorzioilmosaico.org
-contextualyield.com
-counciltally.com
-craftburg.ru
-crazyprotocol.com
-creditwell.ru
-customboxes4less.com
-customcatchcan.com
-customerguru.in
-cymbaltaandtizanidine.blogspot.com
-cyse.tk
-d0t.ru
+concordexoticrentals.com
+confib.ifmo.ru
+connectingsingles.com
+connectionstrenth.com
+continent-e.tv
+cookie-law-enforcement-aa.xyz
+copenergo.ru
+cowblog.fr
+crd.clan.su
+cryptoswap.biz
+cs-passion.pl
+custodieva.ru
dataloading.net
-deadlinkchecker.com
-dieswaene.com
-digilander.libero.it
-dildofotzen.net
-dmmspy.com
-dms-sw.ru
-doctormakarova.ru
-donvito.unas.cz
-dood.live
-downloadkakaotalk.com
-downloadmefiranaratb1972.xpg.com.br
-druzhbany.ru
-dstroy.su
-dumpsmania24.com
-duplicashapp.com
-earn-from-articles.com
-edudocs.net
-electricwheelchairsarea.com
-elektir.ru
-elkacentr.ru
-eloconcream.blogspot.com
-empis.magix.net
-encodable.com
-engines-usa.com
-englishdictionaryfree.com
-englishgamer.com
-e-poker-2005.com
-erotik-kostenlos.net
-es5.com
-escort-russian.com
+dating-time-now.com
+davidovic.info
+dealwifi.com
+dear-diary.ru
+delayreferat.ru
+detmebel.su
+detsky-nabytek.info
+dfwu1019.info
+diamond-necklace.info
+dimkino.ru
+dirtpics.pw
+discovertreasurenow.com
+div.as
+dnepr-avtospar.com.ua
+dogsrun.net
+domain-submit.info
+doska-vsem.ru
+dotmass.top
+doxyporno.com
+drugs-no-rx.info
+drugstoreforyou.com
+eachdayisagift.review
+earthmagic.info
+easyshoppermac.com
+editmedios.com
+ege-essay.ru
+ekspresihati.info
+electrik-avenue.com
+elektrozigarette1.dreamwidth.org
+elektrozigaretten2.yn.lt
+energydiet-info.ru
+enews.tech
+eorogo.top
+eredijovon.com
+erotik0049.com
eshop.md
-excitacion.info
-exlarseva.webblog.es
-extremez.net
-eyemagination.com
-faracontrol.ir
-feeriaclub.ru
-femdom.twiclub.in
-femmesdenudees.com
-fertilitetsradgivningen.se
-filmidivx.com
-finemanteam.com
-floating-share-buttons.com
-flowwwers.com
-footbalive.org
-forsex.info
-foto-basa.com
-foto-sisek.porngalleries.top
-freakycheats.com
-free-social-buttons.com
-frustrated-favorable.gq
-fxgallery.com
-fz139.ttk.ru
+essay-edu.biz
+essaypro.com
+eupornstar.info
+evehemming.blogspot.com.au
+expolicenciaslatam.co
+extlinks.com
+ezaz.info
+facecup.top
+fanoboi.com
+fanpagerobot.com
+fast-wordpress-start.com
+feel-planet.com
+ferretsoft.com
+fetishinside.com
+fettefrauen.net
+fickkontaktehobbyhuren.com
+filefilter.weebly.com
+filmi-v.online
+fishingwholesale.us
+fivedwld.cf
+flavors.me
+forexgb.ru
+formseo.com
+forum.tvmir.org
+fr-bearings.ru
+frbizlist.com
+free-fb-traffic.com
+fullfileaccess.com
+fun-mobi.pl
+galaxyflowers.ru
+gallerily.com
+gallictures.com
+gameonasia.com
gameprimary.com
+gamesprite.me
+game-top.su
gamevalue7.weebly.com
-gaming-journal.com
-gazoblok.net.ua
-getlaid-xxxhookupdirect.com
-getlamborghini.ga
+gay-file.com
+gazobeton-p.com.ua
+gcup.ru
+gdebestkupit.ru
+generalporn.org
+getdot.ru
gfaq.ru
-gilsonchiro.xyz
-glavprofit.ru
-globalscam.ga
-gobongo.info
-good-mummy.ru
-googst2.ru
-gosarhivrt.ru
-gotcher.us
-gov.yanao.ru
-greenidesign.co
-green-tea.tv
-grizzlysgrill.com
-gsasearchengineranker.xyz
-guod.me
-happy.new.yeartwit.com
-hazardky.net
-hdapp1008-a.akamaihd.net
-hdpixent.com
-hdpixion.com
-heroz.fr
-hildinghr.se
-hitcpm.com
-hledejvshopech.cz
-hoholikik.club
-homedecoguide.info
-homemature.net
-hostsshop.ru
+ghernnqr.skyrock.com
+gifspics.com
+girlporn.ru
+girlsatgames.ru
+glall.ru
+glicol.kz
+glopages.ru
+gne8.com
+googlemare.com
+gopixdatabase.com
+gotorussia.com
+gpms.org.my
+gribokstop.com
+guardlink.com
+guitar-master.org
+gungamesz.com
+gz2.bbsoldes.fr
+handicapvansarea.com
+hao123.com
+hdhc.ru
+hentaiheroes.com
+herehloadibs.cf
+herokuapp.com
+highstairs-a.akamaihd.net
+hiwibyh.bugs3.com
+hkdiiohi.skyrock.com
+hoporno.com
+horoshieokna.com
+hostcritique.com
host-tracker.com
-houdom.net
howmuchdoestizanidinecost.blogspot.com
-hubbble.com
-hully.altervista.org
-hyiphunter.org
-iamsport.org
-igrovyeavtomaty777.ru
-igru-xbox.net
-ilmen.net
-ilmexico.com
-ilte.info
-images-free.net
-instabid.tech
-intermesh.net
-inzn.ru
-istizanidineanarcoticdrug.blogspot.com
-iweblist.info
-jasonpartington.com
-javcoast.com
-jav-fetish.com
-jav-fetish.site
-javrip.net
-joessmogtestonly.com
-jose.mulinohouse.co
-jus0wil.pp.ua
-justdating.online
-kamen-e.ru
-karusel-market.ru
-keki.info
-kerwinandcariza.com
-kidd.reunionwatch.com
-kino-filmi.com
-kino-rating.ru
-kinostorm.net
-konoplisemena.com
-kosmetyki.tm.pl
-koversite.info
-kredit-pod-zalog-krasnodar.ru
-krynica.info
-kung-fu-ru.com
-labelwater.se
-laexotic.com
-laptoper.net
-larose.jb2c.me
-laudit.ru
-law-check-seven.xyz
-lerporn.info
-letmacworkfaster.site
-letsrepair.in
-li-er.ru
-limtu.ifmo.ru
+humanorightswatch.org
+hustoon.over-blog.com
+ideoworld.org
+iklysha.ml
+ilovevitaly.xyz
+imfamous.info
+imgarit.pw
+imperia31.ru
+import-sales.com
+impotentik.com
+impresagaia.it
+increasewwwtraffic.info
+indiasourcemart.in
+informatiecentro.be
+inmate-locator.us
+instasexyblog.com
+intellekt21.ru
+interferencer.ru
+internetproviderstucson.com
+intim-uslugi.info
+ipostroika.ru
+islandminingsupply.wordpress.com
+istanbulit.com
+istizanidineacontrolledsubstance.blogspot.com
+istock-mebel.ru
+ixora.pro
+job.icivil.ir
+joingames.org
+jongose.ninja
+kamagragelusa.net
+kam-dom.ru
+karting196.ru
+kazka.ru
+keywordhouse.com
+keywords-monitoring-your-success.com
+keywordteam.net
+kfon.eu
+kino2018.cc
+kinobaks.com
+kinogonew.ru
+kiprinform.com
+kleine-titten.biz
+kletkimehan.ru
+krafte.ru
+krasnodar-avtolombards.ru
+krasnodar.xrus.org
+kupiproday.com.ua
+kvartir-remont.biz
+lacapilla.info
+landscaping.center
+laptop-4-less.com
+lasvegaslockandsafe.com
+lider-zhaluzi.kiev.ua
+lietaer.com
+lifehacĸer.com
+lifeinsurancekb.com
+liffa.ru
+likrot.com
+limads.men
+linkarena.com
+lion.conboy.us
+littlesunraiser.com
+livejasmin.com
+live-sexcam.tk
lkbennettoutlet.online
-lol-smurfs.com
-louboutinbooties.xyz
-lubetube.com
-lyngdalhudterapi.no
-mactechinfo.info
-magnetic-bracelets.ru
-mahnwachen-helfen.info
-mainlinehobby.net
-manifestation.betteroffers.review
-marinetraffic.com
-marketingtechniques.info
-maxxtor.eu
-m.b00kmarks.com
+lodki-pvh.dp.ua
+lomb.co
+lotto6888.com
+m1media.net
+mackeeper-center.club
+malls.com
+marblestyle.ru
+marketland.ml
+master-muznachas.ru
mcnamaratech.com
-mearns-tractors.co.uk
-mebel-vstroika.ru
-medicineseasybuy.com
-medosmotr-ufa.ru
-megavolt.net.ua
-metascephe.com
-mex-annushka.ru
-migente.com
-mlvc4zzw.space
-mobilemedia.md
-mojpregled.com
-moneyviking-a.akamaihd.net
-monsterdivx.com
-moscow.online-podarki.com
-moscow.xrus.org
-moviemail-online.co.uk
-mp3downloadhq.com
-mtmtv.info
-mvpicton.co.uk
-myhitmp3.club
-najaden.no
-nakozhe.com
-narkologiya-belgorod.ru
-needtosellmyhousefast.com
-ngps1.ru
-nippon-bearings.ru
-nordstar.pro
-noumeda.com
-november-lax.com
-nowtorrents.com
-nrv.co.za
-o333o.com
-obesidadealgarve.com
-odywpjtw.bloger.index.hr
-oklogistic.ru
-omoikiri-japan.ru
-online-x.ru
-onlywoman.org
-o-o-11-o-o.com
-open-odyssey.org
-opinionreelle.com
-optikremont.ru
-orgasmus-virtual.com
-orion-v.com
-ourtherapy.ru
-paramountmarble.co.uk
-park.above.com
-partnerworkroom.men
-pastaleads.com
-pc-test.net
-pdamods.ru
-perfection-pleasure.ru
-physfunc.ru
-pila.pl
-pinstake.com
-pistonclasico.com
-piter.xrus.org
-pizdopletka.club
-playtap.us
-polytopesexempt.com
-pontiacsolstice.info
-pooleroadmedicalcentre.co.uk
-popads.net
-popup-hgd.xyz
-popup-jdh.xyz
-porno.simple-image.com.ua
-pornosmola.info
-positive2b.ru
+meble-bogart.info
+medictube.ru
+mega-polis.biz.ua
+mejoresfotos.eu
+minecraft-rus.org
+mini.7zap.com
+minneapoliscopiers.com
+mojowhois.com
+money-for-placing-articles.com
+moskva.nodup.ru
+mountainstream.ms
+mowser.com
+moyaterapiya.ru
+mp3ringtone.info
+msfsaar.de
+musicvidz.ru
+muzaporn.com
+my-cash-bot.co
+myfreemp3.eu
+myperiod.club
+mywallpaper.top
+nationalbreakdown.com
+negociosdasha.com
+neobux-bg.info
+niki-mlt.ru
+nonews.co
+nootrino.com
+normalegal.ru
+novatech.vn
+novosibirsk.xrus.org
+nrjmobile.fr
+nudepatch.net
+oakleyglassesonline.us
+offf.info
+okel.co
+oksrv.com
+online247.ml
+onlinebay.ru
+onlinedomains.ru
+online-hit.info
+onstrapon.purplesphere.in
+o-o-8-o-o.ru
+oops-cinema.ru
+openstat.com
+orel-reshka.net
+osb.se11.ru
+ossmalta.com
+ownshop.win
+partners-ship.pro
+pendelprognos.se
+pensplan4u.com
+pictures-and-images.com
+pictures-and-images.net
+platesauto.com
+play.leadzupc.com
+pochtovyi-index.ru
+pokerniydom.ru
+poquoson.org
+porndairy.in
+pornogratisdiario.com
+pornoklad.ru
+pornosemki.info
pp-budpostach.com.ua
-preparevideosafesystem4unow.space
-presleycollectibles.com
-preventheadacheguide.info
-privacyassistant.net
-propranolol40mg.blogspot.com
-prospekt-st.ru
-prostitutki-kiev.org
+predmety.in.ua
+pregnant.guru
+primedice.com
+princevc.com
+printdirectforless.com
+privatefx.all4invest.info
+prizestohandle.club
+professionalwritingservices15.blogspot.ru
+proftests.net
prostitutki-tyumeni.org
-putevka24.ru
-quality-traffic.com
-randalljhoward.com
-razorweb-a.akamaihd.net
-recordpage-a.akamaihd.net
-reeyanaturopathy.com
-referencemoi.com
-registratciya-v-moskve.ru
-renewablewealth.com
-restaurantlescampi.com
-rezeptiblud.ru
-rimedia.org
-rocketchange.ru
+pyrodesigns.com.au
+qld10000.net
+radiodigital.co
+rank-checker.online
+ranksonic.org
+r-control.ru
+regionshop.biz
+reklamuss.ru
+replica-watch.ru
+resant.ru
rockprogblog.com
-rogervivierforsale.com
-rozalli.com
+romhacking.ru
+ruscopybook.com
+rusexy.xyz
rusprostitute.com
-russkoe-zdorovie.ru
-saltspray.ru
-sammyweaver.com
-sarafangel.ru
-sardinie.us
-scanner-ivan.top
-scanner-jessica.top
-search-error.com
-seeresultshub-a.akamaihd.net
-seo-prof1.xyz
-seorankinglinks.com
-servethis.com
-sethrollins.net
-sexiporno.net
-sexyali.com
-shama-rc.net
-shisha-swag.de
-signoredom.com
-sildenafil-tadalafil.info
-sisi-go.ru
-sispe.com.br
-sitevaluation.org
-skincrate.net
-slonechka.ru
-slujbauborki.ru
-snip.to
-soaksoak.ru
-socialmediasuggest.com
-social-s-hhh.xyz
-spamnuker.com
-spasswelt.net
-sports-supplements.us
-spylog.com
-stairliftstrue.com
-starpages.net
-suchenindeutschland.com
-supermama.top
-supernew.org
-supervesti.ru
-sureone.pro
-svetlotorg.ru
-tako3.com
-tampabaywatch.org
-tandvardshuset.net
-tattooreligion.ru
-telegramdownload10.com
+rustic-quiver.win
+rutor.vip
+sack.net
+sammlungfotos.online
+samo-soznanie.ru
+sanatorrii.ru
+sape.top
+sbwealthsolutions.ca
+scanner-mary.top
+school-diplomat.ru
+scottbywater.com
+scrnet.biz.ua
+securityallianceservices.com
+seolab.top
+seo-traffic-ranking.info
+seryeznie-znakomstva.ru
+sevendays.com.ua
+sex.hotblog.top
+sexreliz.com
+shakhtar-doneck.ru
+sharebutton.org
+sharpchallenge.com
+shemalegalls.blogporn.in
+shoppingjequiti.com.br
+shoppytoolmac.com
+sideeffectsoftizanidine.blogspot.com
+sims-sims.ru
+siteonomy.com
+sitevaluation.com
+skutecznetabletkinaporostwlosow.pl
+sky-mine.ru
+snip.tw
+social-search.me
+solicita.info
+spbchampionat.ru
+spidtest.org
+spinnerco.ca
+spy-app.info
+sstroy44.ru
+stardevine.com
+stariy-baku.com
+startwp.org
+staynplay.net
+stpicks.com
+stpolice.com
+strongholdsb.ru
+sukirgenk.dvrlists.com
+supers.com.ua
+surfbuyermac.com
+svarbit.com
+taaaak.com
+td-33.ru
+tech4master.com
+tengohydar.tk
thebestweightlosspills.ovh
-thejournal.ru
-themestotal.com
-thenetinfo.com
-tizanidineforopiatewithdrawal.blogspot.com
-tizanidineingredients.blogspot.com
-tizanidineonline.blogspot.com
-toolsky.com
-topkarkas.com
-torrents-tracker.com
-torture.ml
-traffic100.com
-trafficfactory.biz
-trafficmania.com
-trafficstars.com
-traxdom.ru
-tv-spoty.info
-uasb.ru
-uchebavchehii.ru
-uggbootsoutletsale.us
-ugguk.online
-ukrobstep.com
-umaseh.com
-undergroundcityphoto.com
-unisexjewelry.org
-urccvfmc.bloger.index.hr
+thefds.net
+thewebsitetemplate.info
+thfox.com
+threecolumnblogger.com
+tizanidine4mgprice.blogspot.com
+tizanidine4mgstreetprice.blogspot.com
+tizanidineformigraines.blogspot.com
+tizanidinesideeffectsweightloss.blogspot.com
+tizanidinestreetvalue.blogspot.com
+tizanidinevsvalium.blogspot.com
+tn811.us
+tongkatmadura.info
+top-l2.com
+topshef.ru
+topsy.com
+toxicwap.com
+tpu.ru
+trackmedia101.com
+traffpartners.com
+trkur.com
+truebeauty.cc
+try-rx.com
+tversvet.ru
+underthesite.com
+uptimebot.net
+uroffer.link
usiad.net
-vapsy.com
+vacuumscleaner.com
+valmetrundan.se
vchulkax.com
-vclicks.net
-vedomstvo.net
-ventelnos.com
-video-woman.com
-viktoria-center.ru
-visionwell.com.cn
-visitcambridge.org
-vizitki.net
-vodaodessa.com
-voluumtracker1.com
-vseuznaem.com
-vzlom-na-zakaz.com
-warning.or.kr
+vip2ch.com
+vip-dom.in
+vipsiterip.org
+vitanail.ru
+vkak.ru
+vkgaleria.com
+vkontaktemusic.ru
+vod.com.ua
+vriel.batcave.net
+vykup-avto-krasnodar.ru
webadvance.club
-webkeyit.com
-webmasterhome.cn
-website-speed-check.site
-website-speed-up.site
-weburlopener.com
-wingsofrefuge.net
-workle.website
-wsgames.ru
-www.get-free-traffic-now.com
-wyniki-lista.pl
-x5market.ru
-x-diesel.biz
-xn-----7kcabaipgeakzcss7bjdqdwpfnhv.xn--p1ai
-xn--80aikhbrhr.net
-xn--h1aakne2ba.xn--p1ai
-yebocasino.com
-youdao.com
-yougetsignal.com
-yourothersite.com
-youtube-downloader.savetubevideo.com
+webalan.ru
+webextract.profound.net
+weblo.com
+webscouter.net
+websites-reviews.com
+weprik.ru
+whiteelephantwellington.com
+womensterritory.ru
+woodyguthrie.se
+wowcasinoonline.ooo
+writingservices17.blogspot.ru
+xbaboon.com
+x-lime.net
+xmronta.com
+xn--24-glceagatoq7c2a6ioc.xn--p1ai
+yachts-cruise.info
+yaderenergy.ru
+yourdesires.ru
+yourporn.com
+yoursearch.me
+youtubedownload.org
youtuhe.com
-yuarra.pluto.ro
-zaimite.ru
-zakazfutbolki.com
-zarabotok--doma.ru
-zarenica.net
-zigarettenonlinekaufen1.bloog.pl
-zootravel.com
-zverokruh-shop.cz
+z-master.ru
+znakom.sibtest.ru
+zoogdiesney.com
diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list
index 47df0f97b..581b78064 100644
--- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list
+++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list
@@ -1,250 +1,250 @@
-7makemoneyonline.com
-7minuteworkout.com
+08800.top
+0912701309f8ce.com
+100dollars-seo.com
+1kinobig.ru
+2daytrendingnews.com
+5u.com
+6hopping.com
+aaa38852e886ac4af1a3cff9b47cab6272.com
+aac52006ec82a24e08b665f4db2b5013f7.com
aad1f4acb0a373420d9b0c4202d38d94fa.com
-abovetherivernc.com
-adultfullhd.com
-advokat-grodno.by
-agysya.ru
-alert.scansafe.net
-allfinweb.com
-alphahoverboards.com
-alyeskaresort.com
-animali.deagostinipassion.it
-anjalika.co.in
-apccargo.com
-apimegabrowsebiz-a.akamaihd.net
-apisecretsaucebi-a.akamaihd.net
-appleid-verification.com
-a.pr-cy.ru
-ap.senai.br
-asiavirtualsolutions.com
-atelielembrancaqueencanta.com.br
+actices.com
+adrunnr.com
+adserver-e7.com
+agroeconom.kz
+allblogroll.com
+allsilver925.co.il
+allstatesugarbowl.org
+amehdaily.com
+aosheng-tech.com
+appartement-stumm.at
+appfastplay.com
+apxeo.info
+arclk.net
+asseenontv.ru
+atley.eu.pn
+atmagroup.ru
+atofilms.com
+ausmepa.org.au
autorn.ru
-azazaporn.com
+ayakino.net
+azazu.ru
+bang-hotties.com
+bard-real.com.ua
+basedecor.ru
bbtec.net
-best-way.men
-bgviagrachrx.com
-big-boards.info
-blobar.org
-bolezniorganov.ru
+beatifullandscape.co
+belinka.com.ua
+best-seo-offer.com
+blacktwhite.com
+bloggedporn.com
+bonux.nextview.ru
brainzpod.win
-browserprotecter.com
-buigas.00it.com
-businesxxl.com
-buypuppies.ca
-c1.onedmp.com
-camdolls.com
-cartiertrinity.xyz
-cdncash.org
-checktext.org
-chinese-amezon.com
-club-samodelkin.ru
+buntube.net
+business-suggested.tk
+buttons-for-website.com
+bvlgariring.xyz
+callawaygolfoutlet.online
+cardsdumps.com
+centraletermice.us
+chatroulette.si
+chocolateslim-original.com
cntravelre.com
-codq.info
-companies-catalog.com
-compliance-checker.info
-cool-wedding.net
-cornerstone-countertops.com
-cougfan.info
-covetnica.com
-crazyprotocol.com
-crest-poloski.ru
-cyprusbuyproperties.com
-dame-ns.kz
-dataloading.net
-davidovic.info
-db.speedup.gdn
-deanmoore.ie
-dentalpearls.com.au
-deutschland123.de
-digitalassetmanagement.site
-duplicashapp.com
-e-buyeasy.com
-eduinfosite.com
-ege-essay.ru
-elektrozigaretten1.postbit.com
-enge-fotzen.info
-epicbrogaming.com
-e-poker-2005.com
-erotik0049.com
-esfchat.tk
-extlinks.com
-extrabot.com
-eyessurgery.ru
-familyphysician.ru
-fashion-mk.net
-femmesdenudees.com
-fetroshok.ru
-filesmonster.porn
-finemanteam.com
-focalink.com
-forex.osobye.ru
+compliance-elena.xyz
+compliance-jack.top
+concealthyself.com
+copypaste.traderzplanet.in
+culturevie.info
+cv.wallhade.co
+dafatiri.com
+daneshetabiat.com
+davebestdeals.com
+dealwifi.com
+de.zapmeta.com
+distver.ru
+dostavimvdom.ru
+easytuningshop.ru
+edwinkonijn.com.au
+enews.tech
+eu-cookie-law.blogspot.com
+excitacao.com
+exct.net
+ezigaretteshop.webs.com
+farmprofi.net
+fatmaelgarny.com
+fearcrow.com
+forum69.info
fotopop.club
-freenom.link
-freshwallpapers.info
-frivgame250.com
-funnymama.com
-gamedayassist.com
-game-top.su
-ge0ip.net
-geile-lelly.eu
-getmyads24.com
-ghostvisitor.com
-golmau.host.sk
-gratis-sexkontakte.com
-gsasearchengineranker.space
-hatedriveapart.com
-hd720kino.ru
+foto-sisek.porngalleries.top
+foxjuegos.com
+free-stock-illustration.com
+front.ru
+game300.ru
+gameplexcity.com
+gidonline.one
+girlspicsa.com
+glcomputers.ru
+gobongo.info
+gomusix.com
+gribkovye-zabolevaniya.com
+guod.me
+highspeed5.net
+hop.clickbank.net
+hornymatches.com
hothor.se
-imagefinder.site
-imgarcade.com
-imperia31.ru
-integritylandscapeservices.com
-intelhdgraphicsgtdrive6w.metroblog.com
-intimshop-fantasy.ru
-jasonpartington.com
-jwss.cc
-kanimage.com
-kareliatobacco.ru
-kerch.site
-keywordblocks.com
-keywordsuggests.com
-kimcurlrvsms.com
-kovesszucs.atw.hu
-krasivoe-hd.net
-krassh.ru
-lafriore.ru
-larchik.net
-letslowbefast.site
-liquimondo.com
-liverpool.gsofootball.com
-love-baby.cz
-machicon-akihabara.info
-mactechinfo.info
-magnetic-bracelets.ru
-mecash.ru
-meddesk.ru
-megaindex.ru
-me-ke.com
-metallosajding.ru
-mobile-appster.ru
-moesen-ficken.com
-moscow-region.ru
-mp3films.ru
-musicdaddy.net
-myanyone.net
-myfreemp3.eu
-mypornfree.ru
+housemilan.ru
+humbmosquina.tk
+igrovyeavtomaty777.ru
+iinstalll-fii1leis.jus0wil.pp.ua
+imguramx.pw
+indetiske.ya.ru
+inzn.ru
+joingames.org
+katushka.net
+keywordsdoctor.com
+kinobaks.com
+kinoplen.ru
+kountrylife.com
+ku6.com
+kuzinsp.ru
+lion.misba.us
+liveinternet.ro
+lollypopgaming.com
+luckyshop.net.ua
+magicplayer-s.acestream.net
+mahnwachen-helfen.info
+marcogrup.com
+marcowebonyodziezowe.pl
+mastershef.club
+mebel-alait.ru
+meet-flirt-dating.com
+mesbuta.info
+mhtr.be
+mirzonru.net
+mokrayakiska.com
+moneymaster.ru
+mrwhite.biz
+myonlinepayday.co
+neks.info
newyorkhotelsmotels.info
-notasprensa.info
-ntic.fr
-oconto.ru
-olvanto.ru
-online-x.ru
-oranga.host.sk
-orhonit.com
-pawli.eu
-pensplan4u.com
-pic2fly.com
-piulatte.cz
-playboyfiles.xblog.in
-pl.id-forex.com
-pojdelo.weebly.com
-pons-presse.com
-porn-w.org
-pornzone.tv
-prefersurvey.net
-proekt-gaz.ru
-profitfx.online
-promgirldresses.xyz
-puteshestvennik.com
-q-moto.ru
-quick-seeker.com
-rabot.host.sk
-randalljhoward.com
-randki-sex.com
-rednise.com
-remont-comp-pomosh.ru
-remote-dba.de
-rentaremotecomputer.com
-rewardpoll.com
-rp9.ru
-rubbed.us
-sakhboard.ru
-sammyweaver.com
-saneitconsulting.com
-sbtdesign.co.uk
-scanner-nelson.top
-scanner-willy.top
-se.bnt-team.com
-seminarykansas904.ml
-seoexperimenty.ru
-serialsway.ucoz.ru
-servicecenter.co.ua
-sextracker.be
-sexy-screen-savers.com
-shoesonlinebuy.cn
-shopperifymac.com
-shopwme.ru
-smichovbike.cz
-sneakyboy.com
-spammen.de
+nikhilbahl.com
+niki-mlt.ru
+novosti-hi-tech.ru
+nrjmobile.fr
+obuv-kupit.ru
+offer.wpsecurity.website
+officedocuments.net
+omsk.xrus.org
+onclickpredictiv.com
+onemactrckr.com
+onload.pw
+optibuymac.com
+orel-reshka.net
+paparazzistudios.com.au
+paradontozanet.ru
+pay2me.pl
+photokitchendesign.com
+picphotos.net
+piuminiita.com
+pizdeishn.com
+pizza-tycoon.com
+playtap.us
+poem-paying.gq
+porngalleries.top
+porno-raskazy.ru
+praisong.net
+prizeestates.cricket
+proxyelite.biz
+purplestats.com
+quickloanbank.com
+r-control.ru
+refererx.com
+registry-clean-up.net
+rique.host.sk
+royal-betting.net
+rusprostitute.com
+sashagreyblog.ga
+savefrom.com
+sax-sex.com
+scoopquest.com
+seogadget.ru
+seorank.info
+seorankinglinks.com
+servingnotice.com
+sex-watch.com
+sims-sims.ru
+site3.free-share-buttons.com
+sitevaluation.com
+s-iwantyou.com
+skutecznetabletkinaporostwlosow.pl
+snworks.com
+social-button.xyz
+softlinesolutions.me
+soft-program.com
+softwaretrend.net
+space-worry.ml
spywareit.com
-stickers-market.ru
-stockquotes.wooeb.com
-strigkaomsk.ru
-surflinksmedical.com
-swsociety.se
-syvertsen-da.no
-tam-gde-more.ru
-tds-advert002.info
-teslathemes.com
-thaismartloan.com
-thelottosecrets.com
-thetattoohut.com
-tiandeural.ru
-tizanidinehcl2mgtablet.blogspot.com
-tizanidineshowupondrugtest.blogspot.com
-tizanidinesleepaid.blogspot.com
-token-lab.org
-torrent-to-magnet.com
-tracfone.com
-tradgardspartner.se
-trafficnetzwerk.de
-trafficstars.com
-trahvid.com
-triumf-realty.ru
-truebeauty.cc
-tupper-posuda.ru
-twiclub.in
-uac.net.au
-uhdtv.website
-underthesite.com
-urengoy.pro
-videochat.world
-videositename.com
-vietimgy.pw
-villakohlanta.nu
-vksaver-all.ru
-voltrknc1.com
-webadvance.club
-websitevaluebot.com
-westen-z.life
-wleuaprpxuvr.ga
-woman-orgasm.ru
-x-diesel.com
-xep.info
-xev.ru
-xn-----8kcatubaocd1bneepefojs1h2e.xn--p1ai
-xn--b1ag5cfn.xn--p1ai
-x-porno.video
-x-stars.ru
-xtubeporno.net
-xvideosporn.biz
-yoshkarola.zrus.org
-youbloodyripper.com
-youradulthosting.com
-youtuhe.com
-yubikk.info
-zdesoboi.com
-zs2vm.top
+stair.registrydomainservices.com
+stats-collector.org
+steelmaster.lv
+stillmiracle.com
+streetfire.net
+symphonyintegratedhealthcare.com
+tabletkinaodchudzanie.com.pl
+td-l-market.ru
+telegraf.by
+telemetryverification.net
+tetracsaudi.com
+thaisamkok.com
+thebestweightlosspills.ovh
+thedownloadfromwarez.blogspot.com
+timdreby.com
+tizanidineandhydrocodone.blogspot.com
+tizanidineonline.blogspot.com
+tizanidinestreetprice.blogspot.com
+tmm-kurs.ru
+tongkatmadura.info
+toonfamilies.net
+tootoo.to
+torture.ml
+totu.info
+trackmedia101.com
+trafficreceiver.club
+turbodsp.com
+tvnewsclips.info
+tytoona.com
+ultimateclassicrock.com
+umnovocaminho.com
+unitexindia.com
+unrealcommander.org
+upupa.net
+uroki.net
+uytmaster.ru
+v720hd.ru
+valmetrundan.se
+verymes.xyz
+videospornogratisx.net
+viel.su
+voronezh.xrus.org
+vostoktrade.info
+w7s.ru
+webkeyit.com
+wetgames.ru
+whatzmyip.net
+wholinkstome.com
+woman-h.ru
+workle.website
+xblog.in
+xn----7sbahjd3btneuw1joc.xn--p1ai
+xn--b1agm2d.net
+xn----itbkqkfiq.xn--p1ai
+xrus.org
+xuki.us
+xvideosbay.com
+yottos.com
+youandcredit.ru
+yourporn.com
+yunque.pluto.ro
+zero1.it
+zetgie.com.pl
+zzlgxh.com
diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt
index aea69b849..f371d9e5f 100644
--- a/.dev-tools/test_units/ratelimittest-beta.txt
+++ b/.dev-tools/test_units/ratelimittest-beta.txt
@@ -126,14 +126,4 @@
Welcome to the NginX Ultimate Bad Bot Blocker
This is merely used for testing !!!
-