diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
index f8d2fbdec..b04cc1676 100644
--- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1748
-### Updated: Fri Jul 5 08:31:37 SAST 2019
+### Version: V3.2019.07.1749
+### Updated: Fri Jul 5 08:51:59 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 689d439aa..4dddc08a6 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-34.66.178.120 1;
+35.193.14.140 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index 34cb39aec..40cb4d7a8 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-34.66.178.120 0;
+35.193.14.140 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index f8d2fbdec..b04cc1676 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1748
-### Updated: Fri Jul 5 08:31:37 SAST 2019
+### Version: V3.2019.07.1749
+### Updated: Fri Jul 5 08:51:59 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 689d439aa..4dddc08a6 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-34.66.178.120 1;
+35.193.14.140 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index 34cb39aec..40cb4d7a8 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-34.66.178.120 0;
+35.193.14.140 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
index f8d2fbdec..b04cc1676 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1748
-### Updated: Fri Jul 5 08:31:37 SAST 2019
+### Version: V3.2019.07.1749
+### Updated: Fri Jul 5 08:51:59 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
index 689d439aa..4dddc08a6 100644
--- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-34.66.178.120 1;
+35.193.14.140 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
index 34cb39aec..40cb4d7a8 100644
--- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-34.66.178.120 0;
+35.193.14.140 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
index f8d2fbdec..b04cc1676 100644
--- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1748
-### Updated: Fri Jul 5 08:31:37 SAST 2019
+### Version: V3.2019.07.1749
+### Updated: Fri Jul 5 08:51:59 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template
index f8d2fbdec..b04cc1676 100644
--- a/.dev-tools/globalblacklist-testing.template
+++ b/.dev-tools/globalblacklist-testing.template
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1748
-### Updated: Fri Jul 5 08:31:37 SAST 2019
+### Version: V3.2019.07.1749
+### Updated: Fri Jul 5 08:51:59 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template
index e7c061b71..56397ca21 100755
--- a/.dev-tools/globalblacklist.template
+++ b/.dev-tools/globalblacklist.template
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1748
-### Updated: Fri Jul 5 08:31:36 SAST 2019
+### Version: V4.2019.07.1749
+### Updated: Fri Jul 5 08:51:58 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/setup-ngxblocker/commented/arch.conf.modified b/.dev-tools/setup-ngxblocker/commented/arch.conf.modified
index 25cd0c350..0dfe86885 100644
--- a/.dev-tools/setup-ngxblocker/commented/arch.conf.modified
+++ b/.dev-tools/setup-ngxblocker/commented/arch.conf.modified
@@ -1,4 +1,13 @@
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
#user html;
worker_processes 1;
@@ -26,15 +35,6 @@ http {
sendfile on;
#tcp_nopush on;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
#keepalive_timeout 0;
keepalive_timeout 65;
diff --git a/.dev-tools/setup-ngxblocker/commented/fedora.conf.modified b/.dev-tools/setup-ngxblocker/commented/fedora.conf.modified
index 1a37d53e8..ce4abeece 100644
--- a/.dev-tools/setup-ngxblocker/commented/fedora.conf.modified
+++ b/.dev-tools/setup-ngxblocker/commented/fedora.conf.modified
@@ -1,4 +1,13 @@
# For more information on configuration, see:
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
# * Official English Documentation: http://nginx.org/en/docs/
# * Official Russian Documentation: http://nginx.org/ru/docs/
@@ -23,15 +32,6 @@ http {
sendfile on;
tcp_nopush on;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
diff --git a/.dev-tools/setup-ngxblocker/commented/opensuse.conf.modified b/.dev-tools/setup-ngxblocker/commented/opensuse.conf.modified
index 6f426e57e..8e3a37c0b 100644
--- a/.dev-tools/setup-ngxblocker/commented/opensuse.conf.modified
+++ b/.dev-tools/setup-ngxblocker/commented/opensuse.conf.modified
@@ -1,4 +1,13 @@
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
#user nginx;
worker_processes 1;
@@ -38,15 +47,6 @@ http {
sendfile on;
#tcp_nopush on;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
#keepalive_timeout 0;
keepalive_timeout 65;
diff --git a/.dev-tools/setup-ngxblocker/commented/ubuntu16.conf.modified b/.dev-tools/setup-ngxblocker/commented/ubuntu16.conf.modified
index 672bc00da..4cbde66dd 100644
--- a/.dev-tools/setup-ngxblocker/commented/ubuntu16.conf.modified
+++ b/.dev-tools/setup-ngxblocker/commented/ubuntu16.conf.modified
@@ -1,4 +1,13 @@
user www-data;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
worker_processes auto;
pid /run/nginx.pid;
@@ -15,15 +24,6 @@ http {
sendfile on;
tcp_nopush on;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
diff --git a/.dev-tools/setup-ngxblocker/commented/ubuntu18.conf.modified b/.dev-tools/setup-ngxblocker/commented/ubuntu18.conf.modified
index b8f9bebb3..353cd606a 100644
--- a/.dev-tools/setup-ngxblocker/commented/ubuntu18.conf.modified
+++ b/.dev-tools/setup-ngxblocker/commented/ubuntu18.conf.modified
@@ -1,4 +1,13 @@
user www-data;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
@@ -16,15 +25,6 @@ http {
sendfile on;
tcp_nopush on;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
diff --git a/.dev-tools/setup-ngxblocker/modified/arch.conf.modified b/.dev-tools/setup-ngxblocker/modified/arch.conf.modified
index 25cd0c350..0dfe86885 100644
--- a/.dev-tools/setup-ngxblocker/modified/arch.conf.modified
+++ b/.dev-tools/setup-ngxblocker/modified/arch.conf.modified
@@ -1,4 +1,13 @@
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
#user html;
worker_processes 1;
@@ -26,15 +35,6 @@ http {
sendfile on;
#tcp_nopush on;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
#keepalive_timeout 0;
keepalive_timeout 65;
diff --git a/.dev-tools/setup-ngxblocker/modified/centos.conf.modified b/.dev-tools/setup-ngxblocker/modified/centos.conf.modified
index aaa446334..8660e9a50 100644
--- a/.dev-tools/setup-ngxblocker/modified/centos.conf.modified
+++ b/.dev-tools/setup-ngxblocker/modified/centos.conf.modified
@@ -1,4 +1,13 @@
# For more information on configuration, see:
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
# * Official English Documentation: http://nginx.org/en/docs/
# * Official Russian Documentation: http://nginx.org/ru/docs/
@@ -23,15 +32,6 @@ http {
sendfile on;
tcp_nopush on;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
diff --git a/.dev-tools/setup-ngxblocker/modified/fedora.conf.modified b/.dev-tools/setup-ngxblocker/modified/fedora.conf.modified
index 709b85100..c2a034b98 100644
--- a/.dev-tools/setup-ngxblocker/modified/fedora.conf.modified
+++ b/.dev-tools/setup-ngxblocker/modified/fedora.conf.modified
@@ -1,4 +1,13 @@
# For more information on configuration, see:
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
# * Official English Documentation: http://nginx.org/en/docs/
# * Official Russian Documentation: http://nginx.org/ru/docs/
@@ -23,15 +32,6 @@ http {
sendfile on;
tcp_nopush on;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
diff --git a/.dev-tools/setup-ngxblocker/modified/opensuse.conf.modified b/.dev-tools/setup-ngxblocker/modified/opensuse.conf.modified
index 4e41cc065..a18d796f1 100644
--- a/.dev-tools/setup-ngxblocker/modified/opensuse.conf.modified
+++ b/.dev-tools/setup-ngxblocker/modified/opensuse.conf.modified
@@ -1,4 +1,13 @@
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
#user nginx;
worker_processes 1;
@@ -38,15 +47,6 @@ http {
sendfile on;
#tcp_nopush on;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
#keepalive_timeout 0;
keepalive_timeout 65;
diff --git a/.dev-tools/setup-ngxblocker/modified/ubuntu16.conf.modified b/.dev-tools/setup-ngxblocker/modified/ubuntu16.conf.modified
index 55bba61ef..5393a869b 100644
--- a/.dev-tools/setup-ngxblocker/modified/ubuntu16.conf.modified
+++ b/.dev-tools/setup-ngxblocker/modified/ubuntu16.conf.modified
@@ -1,4 +1,13 @@
user www-data;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
worker_processes auto;
pid /run/nginx.pid;
@@ -15,15 +24,6 @@ http {
sendfile on;
tcp_nopush on;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
diff --git a/.dev-tools/setup-ngxblocker/modified/ubuntu18.conf.modified b/.dev-tools/setup-ngxblocker/modified/ubuntu18.conf.modified
index 5454fdb65..c2bc002c1 100644
--- a/.dev-tools/setup-ngxblocker/modified/ubuntu18.conf.modified
+++ b/.dev-tools/setup-ngxblocker/modified/ubuntu18.conf.modified
@@ -1,4 +1,13 @@
user www-data;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
@@ -16,15 +25,6 @@ http {
sendfile on;
tcp_nopush on;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
diff --git a/.dev-tools/setup-ngxblocker/original/arch.conf.modified b/.dev-tools/setup-ngxblocker/original/arch.conf.modified
index 25cd0c350..0dfe86885 100644
--- a/.dev-tools/setup-ngxblocker/original/arch.conf.modified
+++ b/.dev-tools/setup-ngxblocker/original/arch.conf.modified
@@ -1,4 +1,13 @@
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
#user html;
worker_processes 1;
@@ -26,15 +35,6 @@ http {
sendfile on;
#tcp_nopush on;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
#keepalive_timeout 0;
keepalive_timeout 65;
diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
index c1d6c61cb..ca4fcdf54 100644
--- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
@@ -45,4 +45,4 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.188.73.34 0;
+35.193.14.140 0;
diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
index e7c061b71..56397ca21 100644
--- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1748
-### Updated: Fri Jul 5 08:31:36 SAST 2019
+### Version: V4.2019.07.1749
+### Updated: Fri Jul 5 08:51:58 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 689d439aa..0fcb73d00 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-34.66.178.120 1;
+35.192.187.174 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index 34cb39aec..3dac4f84e 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-34.66.178.120 0;
+35.192.187.174 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index e7c061b71..56397ca21 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1748
-### Updated: Fri Jul 5 08:31:36 SAST 2019
+### Version: V4.2019.07.1749
+### Updated: Fri Jul 5 08:51:58 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 689d439aa..0fcb73d00 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-34.66.178.120 1;
+35.192.187.174 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index 34cb39aec..3dac4f84e 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-34.66.178.120 0;
+35.192.187.174 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
index e7c061b71..56397ca21 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1748
-### Updated: Fri Jul 5 08:31:36 SAST 2019
+### Version: V4.2019.07.1749
+### Updated: Fri Jul 5 08:51:58 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
index 689d439aa..0fcb73d00 100644
--- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-34.66.178.120 1;
+35.192.187.174 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
index 34cb39aec..3dac4f84e 100644
--- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-34.66.178.120 0;
+35.192.187.174 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
index e7c061b71..56397ca21 100644
--- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1748
-### Updated: Fri Jul 5 08:31:36 SAST 2019
+### Version: V4.2019.07.1749
+### Updated: Fri Jul 5 08:51:58 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
index 8375c12ee..021042013 100644
--- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
@@ -45,5 +45,5 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-34.66.178.120 0;
-104.154.113.151 0;
+35.192.187.174 0;
+35.188.73.34 0;
diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
index 8ffe2646a..0e75d1753 100644
--- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1748
-### Updated: Fri Jul 5 08:31:36 SAST 2019
+### Version: V4.2019.07.1749
+### Updated: Fri Jul 5 08:51:58 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
index 8497e42c2..7a9d57444 100644
--- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
@@ -45,5 +45,5 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-34.66.178.120 0;
104.198.131.58 0;
+35.188.73.34 0;
diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf
index 974846a53..658da1401 100644
--- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1748
-### Updated: Fri Jul 5 08:31:36 SAST 2019
+### Version: V4.2019.07.1749
+### Updated: Fri Jul 5 08:51:58 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
index 9e1609b41..fa2ec0d6d 100644
--- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
@@ -45,5 +45,6 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.192.187.174 0;
+34.66.178.120 0;
+34.68.144.114 0;
104.154.113.151 0;
diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
index e7c061b71..56397ca21 100644
--- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1748
-### Updated: Fri Jul 5 08:31:36 SAST 2019
+### Version: V4.2019.07.1749
+### Updated: Fri Jul 5 08:51:58 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
index 73d9328aa..6140a6e39 100644
--- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
@@ -46,5 +46,4 @@
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
34.66.178.120 0;
-34.68.144.114 0;
-35.192.187.174 0;
+35.192.136.167 0;
diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
index e7c061b71..56397ca21 100644
--- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1748
-### Updated: Fri Jul 5 08:31:36 SAST 2019
+### Version: V4.2019.07.1749
+### Updated: Fri Jul 5 08:51:58 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf
index 689d439aa..4dddc08a6 100644
--- a/.dev-tools/test_units/blacklist-ips.conf
+++ b/.dev-tools/test_units/blacklist-ips.conf
@@ -1,2 +1,2 @@
-34.66.178.120 1;
+35.193.14.140 1;
127.0.0.1 1;
diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list
index ddb64b77b..8268a6f7e 100644
--- a/.dev-tools/test_units/random-bots-for-test-quick.list
+++ b/.dev-tools/test_units/random-bots-for-test-quick.list
@@ -1,10 +1,10 @@
-Apexoo
-BDFetch
-Cloud mapping
-GT::WWW
-Haansoft
-Mediatoolkitbot
-Probethenet
-ProWebWalker
-SuperBot
-Voltron
+Brandwatch
+Bullseye
+Deusu
+GetRight
+Litemage_walker
+Nibbler
+SEOkicks-Robot
+Tracemyfile
+UnisterBot
+zauba.io
diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list
index d395c179b..efda65a15 100644
--- a/.dev-tools/test_units/random-bots-for-test.list
+++ b/.dev-tools/test_units/random-bots-for-test.list
@@ -1,250 +1,250 @@
+360Spider
404checker
+Abonti
+Aboundexbot
+ADmantX
AfD-Verbotsverfahren
AhrefsBot
-AIBOT
-AiHitBot
-Aipbot
+Alexibot
Alligator
-archive.org_bot
+AlphaBot
+Anarchie
+Apexoo
ASPSeek
-Asterias
-BackDoorBot
+Attach
+autoemailspider
backlink-check
-BacklinkCrawler
-BackWeb
+BackStreet
Bandit
+Barkrowler
BatchFTP
-BetaBot
+BBBike
+BDCbot
+BDFetch
Bigfoot
Bitacle
-BLEXBot
-Blow
+Blackboard
+Black Hole
BlowFish
-Boardreader
-Buddy
+Brandprotect
BuiltBotTough
BuiltWith
+BunnySlippers
Calculon
+CATExplorador
CCBot
-Cegbfeieh
-CheeseBot
CheTeam
ChinaClaw
Cliqzbot
-Cloud mapping
cognitiveseo
+com.plumanalytics
+CopyRightCheck
+Craftbot
+crawler4j
crawl.sogou.com
-CSHttp
-DatabaseDriverMysqli
-DBLBot
+DataCha0s
Demon
+Deusu
Devil
Digincore
-DIIbot
-Dirbuster
+DigitalPebble
+Discobot
+Discoverybot
+Dispatch
DomainCrawler
-DomainStatsBot
-Dotbot
-Dragonfly
+DomainSigmaCrawler
+Drip
DSearch
-eCatch
+Ebingbong
+ECCP/1.0
Ecxi
-EirGrabber
-EMail Wolf
+EroCrawler
+evc-batch
Exabot
Express WebPictures
ExtLinksBot
-Extractor
+Extreme Picture Finder
EyeNetIE
+Fimap
+Firefox/7.0
Flunky
-Freeuploader
FrontPage
-Fyrebot
GalaxyBot
GermCrawler
-Getintent
GetRight
-GetWeb
-Gigablast
+Gigabot
G-i-g-a-b-o-t
-GoZilla
-GrabNet
-Grafula
+Go-Ahead-Got-It
GrapeshotCrawler
-GridBot
GT::WWW
Haansoft
-Harvest
-Havij
-HEADMasterSEO
Heritrix
-Hloader
HMView
+HTMLparser
HTTP::Lite
HybridBot
Iblog
-IlseBot
+IDBot
Image Sucker
IndeedBot
InfoTekies
+InterGET
Internet Ninja
IRLbot
Iskanie
IstellaBot
JamesBOT
-Jbrofuzz
JennyBot
-JikeSpider
JOC Web Spider
-Joomla
-Kenjin Spider
-Larbin
-LeechFTP
+JustView
+Kozmosbot
LeechGet
-Libwhisker
-Likse
+Lftp
+LinkextractorPro
LinkpadBot
-LinksManager
-LinqiaRSSBot
+LinkScan
+LinkWalker
+LinqiaMetadataDownloaderBot
Lipperhey Spider
-Lmspider
-LWP::Simple
+Litemage_walker
+Ltx71
+lwp-request
lwp-trivial
Magnet
Mag-Net
-Mail.RU_Bot
+magpie-crawler
+Majestic12
Majestic SEO
Majestic-SEO
+MarkMonitor
MarkWatch
-Mass Downloader
Mata Hari
+meanpathbot
Meanpathbot
MeanPath Bot
-MegaIndex.ru
+Mediatoolkitbot
MFC_Tear_Sample
Microsoft URL Control
-MIDown tool
Mister PiX
-Mr.4x3
+MJ12bot
+Mojeek
+Msrabot
+MS Web Services Client Protocol
+Musobot
+Name Intelligence
Navroad
-Nessus
+NearSite
+Needle
NetAnts
-netEstate NE Crawler
-NetLyzer
-Ninja
-NPbot
+Netcraft
+NetSpider
+Nettrack
+Net Vampire
+Niki-bot
+NimbleCrawler
+Nimbostratus
Offline Explorer
-Openfind
+Openvas
OrangeBot
OrangeSpider
-Pandalytics
+OutfoxBot
+Page Analyzer
+PageGrabber
+PageScorer
+Panscient
Papa Foto
-Pavuk
-pcBrowser
+PeoplePal
PictureFinder
Pimonster
Pi-Monster
PleaseCrawl
-plumanalytics
+Pockey
POE-Component-Client-HTTP
-Probethenet
ProPowerBot
-Pump
-QueryN Metasearch
-RankActive
-RankFlex
+ProWebWalker
+PyCurl
RankingBot
-RankingBot2
-RankurBot
+RealDownload
RebelMouse
ReGet
-Ripper
RocketCrawler
Rogerbot
-s1z.ru
-Scanbot
+ScanAlert
scan.lol
-ScoutJet
Scrapy
SearchmetricsBot
-Semrush
SemrushBot
-SEOkicks-Robot
-SEOlyticsCrawler
-SEOprofiler
+SEOkicks
+Seomoz
seoscanners
-SEOstats
+SeoSiteCheckup
+serpstatbot
+Siphon
Sitebeam
-SiteExplorer
-Siteimprove
-SiteSnagger
SiteSucker
-SMTBot
-Snoopy
-Sogou web spider
-SpaceBison
-Spammen
-sp_auditbot
-SputnikBot
+Sitevigil
+SlySearch
+SocialRankIOBot
+Sociscraper
+sogouspider
+Sottopop
+SpankBot
spyfu
-Steeler
Stripper
-SuperBot
+Sucker
SuperHTTP
Surfbot
-Swiftbot
+T0PHackTeam
T8Abot
tAkeOut
+TeleportPro
Telesphoreo
Telesphorep
-TheNomad
+Thumbor
TightTwatBot
-Toata
-Toweyabot
+Titan
Tracemyfile
Trendiction
-TurnitinBot
-Typhoeus
-UnisterBot
-URLy Warning
+Trendictionbot
+trendiction.com
+TwengaBot
+Upflow
+URLy.Warning
Vacuum
VeriCiteCrawler
VidibleScraper
Virusdie
VoidEYE
-Voil
Wallpapers/3.0
+WallpapersHD
WBSearchBot
-Webalta
WebAuto
Web Auto
-WEBDAV
-WebEnhancer
-Web Enhancer
-WebFetch
+WebBandit
+WebCollage
Web Fetch
Web Fuck
+WebGo IS
WebImageCollector
-WebLeacher
-WebmasterWorldForumBot
-WebPix
+Web Pix
+WebReaper
WebSauger
Web Sauger
+Webshag
Website Quester
-Webster
+WeSEE
Whack
-Whacker
Whatweb
-WISENutbot
-Wonderbot
-Woobot
-Wprecon
+Who.is Bot
+WiseGuys Robot
WPScan
-WWW-Collector-E
WWW-Mechanize
-Xaldon_WebSpider
-xpymep1.exe
-Zade
+WWWOFFLE
+Zauba
+zauba.io
+Zermelo
Zeus
+zgrab
Zitebot
+ZmEu
ZyBorg
diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list
index 15f67b1ff..8090ab627 100644
--- a/.dev-tools/test_units/random-bots-for-whitelist-test.list
+++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list
@@ -1,250 +1,250 @@
80legs
-Aboundexbot
-Acunetix
-ADmantX
+Aboundex
AhrefsBot
-AiHitBot
Aipbot
+Alligator
AlphaBot
-Anarchie
+archive.org_bot
ASPSeek
Asterias
autoemailspider
-BackDoorBot
-Backlink-Ceck
backlink-check
-BacklinkCrawler
+BackStreet
+BackWeb
+Badass
Bandit
-Barkrowler
-BatchFTP
Battleztar Bazinga
BBBike
BDFetch
-BetaBot
Bigfoot
BlackWidow
BLEXBot
Boardreader
-Buddy
-BuiltWith
-CATExplorador
+Bolt
+Brandprotect
+Bullseye
+BuzzSumo
+CCBot
+CheeseBot
CheTeam
+ChinaClaw
Cliqzbot
-coccocbot-web
+Cogentbot
cognitiveseo
+Collector
Copier
CopyRightCheck
-Cosmos
+Copyscape
+Craftbot
+crawler4j
crawl.sogou.com
-Crescent
CrunchBot
CSHttp
-Custo
-DataCha0s
+Curious
+DatabaseDriverMysqli
+DBLBot
+Demon
Digincore
+DigitalPebble
DIIbot
Disco
Discobot
-Discoverybot
+Dispatch
DnyzBot
DomainCrawler
-Dotbot
-Download Wonder
-Dragonfly
Drip
DSearch
DTS Agent
EasyDL
+ECCP/1.0
EirGrabber
-evc-batch
+EroCrawler
+Express WebPictures
+ExtLinksBot
EyeNetIE
Ezooms
+facebookscraper
+FDM
FemtosearchBot
Firefox/7.0
-Flunky
+Fyrebot
GalaxyBot
-GetWeb
-Gigabot
-G-i-g-a-b-o-t
-Go!Zilla
+GermCrawler
+Getintent
+Go-Ahead-Got-It
Grabber
+GrabNet
Grafula
GrapeFX
-GrapeshotCrawler
-Haansoft
-HaosouSpider
HEADMasterSEO
Hloader
-HTMLparser
+HMView
HTTrack
Humanlinks
-HybridBot
-IDBot
-IlseBot
-Indy Library
+Image Fetch
+Image Sucker
+IndeedBot
InfoNaviRobot
instabid
Intelliseek
-Internet Ninja
-InternetSeer
-internetVista monitor
-ips-agent
Iria
-JetCar
-JikeSpider
+Iskanie
+IstellaBot
+JamesBOT
+Jbrofuzz
+Jetty
+JOC Web Spider
Joomla
Jorgee
+JustView
Jyxobot
Kenjin Spider
Lanshanbot
-LeechFTP
-Lftp
Lightspeedsystems
+Likse
LinkextractorPro
-LinkpadBot
LinksManager
LinqiaMetadataDownloaderBot
LinqiaRSSBot
-Lipperhey
-Litemage_walker
Lmspider
-LWP::Simple
-Mag-Net
-magpie-crawler
+Ltx71
+lwp-request
+lwp-trivial
+Magnet
Mail.RU_Bot
Majestic12
-Majestic-SEO
MarkMonitor
-MeanPath Bot
+MarkWatch
+Meanpathbot
+Mediatoolkitbot
mediawords
-Metauri
-Microsoft Data Access
+MFC_Tear_Sample
Microsoft URL Control
-MIDown tool
Mister PiX
-MJ12bot
+Mojeek
+MSFrontPage
MSIECrawler
Msrabot
+muhstik-scan
+Musobot
Name Intelligence
-Navroad
-Nessus
-NetLyzer
-Nettrack
-Nikto
-NimbleCrawler
-Nimbostratus
-Ninja
+NetSpider
+NextGenSearchBot
+Niki-bot
+Nmap
+NPbot
Nutch
-Octopus
+OnCrawl
+Openfind
Openvas
-OpenVAS
-OrangeBot
-OrangeSpider
-OutfoxBot
-PageAnalyzer
-page scorer
-Pandalytics
-Panscient
+Page Analyzer
+PageScorer
Papa Foto
Pavuk
pcBrowser
PeoplePal
-PHPCrawl
-Picsearch
PictureFinder
Pimonster
-Pi-Monster
Pockey
+ProWebWalker
Psbot
PxBroker
-QueryN Metasearch
-RankActive
+Quick-Crawler
+RankFlex
+RankingBot
+Rankivabot
+Reaper
RebelMouse
Recorder
-ReGet
+RedesScrapy
Ripper
-s1z.ru
+RocketCrawler
SalesIntelligent
+SBIder
ScanAlert
+Scanbot
scan.lol
-ScoutJet
Scrapy
-Searchestate
-SearchmetricsBot
-SEOkicks-Robot
+ScreenerBot
+SemrushBot
SEOlyticsCrawler
+Seomoz
seoscanners
-serpstatbot
-sexsearcher
+SeoSiteCheckup
+SEOstats
Siphon
SISTRIX
+Sitebeam
SiteExplorer
-SiteSnagger
-SiteSucker
-SMTBot
+Site Sucker
+SlySearch
+SmartDownload
Snake
Snapbot
+SocialRankIOBot
Sociscraper
sogouspider
-Sogou web spider
-Sottopop
+SpaceBison
+Spammen
SpankBot
Spanner
-Spbot
+sp_auditbot
spyfu
-Sqworm
+Steeler
Sucuri
SurveyBot
+Suzuran
+sysscan
+T0PHackTeam
+T8Abot
+tAkeOut
Teleport
+Telesphoreo
Telesphorep
+The Intraformant
+TheNomad
Thumbor
Titan
-Toata
-Toweyabot
-Trendiction
Trendictionbot
-trendiction.com
trendiction.de
+True_Robot
TurnitinBot
+Twice
+Typhoeus
+UnisterBot
+URLy Warning
Vacuum
Vagabondo
-VB Project
-VeriCiteCrawler
+VCI
+Virusdie
+VoidEYE
+Wallpapers/3.0
WASALive-Bot
-WebAuto
-Web Collage
-WebCopier
-WebEnhancer
-WebFuck
-Web Fuck
-WebGo IS
-WebImageCollector
+WBSearchBot
+Webalta
+WebBandit
+WEBDAV
WebmasterWorldForumBot
-webmeup-crawler
+Web Pix
WebReaper
-Web Sauger
Webshag
-Website Quester
+WebsiteExtractor
Webster
WebSucker
-Web Sucker
WebWhacker
WebZIP
-Whack
-Who.is Bot
+Whatweb
Widow
WinHTTrack
-WiseGuys Robot
WISENutbot
-Wotbox
-WPScan
+Wonderbot
+WWW-Collector-E
+WWW-Mechanize
WWW::Mechanize
x09Mozilla
-Xenu
-xpymep1.exe
+x22Mozilla
Zade
-zauba.io
-zgrab
+Zermelo
+Zitebot
ZmEu
-ZumBot
ZyBorg
diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list
index c4c57ee80..366453892 100644
--- a/.dev-tools/test_units/random-referrers-for-test-quick.list
+++ b/.dev-tools/test_units/random-referrers-for-test-quick.list
@@ -1,10 +1,10 @@
-dvrlists.com
-ege-essay.ru
-ertelecom.ru
-everflora.ru
-nextrent-crimea.ru
-ruex.org.ua
-savememoney.co.za
-smartshoppymac.com
-tizanidinesideeffectsweightloss.blogspot.com
-w-journal.ru
+best-way.men
+cementaresearch.se
+cialischstgerts.com
+filesclub.net
+superstarfloraluk.com
+trustaffs.com
+vremya.eu
+watsonrealtycorp.com
+yuarra.pluto.ro
+zhorapankratov7.blogspot.com
diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list
index ab082b593..975425a0b 100644
--- a/.dev-tools/test_units/random-referrers-for-test.list
+++ b/.dev-tools/test_units/random-referrers-for-test.list
@@ -1,500 +1,500 @@
-0lovespells0.blogspot.com
-101billion.com
123any.com
-123locker.com
-1flag.co.za
+1stat.ru
+2000k.ru
202ch.com
-2345.com
-3dracergames.com
-4senses.co
-57883.net
-7search.com
-8jn.dating-app.ru
-8lufu.com
-a14download.com
-aab94f698f36684c5a852a2ef272e031bb.com
+34.gs
+4kepics.com
+66cpwgln.space
+aarbur.com
+abcdeg.xyz
abcguru.xyz
-absolute-s.ru
-acc.eu.org
-actices.com
-addbin.men
+academiacsmendoza.org
+actionnooz.com
adidas.frwebs.fr
-adpremium.org
-ad-words.ru
-aktobe.xkaz.org
-alborzan.com
-alfa-medosmotr.ru
+adrenalin-stk.ru
+adtiger.tk
+adultgalls.com
+advancedsoftwaresupport.com
+aero2.ru
+agardomains.com
+ahrefs.com
+akama.com
+akuhni.by
+akusherok.ru
+alert-jdh.xyz
alibestsale.com
-all4invest.ru
-alldownload.pw
-allnews.md
-allproblog.com
-allstatesugarbowl.org
-alot.com
-alvtank.se
-amanda-porn.ga
-amateurmatch.com
-amehdaily.com
-ampower.me
-anjalika.co.in
-apibatbrowsecom-a.akamaihd.net
-apidigidocketnet-a.akamaihd.net
-apiluckyleapnet-a.akamaihd.net
-apimountainbikei-a.akamaihd.net
-apispringsmartne-a.akamaihd.net
-apiwoodensealcom-a.akamaihd.net
+all4invest.info
+allcredits.su
+alldezservice.kz
+allwidewallpapers.com
+allwomen.info
+alphacarolinas.com
+alphaforum.ru
+alphahoverboards.com
+androids-store.com
+android-vsem.org
+angigreene.com
+animal-drawings.com
+anniemation.com
+anonymous-redirect.com
appfixing.space
-applicationg29.com
-arate.ru
-arclk.net
-arcteryxstore.online
-aruplighting.com
-arvut.org
+applyneedy.xyz
+apps-analytics.net
+arenda-yeisk.ru
asophoto.com
-asseenontv.ru
+asrvvv-a.akamaihd.net
+atlasvkusov.ru
audiofree.ru
-ausmepa.org.au
-autonew.biz
-autotrends.today
-aversis.net
-ayanaspa.com
-azartmix.com
-azbukadiets.ru
-bagcionderlab.com
-bannerads.de
-barbourjackets.online
-barnfurnituremart.com
-bayareaaeroclub.org
-beachfix.co
-beauty-clinic.ru
-belinka.com.ua
-bestcurs.org
-bestempresas.es
-besthomemadepornsites.com
-bezsporno.ru
-bioinnovate.co
-bizlist.com.de
-blobar.org
-bloggedporn.com
-blogporn.in
-board.f00d.de
-bochemit.com.ua
-bolezniorganov.ru
-boltalko.xyz
-bonkers.name
-bookmark4you.com.biz
-brewdom.ru
-btnativenav.com
-buigas.00it.com
-buqyxa.rincian.info
-burkesales.com
-buynorxx.com
-bwlx.prepedu.cn
-callcafe.info
-canoncdriverq3.pen.io
-carmuffler.net
-cartierreplica.win
-catalogs-parts.com
-cbcseward.com
-celejihad.info
-cete.ru
-cheapsergiorossi.online
-cheap-trusted-backlinks.com
-cherrypointplace.ca
-children-learningreading.info
-chinaelitecheapjerseys.com
-cicaki.net
-classiquebijoux.ru
-clean-virus-mac.com
-clevernt.com
-clickzzs.nl
-clothesforcash.com
-cobaltpro.ru
-communisave.co.za
-compliance-andrew.xyz
-compliance-barak.top
-compliance-elena.top
-compliance-jessica.top
-computernetworksonline.com
-com-secure.download
-connectionstrenth.com
-construmac.com.mx
-cookie-law-enforcement-ff.xyz
-coolyarddecorations.com
-cornomase.win
-cowlmash.com
-craftinsta.ru
-culturevie.info
-cv.wallhade.co
-dangphoto.trade
+autoua.top
+availit.weebly.com
+avon-ukraine.com
+azadnegar.com
+balans.shahterworld.org
+bankcrediti.ru
+bannerpower.com
+battleforupdating.site
+belstaffstore.online
+bestadbid.com
+bet365.com
+bitporno.sx
+blacktwhite.com
+blagovest-med.ru
+boazpower.com
+bongiornos.info
+bonvillan.com
+brandov.ru
+bumascloset.com
+bupropion-sr-150-mg.us
+candytiner.com
+capture-room.com
+carapics.com
+cartierbracelet.xyz
+cartierloveringreplica.xyz
+cbb1smartlist12.click
+cb.iphantom.com
+cdncash.net
+check-host.net
+checktext.org
+chimiver.info
+chinajerseyswholesalecoupons.com
+chorus.terakeet.com
+christianlouboutinreplica.win
+cialisovercounteratwalmartusa.com
+cindymatches.com
+cinemaenergy-hd.ru
+classicakuhni.ru
+cleansearch.net
+clickintext.net
+clickprotects.com
+club-lukojl.ru
+cmhomestayagency.com
+cometorussia.net
+compliance-alexa.xyz
+compliance-brian.xyz
+compliance-elena.xyz
+compucelunlock.net
+concealthyself.com
+coolbar.pro
+coolingoods.com
+coprofam.org
+counterbot.com
+crd.clan.su
+cutalltheshit.com
+cvety24.by
+cyclobenzaprinevstizanidine.blogspot.com
+czeshop.info
+data-mining.tk
+date-withme.com
dating-app.ru
-dealitright.click
-dekoration.us
-delitime.info
-dellalimov.com
-dengi-pod-zalog-nedvizhimosti.ru
-detalizaciya-tut.biz
-diesel-parts28.ru
-digest-project.ru
-dinosaurus.site
-dirtpics.pw
-divisioncore.com
-dms-sw.ru
-domaincdn.xyz
-dood.live
-dostavimvdom.ru
-dptaughtme.com
-dyt.net
-easync.io
-ecommerce-seo.com
-ednorxmed.com
-ege-russian.ru
-electrik-avenue.com
-elektrischezigarettekaufen2.cowblog.fr
+dating-time-now.com
+datsun-do.com
+dealighted.com
+dear-diary.ru
+defenderxtactical.com
+delfin-aqua.com.ua
+descargar-musica-gratis.net
+dfiles.me
+dfwu1013.info
+distver.ru
+div.as
+djstools.com
+doctormakarova.ru
+dogclothing.store
+dollartree.info
+downloads-whatsapp.com
+drivotracker.com
+duplicashapp.com
+educontest.net
+eduinfosite.com
+eldiariodeguadalajara.com
+elektrischezigaretten1.blogse.nl
+elektronischezigarettekaufen1.beeplog.com
elektrozigarette1.dreamwidth.org
-elitepcgames.com
-emerson-rus.ru
-en.altezza.travel
-estelight.ru
-evaashop.ru
-exoclick.com
-expresstoplivo.ru
-ezigarettenkaufen1.deviantart.com
-fakehandbags.xyz
-fanoboi.com
-fatmaelgarny.com
-faz99.com
-fickenbumsen.net
-fickluder69.com
-finemanteam.com
-flashbannernow.com
-flex4launch.ru
-fordsonmajbor.cf
-formulaantiuban.com
-formularz-konkurs.tk
-forsex.info
-fortunejack.com
-forumprofi.de
-fotzen-ficken.com
-foxtechfpv.com
-fullfileaccess.com
-funcrushgames.com
-fxtips.ru
-fym.com.ru
-game300.ru
-gemgrab-a.akamaihd.net
-getlamborghini.ga
-gidonline.one
+elektrozigarettekaufen1.devhub.com
+elektrozigarettekaufen2.blogse.nl
+eluxer.net
+eng-lyrics.com
+eshop.md
+evening-dating-club.info
+exchanges-bet.com
+exhibitionplus.eu
+expertnaya-ocenka.ru
+eyessurgery.ru
+ezigarettekaufen2.mpbloggar.se
+famix.xyz
+farm26.ru
+fashion-mk.net
+ferrotodo.com
+figuringmoneyout.com
+filesclub.net
+filesvine.com
+floridamhca.org
+foodcrafts.website
+formseo.com
+forpackningsutveckling.se
+foto-basa.com
+freecamdollars.com
+freeseedsonline.com
+free-video-tool.com
+freewareseek.com
+frequiry.com
+froggytube.com
+fsalas.com
+fx-brokers-review.com
+gallerylisting.com
+gambarxkata.co
+game-mmorpg.net
+gameprimary.com
+gcup.ru
+gembird.com
+getoutofdebtfree.org
+gigapeta.com
gilbertbanda.net
-glogow.pl
-gok-kasten.net
-goldadpremium.com
-goldandcard.ru
-gombita.info
-goosefishpost.bid
-goroda-vsego-mira.ru
-gov.yanao.ru
-gsasearchenginerankerdiscount.com
-guiadeserraazul.com
-gwagka.com
-hahashka.ru
+go2jump.org
+golden-catalog.pro
+gotorussia.com
+gotwebsite1.com
+greatdealshop.com
+greatfind-a.akamaihd.net
+guod.me
hamilton.ca
-hazardky.net
-hd-film.pl
+hao123.com
+harmonyglen.us
+hdapp1008-a.akamaihd.net
hdpixion.com
-himazin.info
-histats.com
-hit-men.men
+hdwalls.xyz
+healgastro.com
+health-medical-portal.info
+heatpower.ru
+heliko.no
+helvetia.com.ua
+hermesbirkinhandbagoutlets.com
+highstairs-a.akamaihd.net
+hildinghr.se
+himgaws.pw
+hkdiiohi.skyrock.com
+home.myplaycity.com
hosting-tracker.com
-host-protection.com
hotblog.top
-houseofrose.com
-hoverboard360.at
-hspline.com
-htmlcorner.com
-hvd-store.com
-hyiphunter.org
-iboss.com
-ikearugs.xyz
-iklysha.ml
-ilikevitaly.com
-imabase.com
+hotxnights.info
+houdom.net
+hptwaakw.blog.fc2.com
+huhn.altervista.org
+hyipmanager.in
+hystersister.com
+ictizanidinehcl4mg.blogspot.com
+ifmo.ru
+ikritikimou.gr
+iloveitaly.ru
+imagefinder.site
+imagui.eu
inclk.com
-infobabki.ru
-intellekt21.ru
-ip-guide.com
+infoupdate.org
+inmate-locator.us
+intelhdgraphicsgtdrive6w.metroblog.com
+iqoption.com
+istanbulit.com
+istock-mebel.ru
itsdp3.com
-ivanstroi.ru
-javxxx18.com
-jbl-charge.info
-jerseyschinabizwholesale.us
-jikoman.info
-jobmarket.com.ua
-jurajskie.info
-kadashihotel.com
-kaliningrad.zrus.org
-karachev-city.ru
-kazan.zrus.org
-kaz.kz
-keyword-suggestions.com
-kinotorka.ru
-kletkimehan.ru
-kongoultry.net
-kongruan.com
-kovrenok.ru
-lacave.ntic.fr
+jacago.com
+jackpotchances.com
+japan-bearings.ru
+japfm.com
+jerseysbizwholesalecheap.com
+jerseywholesalechinabiz.com
+job.icivil.ir
+j-times.ru
+kamagragelusa.net
+kambasoft.com
+keywordblocks.com
+keywordglobal.co.za
+keywordsking.com
+keywordsuggests.com
+kimcurlrvsms.com
+kinobaks.com
+kinohall.ru
+kinopolet.net
+kinostorm.net
+kiskinhouse.com
+knogg.net
+koleso24.com.ua
+koopilka.com
+kovesszucs.atw.hu
+kraljeva-sutjeska.com
+krasivye-devushki.net
+kredit-pod-zalog-krasnodar.ru
+kursy-ege.ru
+kurwa.win
+kustanay.kz
+labelwater.se
+labplus.ru
+lafourmiliaire.com
lanasshop.ru
-landscaping.center
-lashstudia.ru
-lernur.net
-lescinq.com
-letmacworkfaster.world
-letslowbefast.today
-lifebyleese.com
-linkbuddies.com
-liran-locks.com
-littleberry.ru
-lyngdalhudterapi.no
-lyrics.home-task.com
-machicon-akihabara.info
-mackeeper-land-672695126.us-east-1.elb.amazonaws.com
-mahnwachen-helfen.info
-mailemedicinals.com
-mansiondelrio.co
+lankarns.com
+law-check-nine.xyz
+laxdrills.com
+lechtaczka.net
+lenvred.org
+lineshops.biz
+livingcanarias.com
+llastbuy.ru
+lottospring.com
+macfix.life
+machicon-ueno.info
+macresource.co.uk
+mail.allnews24.in
+manipulyator-peterburg.ru
mararoom.ru
-martlinker.com
-mastershef.club
-mebelcomplekt.ru
-medispainstitute.com.au
-medkletki.ru
-med-zdorovie.com.ua
-metallosajding.ru
-mezaruk.info
-micasainvest.com
-michaelkorssaleoutletonline.net
+marcowebonyodziezowe.pl
+marinetraffic.com
+marktforschung-stuttgart.com
+maslenka.kz
+mb140.ru
+meccadumps.net
+medicinacom.ru
+medtherapy.ru
+megakino.net
+meltwater.com
+merryhouse.co.uk
migente.com
-miniads.ca
-mlvc4zzw.space
-mockupui.com
-mokrayakiska.com
-moncleroutletonline.pw
-monthlywinners.com
-morefastermac.trade
-my-big-family.com
-myblogregistercm.tk
-mydirtystuff.com
-myliveblog.ru
-nac-bearings.ru
-naperehresti.info
-nardulan.com
-nate.com
-national-today-winning-winner.club
-naturalbreakthroughsresearch.com
-neobux-bg.info
-netcheckcdn.xyz
-newsperuse.com
-newstraveller.ru
-nextbackgroundcheck.gq
-next-dentists.tk
-nhl17coins.exblog.jp
-niki-mlt.ru
-niroo.info
-nudo.ca
-oklogistic.ru
-online7777.com
-online-templatestore.com
-o-o-8-o-o.com
-openmultipleurl.com
-orhonit.com
-osnova3.ru
-ossmalta.com
-ownshop.win
-panamaforbeginners.com
-pandarastore.top
-paparazzistudios.com.au
-pewit.pw
-photo.houseofgaga.ru
-physfunc.ru
-pijoto.net
-pirateday.ru
-pistonclasico.com
-pizdeishn.com
-platezhka.net
+milblueprint.com
+mirzonru.net
+missis.top
+mlf.hordo.win
+mobot.site
+mojowhois.com
+monclercoats.xyz
+moscow.online-podarki.com
+motherboard.vice.com
+mouselink.co
+mrbitsandbytes.com
+mriyadh.com
+musicspire.online
+myfreemp3.eu
+mymobilemoneypages.com
+nagdak.ru
+najaden.no
+nakozhe.com
+narkologiya-belgorod.ru
+narkologiya-orel.ru
+naturalshair.site
+naval.jislaaik.com
+netoil.no
+net-profits.xyz
+nfljerseys.online
+nlfjjunb5.ru
+nullrefer.com
+o00.in
+obnal.org
+offers.bycontext.com
+office2web.com
+offtime.ru
+okayimage.com
+olgacvetmet.com
+olvanto.ru
+online-hit.info
+optitrade24.com
+ororodnik.goodbb.ru
+ostroike.org
+ourtherapy.ru
+outclicks.net
+paradontozanet.ru
+partnerads.men
+pay2me.pl
+pcboa.se
+pcimforum.com
+pdamods.ru
+photochki.com
+photo-clip.ru
play-mp3.com
-plohaya-kreditnaya-istoriya.ru
-pl-vouchers.com
-pobeiranie.pl
-podshipniki-ntn.ru
-pojdelo.weebly.com
-popup.matchmaker.com
-pornodojd.ru
-pornoslive.net
-pornotubexxx.name
-printdirectforless.com
-priscilarodrigues.com.br
-privatefx-in.ru
-prizeestates.cricket
-profit-opportunity.com
-proxyradar.com
-psoriasis-file.trade
-pulse33.ru
-purplestats.com
-puzzleweb.ru
-q-moto.ru
-qpypcx.com
-quangcaons.com
-queerspace.com
-rabot.host.sk
-radiodigital.co
-rank-checker.online
-ranksignals.com
-rasteniya-vs-zombi.ru
-redirectme.net
-refererx.com
-reklama-i-rabota.ru
-research.ifmo.ru
-rff-cfal.info
-rfid-locker.co
+pl.id-forex.com
+pliks.pl
+poem-paying.gq
+pomoc-drogowa.cba.pl
+pornoelita.info
+pornogig.com
+pornophoto.xyz
+privacyassistant.net
+privatamateure.com
+privatefx.all4invest.info
+privatov-zapisi.ru
+prizesbook.online
+prod2016.com
+produkto.net
+prodvigator.ua
+prom23.ru
+prostitutki-novgoroda.org
+puzo2arbuza.ru
+qor360.com
+quelle.ru
+razyboard.com
+r-control.ru
+realmonte.net
+rebelmouse.com
+redhotfreebies.co.uk
+registrationdomainsite.com
+reining.lovasszovetseg.hu
+rewardpoll.com
+reward-survey.net
+rfd-split.hr
+riders.ro
+rightenergysolutions.com.au
+rniaeba.ga
roll123.com
-roofers.org.uk
-rostov.xrus.org
-rusenvironmental.net
-sack.net
-saecsa.co
-sashagreyblog.ga
-savememoney.co.za
-search-error.com
-searchimage.co
-seocdvig.ru
-seo-prof1.xyz
-sergiorossistore.online
-serialsway.ucoz.ru
-serpstat.com
+runtnc.net
+rvzr-a.akamaihd.net
+sabizonline.com
+sadaholding.com
+samara.rosfirm.ru
+sayyoethe.blogspot.co.za
+scanner-don.top
+scat.porn
+scenicmissouri.us
+searchtooknow-a.akamaihd.net
+seoanalyses.com
+seocheckupx.com
+seoholding.com
+seoprofiler.com
+seo-traffic-ranking.info
+servicecenter.co.ua
sexcamamateurchat.com
-sexs-foto.com
-sexs-foto.top
-shodanhq.com
-shoesonlinebuy.xyz
-shoplvlv.us
+sex-dating.co
+sexreliz.com
+sextracker.de
+sfd-chess.ru
+shop.acim.org
shtaketniki.kz
-shymkent.xkaz.org
-signal03.ru
-signoredom.com
-site-auditor.online
-sitebeam.net
-site-speed-check.site
-s-iwantyou.com
-sladkoevideo.com
-slavkokacunko.de
-songplanet.ru
-sotkal.lark.ru
-special-porn.com
-stefanbakosab.se
-stjamesschool.info
-stockquotes.wooeb.com
-strana-solnca.ru
-suralink.com
-surflinksmedical.com
-svnuppsalaorebro.se
-tagil.zrus.org
+sientalyric.co
+sildenafilcitratemed.com
+simul.co
+sisiynas.ru
+sispe.com.br
+site3.free-share-buttons.com
+siteimprove.com
+skladvaz.ru
+slomm.ru
+social-buttons.com
+social-search.me
+songoo.wz.cz
+spabali.org
+specialfinanceoffers.com
+splendorsearch-a.akamaihd.net
+sptslmtrafms.com
+spymac.net
+stanthonyscatholicchurch.org
+stats-collector.org
+steame.ru
+stickers-market.ru
+strag-invest.ru
+streetfire.net
+strfls.com
+studiofaca.com
+summerlinhomes411.info
+svtrd.com
+swsociety.se
tandvardshuset.net
-tattooreligion.ru
-t-bygg.com
-tds-advert005.info
-technopellet.gr
-tecspb.ru
-telefonsexi.com
-theendivechronicles.com
-thegolfclub.info
-theporndude.com
-timhost.ru
-tizanidineandgabapentin.blogspot.com
-tizanidineduringpregnancy.blogspot.com
-tizanidinehcl2mgtablet.blogspot.com
-tizanidinehcl4mgisitanarcotic.blogspot.com
-tizanidineshowupondrugtest.blogspot.com
-tizanidinestreetvalue.blogspot.com
-toondinsey.com
-top250movies.ru
-torrentdownloadhub.com
-tozup.com
-tpu.ru
-track2.shop
-track-rankings.online
-trafaret74.ru
+telsis.com
+templates.franklinfire.co
+tetracsaudi.com
+tfxiq.com
+thecounter.com
+thegameriders.com
+theguardlan.com
+the-torrent-tracker.blogspot.com
+the-usa-games.blogspot.com
+thfox.com
+thiegs.reco.ws
+thruport.com
+titan-cloud.life
+tizanidinedosage.blogspot.com
+tizanidinehcl4mgtab.blogspot.com
+tizanidineonline.blogspot.com
+tizanidinerestlesslegsyndrome.blogspot.com
+tizanidinesideeffects.blogspot.com
+tonerbox.kz
+top10-online-games.com
+topbestgames.com
+top-karkas.ru
+topmira.com
+topsiteminecraft.com
+topvidos.ru
+totu.us
+tradgardspartner.se
+traffic2money.com
trafficfactory.biz
-truck-addzilla.life
-truemfilelj.gq
-tvorozhnaja-zapekanka-recept.ru
-twitlinks.com
-twittrading.com
-twojebook.pl
-txxx.com
-typer.one
-tyumen.xrus.org
-ua-company.ru
-ukkelberg.no
-unmaroll.ya.ru
-uptime-alpha.net
-urlcut.ru
+traffictrade.life
+trahvid.com
+treasuretrack-a.akamaihd.net
+tv-spoty.info
+typimga.pw
+tytoona.com
+uk-zheu20.ru
+unitygame3d.com
+uprour.com
+upupa.net
uroki.net
-usacasino.com
-usswrite.com
-vacuumscleaner.com
-validccseller.com
-vapsy.com
+usbggettwku.ga
+vandrie-ict.nl
+varikoz24.com
varikozdok.ru
-vavilone.com
-vegascosmetics.ru
-versaut.xxx-cam.webcam
-veselokloun.ru
-via-energy-cumpara.com
-viandpet.com
-viddyoze.com
-videochat.mx
+ventelnos.com
+veopornogratis.xxx
+vgoloveboli.net
+video-hollywood.ru
+videospornogratisx.net
video-woman.com
-vintontech.info
-vip-dom.in
-virtuagirl.com
-visionwell.com.cn
-vitoriacabos.com
-vizitki.net
+vietimgy.pw
vkgaleria.com
-vkontarkte.com
-vozbujdenie.com
-vzglyadriv.kg
-wallpapersdesk.info
-wallpapersist.com
-webalan.ru
-website-audit.com.ua
-websites-reviews.com
-weburlopener.com
-westvilletowingservices.co.za
-whatsappdownload10.com
-wrona.it
-wstroika.ru
-www.get-free-traffic-now.com
-wzgyyq.com
-xaijo.com
-xerox-douglas.cf
-xfluro.com
+voloo.ru
+voluumtracker1.com
+webpromotion.ae
+webs.com
+weprik.ru
+wesharepics.site
+westen-z.life
+whatzmyip.net
+wineitudes.wordpress.com
+wladimirpayen.com
+wnhjavlhezp.gq
+woman-h.ru
+wonderfulflowers.biz
+worldlovers.ru
+writersgroup580.web.fc2.com
+wttavern.com
xgftnlrt.bloger.index.hr
-xmladserver.com
-xn--d1aifoe0a9a.top
-xnxx-n.com
-yogamatsexpert.com
-yoshkarola.zrus.org
-youjizz.com
-youthreaders.com
-youtuhe.com
-yubikk.info
-yunque.pluto.ro
-zambini.ru
-zeg-distribution.com
-zetgie.com.pl
-zigarettenonl.canalblog.com
-zionstar.net
-zog.link
-zone-kev717.info
-zoomovies.org
-zrus.org
+x-musics.com
+xn----7sbbagbq7bd5aheftfllo4m.xn--p1ai
+xn----7sbbahaq9bb5afgiqfliv4m.xn--p1ai
+xn--80aafb2a.xn--p1ai
+xn--80abgj3a5acid6ghs.top
+xwatt.ru
+yes-com.com
+youdesigner.kz
+yourmovies.pl
+ytmnd.com
+yur-p.ru
+zaapplesales.blogspot.com
+zajm-zalog-krasnodar.ru
+zdesformula.ru
+zeleznobeton.ru
+zscalerone.net
diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list
index 077df5b49..38d5ef26b 100644
--- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list
+++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list
@@ -1,250 +1,250 @@
-a14download.com
-absolutelyanalog.com
-adamoads.com
-admitad.com
-ads.gold
-agroeconom.kz
-allbizne.co.ua
-amazon-seo-service.com
-amung.us
-anaksma.info
-animali.deagostinipassion.it
-anniemation.com
-anonymizeme.pro
-apartmentbay.ru
-apioutoboxnet-a.akamaihd.net
-arabsexxxtube.com
-asacopaco.tk
-asia-forum.ru
-asrvrep-a.akamaihd.net
-atelielembrancaqueencanta.com.br
-australia-opening-times.com
-backgroundpictures.net
-baersaratov.ru
-balans.shahterworld.org
-baltgem.com
-beatifullandscape.co
-bestadbid.com
-bkgr.se
-bocoarchives.org
-bonzbuddy.com
-bookmark4you.com
-brendbutik.ru
-bvlgariwallet.xyz
-cdnanalytics.xyz
-chuckguilford.com
-clickaider.com
-clmforexeu.com
-compliance-mark.top
-compliance-walt.top
-compliance-willy.top
-custom-electric-guitar.com
-dangphoto.trade
-db.speedup.gdn
-dealwifi.com
-devochki.top
-dfiles.me
-dlya-android.org
-doeco.ru
-domoysshop.ru
-drev.biz
-drpornogratisx.xxx
-dvd-famille.com
-dverimegapolis.ru
-e-biznes.info
-engines-usa.com
-en.home-task.com
-eshop.md
-evening-dating-club.info
-exchangeit.gq
-ezigarettekaufen2.mpbloggar.se
-ezrvrentals.com
-familienzahnaerzte.com
-family1st.ca
-familyphysician.ru
-fastfixing.tech
-fatmaelgarny.com
-fettefrauen.net
-fickkontakte.org
-financeloan.us
-financetip.eu
-flavors.me
-flirt4free.com
-flytourisme.org
-fotoxxxru.com
-foxinsocks.ru
-freelinkbuilding.website.tk
-gateway.zscalerone.net
-generic-pills-online.com
-getaclueamerica.com
-gifspics.com
-gktt.ru
-gratuitxblcodes.com
-h2monline.com
-hatedriveapart.com
-haveinc.xyz
-hermesreplica.pw
-homeandhealth.ru
-hscsscotland.com
-hvd-store.com
-incep.imagine-ex.co
-indianmedicaltourismshop.com
-internet-apteka.ru
-intim-uslugi.info
-invitefashion.com
-jav-fetish.site
-jaxcube.info
-jbl-charge.info
-jolic2.com
-jovencitas.gratis
-jpcycles.com
-keywesthideaways.co
-kollekcioner.ru
-komputers-best.ru
-kosynka-games.ru
-kozhniebolezni.com
+214jaluwobapef.cf
+4rent.ru
+5-steps-to-start-business.com
+5ws.dating-app.ru
+7a2rnuey1tw9ar.ru
+7xc4n.com
+aa81bf391151884adfa3dd677e41f94be1.com
+aa8780bb28a1de4eb5bff33c28a218a930.com
+adconscious.com
+adelly.bg
+adserver-e7.com
+adsland.men
+ads-seo.men
+aosheng-tech.com
+arkkivoltti.net
+as5000.com
+asrv-a.akamaihd.net
+asrvvv-a.akamaihd.net
+autoblogger24.info
+avtocredit-legko.ru
+azadnegar.com
+bannerswap.com
+bashtime.ru
+bestcoin.cc
+bestserials.com
+blackcurranthumidifiers.site
+blog.axant.it
+blog.yam.com
+bonkers.name
+buy-loft.ru
+buynorxx.com
+buytizanidineonlinenoprescription.blogspot.com
+catalogs-parts.com
+cdncash.org
+cdnnetwok.xyz
+cheatcode-lita12.rhcloud.com
+choice-credit.ru
+christianlouboutinshoes.xyz
+ci.ua
+clarithromycin500mg.com
+claytransformations.info
+compliance-alex.top
+connectionstrenth.com
+cornerstone-countertops.com
+cornomase.win
+crystalslot.com
+davebestdeals.com
+deda-moroza-zakaz.ru
+dekorkeramik.ru
+delfin-aqua.com.ua
+demian.kz
+destinationrealestate.com
+detalizaciya-tut.biz
+dgfitness.co
+digitalassetmanagement.site
+dnepropetrovsk.mistr-x.org
+dneprsvet.com.ua
+domain-tracker.com
+dotnetdotcom.org
+eandsgallery.com
+earl-brown.info
+efkt.jp
+erotik0049.com
+eu-cookie-law.info
+europages.com.ru
+everypony.ru
+expertblog.info
+extremepornos.net
+ezigarettekaufen1.hpage.com
+ezigarettenshop2.myblog.de
+f07.de
+favoritemoney.ru
+femdom.twiclub.in
+ferrotodo.com
+fickenprivat.info
+firma-legion.ru
+flprog.com
+franklinfire.co
+freecamdollars.com
+frivgame250.com
+funnymama.com
+g33.org
+gcup.ru
+generalporn.org
+generousdeal-a.akamaihd.net
+golaya.pw
+googlepositions.com
+gox.com.ua
+graphid.com
+gratis-sexkontakte.com
+gwebtools.com.br
+hamilton.ca
+hd-filmy.net
+hdhc.ru
+hdimagegallery.net
+healmytrauma.info
+helvetia.com.ua
+hotgirlhdwallpaper.com
+hoverboard360.at
+howlongdoestizanidinestayinyoursystem.blogspot.com
+ilmen.net
+imgata.com
+inbabes.sexushost.com
+interferencer.ru
+investzalog.ru
+iqoption.pro
+isotoner.com
+istizanidineacontrolledsubstance.blogspot.com
+istizanidinelikexanax.blogspot.com
+janavibekken.no
+jasonpartington.com
+je7.us
+jikoman.info
+jyvopys.com
+keyhantercume.com
+keywordhouse.com
+kinopolet.net
+koleso24.com.ua
+konoplisemena.com
+kountrylife.com
kredit-blog.ru
-labplus.ru
-larchik.net
-ledpolice.ru
-light.ifmo.ru
-linerdrilling.com
-littlecity.ch
-liveu.infoteka.hu
-lolnex.us
-longlifelomilomi.info
-lotto6888.com
-ltvperf.com
-macresource.co.uk
-magento-crew.net
-mainlinehobby.net
-manve.info
-maximilitary.ru
-medi-fitt.hu
-meendo-free-traffic.ga
-meget.co.za
+la-fa.ru
+larger.io
+live-cam6.info
+lsp-awak-perikanan.com
+medicovi.com
+medmajor.ru
+medosmotr-ufa.ru
+memberty.com
mensandals.xyz
-metascephe.com
-mir-betting.ru
-mirtorrent.net
-missclub.info
-mobstarr.com
-monarhs.info
-most.gov.iq
-moviezbonkerssk.cf
-musicspire.online
-mxgetcode.com
-mydownloadengine.com
-myfreetutorials.com
-myftpupload.com
-mygameplus.com
-naj-filmy24.pl
-new-post.tk
-nhl09.ru
-nmrk.ru
-o333o.com
-oakleyglassesonline.us
-oakridgemo.com
-oda.as
-offer.wpsecurity.website
-onlinadverts.com
-oralsexfilme.net
-orion-v.com
-outclicks.net
-oz-offers.com
-partner-host.men
-partybunny.ru
-pdns.cz
-perfectpracticeweb.com
-pijoto.net
-pixell.club
-pizdeishn.com
+mixx.com
+mobile.ok.ru
+moje-recenze.cz
+moncleroutletonline.xyz
+mosdverka.ru
+mowser.com
+mrinsidesales.com
+mulberryoutletonlineeu.com
+musicdaddy.net
+mybinaryoptionsrobot.com
+myseoconsultant.com
+next-dentists.tk
+nikitabuch.com
+nrv.co.za
+olvanto.ru
+oneminutesite.it
+onstrapon.purplesphere.in
+orion-code-access.net
+pattersonsweb.com
+perosan.com
+perso.wanadoo.es
+photo.houseofgaga.ru
+piluli.info
+pio.polytopesexempt.com
+piski.top
plastgranar.nu
-porno-dojki.net
-pornolook.net
-porno-play.net
-porno-raskazy.ru
-privat-girl.net
-prodess.ru
-produkto.net
-profitsport.club
-prohoster.info
-pr-ten.de
-puzo2arbuza.ru
-rainbowice.ru
-recipedays.com
-redirlock.com
-rethinkwasteni.info
-roznica.com.ua
-ruex.org.ua
-russia-today-video.ru
-salmonfishingsacramentoriver.com
-salut-camp.ru
-sbetodiodnye-lampy.ru
-sbwealthsolutions.ca
-scanner-alex.top
-schoolfiles.net
-seoexperimenty.ru
-sexkontakteao.info
-shemale-sex.net
+playlott.com
+plohaya-kreditnaya-istoriya.ru
+pomoc-drogowa.cba.pl
+pontiacsolstice.info
+popup.matchmaker.com
+pornofeuer.com
+portadd.men
+praisong.net
+pravoholding.ru
+preparevideosafesystem4unow.site
+promgirldresses.xyz
+prpops.com
+pulse33.ru
+push-ad.com
+queerspace.com
+raavidesigns.com
+ragecash.com
+recinziireale.com
+redirectingat.com
+remont-komputerov-notebook.ru
+replicaclub.ru
+ring4rhino.com
+ringtonepartner.com
+rootandroid.org
+rossanasaavedra.net
+royal-investments.net
+runtnc.net
+rus-pornuha.com
+saitevpatorii.com
+savetubevideo.com
+scanner-jess.top
+scanner-willy.top
+scripted.com
+searchmywindow-a.akamaihd.net
+seccioncontrabajo.com
+seksvideoonlain.com
+seo-2-0.com
+seorankinglinks.us
+seo-smm.kz
+sexyteens.hol.es
shopcheermakeup.info
-shop.xz618.com
+shopperifymac.com
+shymkent.xkaz.org
sicfor.bcu.cc
-sitevaluation.com
-s-iwantyou.com
-soaksoak.ru
-social-s-ggg.xyz
-soft-program.com
-sonnikforme.ru
-sortthemesitesby.com
-spitfiremusic.com
-sprttrack.com
-spywarebegone.com
-sribno.net
-stair.registrydomainservices.com
-strana-krasoty.ru
-superlist.biz
-supermesta.ru
-suture.co
-thenews-today.info
-ticketsys.inetwd.com
-tizanidine4mgstreetvalue.blogspot.com
-tizanidineforopiatewithdrawal.blogspot.com
-tizanidinepills.blogspot.com
-topads.men
-torrent-newgames.com
-trafaret74.ru
-turizmus.us
-twinderbella.com
-uggbootsoutletsale.us
-unionmarkt.de
-urlcut.ru
-ussearche.cf
-valmetrundan.se
-vepad.com
-videotuber.ru
-vladimir.zrus.org
-vod.com.ua
-vrotike.ru
-watchmyfb.pl
-weddingdresses.xyz
-wfb.hatedriveapart.com
-wildnatureimages.com
-word-vorlagen.net
-wowas31.ucoz.ru
-www.kabbalah-red-bracelets.com
-xn--lifehacer-1rb.com
-xportvusbdriver8i.snack.ws
-xpresscare.ru
-xxx-treker.ru
-ynymnwbm.bloger.index.hr
-yopoint.in
-yourserverisdown.com
-yousense.info
-zaim-pod-zalog-krasnodar.ru
-zawyna.ua
-zazagames.org
-zlatnajesen.com
-zona-aqua.ru
-zophim.me
-zverokruh-shop.cz
+sildenafilcitratemed.com
+silktide.com
+sisi-go.ru
+slonechka.ru
+softomix.com
+softtor.com
+spy-app.info
+spywarenuker.com
+taihouse.ru
+tds-advert002.info
+technopellet.gr
+thecoolimages.net
+thesmartsearch.net
+thetardistimes.ovh
+tizanidineinfo.blogspot.com
+topclickguru.com
+torrents.cd
+trafficnetzwerk.de
+trkur.com
+trucri.me
+umaseh.com
+umg-stroy.ru
+us-america.ru
+valkiria-tk.ru
+valoresito.com
+vesnatehno.ru
+videokrik.net
+vielporno.net
+vilingstore.net
+vitanail.ru
+vkgaleria.com
+vksaver-all.ru
+voditeltrezviy.ru
+voluumtrk.com
+webmonetizer.net
+whos.amung.us
+woman-orgasm.ru
+wurr.voila.net
+xgames-04.com
+xjrul.com
+x-true.info
+xvideosporn.biz
+yellowfootprints.com
+youjizz.com
+zambini.ru
+zeroredirect11.com
+znakomstvaonlain.ru
+zoodrawings.com
+zryydi.com
+zscalertwo.net
diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt
index f371d9e5f..3b5fea028 100644
--- a/.dev-tools/test_units/ratelimittest-beta.txt
+++ b/.dev-tools/test_units/ratelimittest-beta.txt
@@ -1,3 +1,31 @@
+
+503 Service Temporarily Unavailable
+
+503 Service Temporarily Unavailable
+
nginx/1.14.2
+
+
+
+503 Service Temporarily Unavailable
+
+503 Service Temporarily Unavailable
+
nginx/1.14.2
+
+
+
+503 Service Temporarily Unavailable
+
+503 Service Temporarily Unavailable
+
nginx/1.14.2
+
+
+
+503 Service Temporarily Unavailable
+
+503 Service Temporarily Unavailable
+
nginx/1.14.2
+
+
@@ -8,35 +36,7 @@
Welcome to the NginX Ultimate Bad Bot Blocker
This is merely used for testing !!!
-
-503 Service Temporarily Unavailable
-
-503 Service Temporarily Unavailable
-
nginx/1.14.2
-
-
-
-503 Service Temporarily Unavailable
-
-503 Service Temporarily Unavailable
-
nginx/1.14.2
-
-
-
-503 Service Temporarily Unavailable
-
-503 Service Temporarily Unavailable
-
nginx/1.14.2
-
-
-
-503 Service Temporarily Unavailable
-
-503 Service Temporarily Unavailable
-
nginx/1.14.2
-
-
-
+