diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
index 2bbce7148..6cadc7f55 100644
--- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1746
-### Updated: Thu Jul 4 09:09:17 SAST 2019
+### Version: V3.2019.07.1747
+### Updated: Thu Jul 4 09:38:26 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 746c3f03f..479de1217 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.224.112.202 1;
+35.192.85.2 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index 500584a94..d9f0a2c04 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.224.112.202 0;
+35.192.85.2 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index 2bbce7148..6cadc7f55 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1746
-### Updated: Thu Jul 4 09:09:17 SAST 2019
+### Version: V3.2019.07.1747
+### Updated: Thu Jul 4 09:38:26 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 746c3f03f..479de1217 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.224.112.202 1;
+35.192.85.2 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index 500584a94..d9f0a2c04 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.224.112.202 0;
+35.192.85.2 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
index 2bbce7148..6cadc7f55 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1746
-### Updated: Thu Jul 4 09:09:17 SAST 2019
+### Version: V3.2019.07.1747
+### Updated: Thu Jul 4 09:38:26 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
index 746c3f03f..479de1217 100644
--- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.224.112.202 1;
+35.192.85.2 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
index 500584a94..d9f0a2c04 100644
--- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.224.112.202 0;
+35.192.85.2 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
index 2bbce7148..6cadc7f55 100644
--- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1746
-### Updated: Thu Jul 4 09:09:17 SAST 2019
+### Version: V3.2019.07.1747
+### Updated: Thu Jul 4 09:38:26 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template
index 2bbce7148..6cadc7f55 100644
--- a/.dev-tools/globalblacklist-testing.template
+++ b/.dev-tools/globalblacklist-testing.template
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1746
-### Updated: Thu Jul 4 09:09:17 SAST 2019
+### Version: V3.2019.07.1747
+### Updated: Thu Jul 4 09:38:26 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template
index de5feb607..4c7d8c9cb 100755
--- a/.dev-tools/globalblacklist.template
+++ b/.dev-tools/globalblacklist.template
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1746
-### Updated: Thu Jul 4 09:09:16 SAST 2019
+### Version: V4.2019.07.1747
+### Updated: Thu Jul 4 09:38:25 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/setup-ngxblocker/commented/arch.conf.modified b/.dev-tools/setup-ngxblocker/commented/arch.conf.modified
index 0dfe86885..25cd0c350 100644
--- a/.dev-tools/setup-ngxblocker/commented/arch.conf.modified
+++ b/.dev-tools/setup-ngxblocker/commented/arch.conf.modified
@@ -1,13 +1,4 @@
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
#user html;
worker_processes 1;
@@ -35,6 +26,15 @@ http {
sendfile on;
#tcp_nopush on;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
#keepalive_timeout 0;
keepalive_timeout 65;
diff --git a/.dev-tools/setup-ngxblocker/commented/fedora.conf.modified b/.dev-tools/setup-ngxblocker/commented/fedora.conf.modified
index ce4abeece..1a37d53e8 100644
--- a/.dev-tools/setup-ngxblocker/commented/fedora.conf.modified
+++ b/.dev-tools/setup-ngxblocker/commented/fedora.conf.modified
@@ -1,13 +1,4 @@
# For more information on configuration, see:
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
# * Official English Documentation: http://nginx.org/en/docs/
# * Official Russian Documentation: http://nginx.org/ru/docs/
@@ -32,6 +23,15 @@ http {
sendfile on;
tcp_nopush on;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
diff --git a/.dev-tools/setup-ngxblocker/commented/opensuse.conf.modified b/.dev-tools/setup-ngxblocker/commented/opensuse.conf.modified
index 8e3a37c0b..6f426e57e 100644
--- a/.dev-tools/setup-ngxblocker/commented/opensuse.conf.modified
+++ b/.dev-tools/setup-ngxblocker/commented/opensuse.conf.modified
@@ -1,13 +1,4 @@
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
#user nginx;
worker_processes 1;
@@ -47,6 +38,15 @@ http {
sendfile on;
#tcp_nopush on;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
#keepalive_timeout 0;
keepalive_timeout 65;
diff --git a/.dev-tools/setup-ngxblocker/commented/ubuntu16.conf.modified b/.dev-tools/setup-ngxblocker/commented/ubuntu16.conf.modified
index 4cbde66dd..672bc00da 100644
--- a/.dev-tools/setup-ngxblocker/commented/ubuntu16.conf.modified
+++ b/.dev-tools/setup-ngxblocker/commented/ubuntu16.conf.modified
@@ -1,13 +1,4 @@
user www-data;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
worker_processes auto;
pid /run/nginx.pid;
@@ -24,6 +15,15 @@ http {
sendfile on;
tcp_nopush on;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
diff --git a/.dev-tools/setup-ngxblocker/commented/ubuntu18.conf.modified b/.dev-tools/setup-ngxblocker/commented/ubuntu18.conf.modified
index 353cd606a..b8f9bebb3 100644
--- a/.dev-tools/setup-ngxblocker/commented/ubuntu18.conf.modified
+++ b/.dev-tools/setup-ngxblocker/commented/ubuntu18.conf.modified
@@ -1,13 +1,4 @@
user www-data;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
@@ -25,6 +16,15 @@ http {
sendfile on;
tcp_nopush on;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
diff --git a/.dev-tools/setup-ngxblocker/modified/arch.conf.modified b/.dev-tools/setup-ngxblocker/modified/arch.conf.modified
index 0dfe86885..25cd0c350 100644
--- a/.dev-tools/setup-ngxblocker/modified/arch.conf.modified
+++ b/.dev-tools/setup-ngxblocker/modified/arch.conf.modified
@@ -1,13 +1,4 @@
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
#user html;
worker_processes 1;
@@ -35,6 +26,15 @@ http {
sendfile on;
#tcp_nopush on;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
#keepalive_timeout 0;
keepalive_timeout 65;
diff --git a/.dev-tools/setup-ngxblocker/modified/centos.conf.modified b/.dev-tools/setup-ngxblocker/modified/centos.conf.modified
index 8660e9a50..aaa446334 100644
--- a/.dev-tools/setup-ngxblocker/modified/centos.conf.modified
+++ b/.dev-tools/setup-ngxblocker/modified/centos.conf.modified
@@ -1,13 +1,4 @@
# For more information on configuration, see:
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
# * Official English Documentation: http://nginx.org/en/docs/
# * Official Russian Documentation: http://nginx.org/ru/docs/
@@ -32,6 +23,15 @@ http {
sendfile on;
tcp_nopush on;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
diff --git a/.dev-tools/setup-ngxblocker/modified/fedora.conf.modified b/.dev-tools/setup-ngxblocker/modified/fedora.conf.modified
index c2a034b98..709b85100 100644
--- a/.dev-tools/setup-ngxblocker/modified/fedora.conf.modified
+++ b/.dev-tools/setup-ngxblocker/modified/fedora.conf.modified
@@ -1,13 +1,4 @@
# For more information on configuration, see:
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
# * Official English Documentation: http://nginx.org/en/docs/
# * Official Russian Documentation: http://nginx.org/ru/docs/
@@ -32,6 +23,15 @@ http {
sendfile on;
tcp_nopush on;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
diff --git a/.dev-tools/setup-ngxblocker/modified/opensuse.conf.modified b/.dev-tools/setup-ngxblocker/modified/opensuse.conf.modified
index a18d796f1..4e41cc065 100644
--- a/.dev-tools/setup-ngxblocker/modified/opensuse.conf.modified
+++ b/.dev-tools/setup-ngxblocker/modified/opensuse.conf.modified
@@ -1,13 +1,4 @@
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
#user nginx;
worker_processes 1;
@@ -47,6 +38,15 @@ http {
sendfile on;
#tcp_nopush on;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
#keepalive_timeout 0;
keepalive_timeout 65;
diff --git a/.dev-tools/setup-ngxblocker/modified/ubuntu16.conf.modified b/.dev-tools/setup-ngxblocker/modified/ubuntu16.conf.modified
index 5393a869b..55bba61ef 100644
--- a/.dev-tools/setup-ngxblocker/modified/ubuntu16.conf.modified
+++ b/.dev-tools/setup-ngxblocker/modified/ubuntu16.conf.modified
@@ -1,13 +1,4 @@
user www-data;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
worker_processes auto;
pid /run/nginx.pid;
@@ -24,6 +15,15 @@ http {
sendfile on;
tcp_nopush on;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
diff --git a/.dev-tools/setup-ngxblocker/modified/ubuntu18.conf.modified b/.dev-tools/setup-ngxblocker/modified/ubuntu18.conf.modified
index c2bc002c1..5454fdb65 100644
--- a/.dev-tools/setup-ngxblocker/modified/ubuntu18.conf.modified
+++ b/.dev-tools/setup-ngxblocker/modified/ubuntu18.conf.modified
@@ -1,13 +1,4 @@
user www-data;
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
@@ -25,6 +16,15 @@ http {
sendfile on;
tcp_nopush on;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
diff --git a/.dev-tools/setup-ngxblocker/original/arch.conf.modified b/.dev-tools/setup-ngxblocker/original/arch.conf.modified
index 0dfe86885..25cd0c350 100644
--- a/.dev-tools/setup-ngxblocker/original/arch.conf.modified
+++ b/.dev-tools/setup-ngxblocker/original/arch.conf.modified
@@ -1,13 +1,4 @@
-
-
- ##
- # Nginx Bad Bot Blocker Includes
- # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
- ##
- include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
- include /etc/nginx/conf.d/globalblacklist.conf;
-
#user html;
worker_processes 1;
@@ -35,6 +26,15 @@ http {
sendfile on;
#tcp_nopush on;
+
+
+ ##
+ # Nginx Bad Bot Blocker Includes
+ # REPO: https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker
+ ##
+ include /etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /etc/nginx/conf.d/globalblacklist.conf;
+
#keepalive_timeout 0;
keepalive_timeout 65;
diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
index de5feb607..4c7d8c9cb 100644
--- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1746
-### Updated: Thu Jul 4 09:09:16 SAST 2019
+### Version: V4.2019.07.1747
+### Updated: Thu Jul 4 09:38:25 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 42c86bb7d..90fe4f9d3 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.193.7.13 1;
+104.154.120.187 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index e42e64139..bc389ee32 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.193.7.13 0;
+104.154.120.187 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index de5feb607..4c7d8c9cb 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1746
-### Updated: Thu Jul 4 09:09:16 SAST 2019
+### Version: V4.2019.07.1747
+### Updated: Thu Jul 4 09:38:25 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 42c86bb7d..90fe4f9d3 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.193.7.13 1;
+104.154.120.187 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index e42e64139..bc389ee32 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.193.7.13 0;
+104.154.120.187 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
index de5feb607..4c7d8c9cb 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1746
-### Updated: Thu Jul 4 09:09:16 SAST 2019
+### Version: V4.2019.07.1747
+### Updated: Thu Jul 4 09:38:25 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
index 42c86bb7d..90fe4f9d3 100644
--- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.193.7.13 1;
+104.154.120.187 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
index e42e64139..bc389ee32 100644
--- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.193.7.13 0;
+104.154.120.187 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
index de5feb607..4c7d8c9cb 100644
--- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1746
-### Updated: Thu Jul 4 09:09:16 SAST 2019
+### Version: V4.2019.07.1747
+### Updated: Thu Jul 4 09:38:25 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
index ca6bca078..5c7a773cf 100644
--- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
@@ -46,4 +46,4 @@
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
35.184.96.71 0;
-35.188.1.99 0;
+35.224.112.202 0;
diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
index 2c180ceb6..f727ecf7c 100644
--- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1746
-### Updated: Thu Jul 4 09:09:16 SAST 2019
+### Version: V4.2019.07.1747
+### Updated: Thu Jul 4 09:38:25 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
index 6ff35261d..3767371d9 100644
--- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
@@ -45,5 +45,5 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.192.85.2 0;
-35.193.7.13 0;
+35.188.1.99 0;
+104.154.120.187 0;
diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf
index f4571e538..c1b254b72 100644
--- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1746
-### Updated: Thu Jul 4 09:09:16 SAST 2019
+### Version: V4.2019.07.1747
+### Updated: Thu Jul 4 09:38:25 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
index 3eee46194..ec5a8eed1 100644
--- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
@@ -45,5 +45,6 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
+35.184.226.236 0;
+35.202.145.110 0;
104.154.120.187 0;
-35.193.7.13 0;
diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
index de5feb607..4c7d8c9cb 100644
--- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1746
-### Updated: Thu Jul 4 09:09:16 SAST 2019
+### Version: V4.2019.07.1747
+### Updated: Thu Jul 4 09:38:25 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
index 4fd05aec4..fd2b0e8d9 100644
--- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
@@ -45,6 +45,5 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.192.85.2 0;
-35.184.226.236 0;
104.154.120.187 0;
+35.224.112.202 0;
diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
index de5feb607..4c7d8c9cb 100644
--- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1746
-### Updated: Thu Jul 4 09:09:16 SAST 2019
+### Version: V4.2019.07.1747
+### Updated: Thu Jul 4 09:38:25 SAST 2019
### Bad Referrer Count: 6719
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf
index 746c3f03f..479de1217 100644
--- a/.dev-tools/test_units/blacklist-ips.conf
+++ b/.dev-tools/test_units/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.224.112.202 1;
+35.192.85.2 1;
127.0.0.1 1;
diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list
index c130380fc..1ab420fcf 100644
--- a/.dev-tools/test_units/random-bots-for-test-quick.list
+++ b/.dev-tools/test_units/random-bots-for-test-quick.list
@@ -1,10 +1,10 @@
-Aboundexbot
-Battleztar Bazinga
-Bitacle
-crawl.sogou.com
-Jorgee
-Mag-Net
-PyCurl
-RedesScrapy
-Siphon
-WWW-Mechanize
+Bolt
+BuiltWith
+crawler4j
+Exabot
+MauiBot
+Nimbostratus
+Upflow
+WebFetch
+webmeup-crawler
+ZumBot
diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list
index d1d9bc8c4..a150b6542 100644
--- a/.dev-tools/test_units/random-bots-for-test.list
+++ b/.dev-tools/test_units/random-bots-for-test.list
@@ -1,250 +1,250 @@
360Spider
404checker
-Aboundex
-Aboundexbot
-ADmantX
+Abonti
+AhrefsBot
+AiHitBot
+Aipbot
Alexibot
-Alligator
+AllSubmitter
+AlphaBot
+Anarchie
+archive.org_bot
+ASPSeek
+Asterias
Attach
autoemailspider
BackDoorBot
Backlink-Ceck
-backlink-check
BackStreet
Bandit
-BDFetch
-BetaBot
+Barkrowler
+BatchFTP
Bigfoot
+Bitacle
Blackboard
-BLEXBot
-BlowFish
-Bolt
-Brandprotect
+Blow
+BotALot
Brandwatch
Buddy
BuiltBotTough
BuiltWith
+Bullseye
BunnySlippers
-Cegbfeieh
-CheeseBot
-CherryPicker
-Chlooe
-Cogentbot
+Calculon
+CCBot
+CheTeam
+Claritybot
+coccocbot-web
cognitiveseo
-Collector
com.plumanalytics
-CopyRightCheck
-Cosmos
+Copyscape
crawler.feedback
-Custo
-DBLBot
+crawl.sogou.com
+Crescent
+Curious
+DatabaseDriverMysqli
Demon
Deusu
Digincore
+DigitalPebble
DIIbot
-Disco
+Dirbuster
Discobot
-DomainAppender
-DomainSigmaCrawler
+DittoSpyder
+DnyzBot
+Dotbot
+Download Wonder
+Dragonfly
+Drip
Ecxi
EirGrabber
EMail Siphon
+EMail Wolf
EroCrawler
-FDM
-FemtosearchBot
-Firefox/7.0
+Evil
+ExtractorPro
+Extreme Picture Finder
+EyeNetIE
+Ezooms
FlashGet
+Flunky
+Foobot
+Freeuploader
FrontPage
-FyberSpider
-GalaxyBot
-GetRight
-G-i-g-a-b-o-t
+Fyrebot
+Genieo
+Getintent
+Go-Ahead-Got-It
+Gotit
+GoZilla
Go!Zilla
Grafula
GrapeFX
GrapeshotCrawler
-GridBot
-HaosouSpider
-Harvest
+Haansoft
Hloader
-HTMLparser
-HTTrack
-Humanlinks
-HybridBot
-Iblog
Id-search
+Image Fetch
Image Sucker
-IndeedBot
-Indy Library
+InfoNaviRobot
instabid
-Intelliseek
-InternetSeer
-IstellaBot
-JamesBOT
-Jbrofuzz
+Internet Ninja
+JennyBot
JetCar
-JikeSpider
+Jetty
JOC Web Spider
-Joomla
Jorgee
-Jyxobot
+Kenjin Spider
Keyword Density
Kozmosbot
-LeechFTP
+Lanshanbot
+LexiBot
Lftp
-LibWeb
-Lightspeedsystems
Likse
-Linkdexbot
-LinkpadBot
-LinkWalker
-Lipperhey
-Lipperhey Spider
+LinkextractorPro
Litemage_walker
-LNSpiderguy
-lwp-trivial
+Lmspider
+Ltx71
Magnet
Mag-Net
magpie-crawler
-Mail.RU_Bot
+Majestic12
MarkMonitor
+MarkWatch
Masscan
-Mass Downloader
Mata Hari
+meanpathbot
Meanpathbot
-MeanPath Bot
Mediatoolkitbot
-mediawords
+Microsoft Data Access
+MIIxpc
Mister PiX
-MJ12bot
Mojeek
Mr.4x3
-Msrabot
-MS Web Services Client Protocol
+MSIECrawler
muhstik-scan
-Musobot
+Name Intelligence
Nameprotect
-Navroad
-Needle
+NearSite
Nessus
-NetAnts
-Netcraft
netEstate NE Crawler
-Netvibes
-NetZIP
-NextGenSearchBot
-Niki-bot
-Nikto
+NetLyzer
+NetSpider
+NICErsPRO
+NimbleCrawler
+Ninja
Nmap
-oBot
-Octopus
+Nutch
+Offline Explorer
Offline Navigator
-OnCrawl
+OpenLinkProfiler
+Openvas
OrangeBot
-Page Analyzer
+OutclicksBot
+OutfoxBot
+PageAnalyzer
PageGrabber
PageScorer
-pcBrowser
-PECL::HTTP
-PHPCrawl
-Picsearch
+Pandalytics
+Panscient
+Papa Foto
+PeoplePal
+PictureFinder
+Pimonster
Pixray
-POE-Component-Client-HTTP
+Pockey
Probethenet
Pump
PxBroker
PyCurl
-Quick-Crawler
-RankActive
RankActiveLinkBot
-RankFlex
-RankingBot
-RankurBot
+RankingBot2
+Rankivabot
+RealDownload
Reaper
RebelMouse
ReGet
-RocketCrawler
Rogerbot
+SalesIntelligent
+SBIder
+ScoutJet
Scrapy
-Screaming
-Semrush
+Searchestate
+SearchmetricsBot
SemrushBot
-SEOprofiler
-SeoSiteCheckup
+seoscanners
SEOstats
sexsearcher
-Siphon
-SISTRIX
SiteExplorer
-Siteimprove
SiteLockSpider
-SiteSucker
+SiteSnagger
+Sitevigil
SlySearch
-SmartDownload
+SMTBot
+Snapbot
+Snoopy
+SocialRankIOBot
+Sociscraper
Sogou web spider
+Sottopop
SpaceBison
SpankBot
-sp_auditbot
Spbot
-SputnikBot
+Sqlmap
Sqlworm
-Steeler
-Sucker
+Sqworm
Sucuri
-SuperBot
-SurveyBot
Suzuran
Swiftbot
Teleport
TeleportPro
-Telesphorep
-Toweyabot
+Telesphoreo
+Thumbor
+Toata
Trendiction
-Trendictionbot
-Turingos
+True_Robot
Turnitin
TurnitinBot
Twice
Typhoeus
+UnisterBot
+Upflow
URLy Warning
-Vagabondo
-VeriCiteCrawler
+URLy.Warning
+VB Project
+Virusdie
VoidEYE
Wallpapers/3.0
-WallpapersHD
-WebAuto
-Web Auto
-Web Collage
+WBSearchBot
+WebCollage
+WebCopier
WEBDAV
+WebEnhancer
Web Enhancer
-WebFetch
Web Fetch
-WebFuck
-Web Fuck
WebLeacher
-WebReaper
-WebSauger
-Webshag
+WebmasterWorldForumBot
+webmeup-crawler
+WebPix
+Web Pix
+WebsiteExtractor
WebsiteQuester
-Webster
+Website Quester
WebStripper
+WebSucker
WebWhacker
WebZIP
-Who.is Bot
-Widow
-WinHTTrack
-WiseGuys Robot
-Wonderbot
+WeSEE
+Whacker
Woobot
-WPScan
+Wprecon
WWW-Collector-E
+WWW-Mechanize
WWW::Mechanize
WWWOFFLE
-xpymep1.exe
-YoudaoBot
-Zade
Zauba
Zermelo
zgrab
Zitebot
-ZmEu
+ZumBot
+ZyBorg
diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list
index 2ec0e899b..43a46d69b 100644
--- a/.dev-tools/test_units/random-bots-for-whitelist-test.list
+++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list
@@ -1,250 +1,250 @@
404enemy
-Abonti
+80legs
+Aboundex
+Acunetix
+ADmantX
AfD-Verbotsverfahren
+AIBOT
AiHitBot
-Alligator
-Anarchie
-ASPSeek
-Attach
-BackStreet
+Aipbot
+archive.org_bot
+Asterias
+autoemailspider
+BackDoorBot
+Backlink-Ceck
+backlink-check
+BackWeb
+Badass
Barkrowler
Battleztar Bazinga
BBBike
BDCbot
BDFetch
-Bitacle
-BLEXBot
-Boardreader
+Blackboard
+Black Hole
+BlackWidow
+Blow
+BlowFish
BotALot
-Brandprotect
-BunnySlippers
+Brandwatch
+BuiltBotTough
+Bullseye
BuzzSumo
-Calculon
-CATExplorador
-CherryPicker
-CheTeam
-ChinaClaw
+Cegbfeieh
Chlooe
Claritybot
Cliqzbot
Cloud mapping
coccocbot-web
-Cogentbot
+cognitiveseo
+Collector
com.plumanalytics
-Copier
CopyRightCheck
+Cosmos
+Craftbot
crawler4j
crawl.sogou.com
-CrazyWebCrawler
-CrunchBot
+Crescent
+Curious
+Custo
DatabaseDriverMysqli
DataCha0s
-DBLBot
-demandbase-bot
-Demon
Deusu
-Devil
-Digincore
-Discobot
-DittoSpyder
+DIIbot
+Dirbuster
DomainAppender
-DomainCrawler
-DomainSigmaCrawler
-DomainStatsBot
Dotbot
+Download Wonder
DSearch
+EasyDL
eCatch
-ECCP/1.0
-EirGrabber
+Ecxi
EMail Siphon
-EroCrawler
-Evil
+EMail Wolf
+evc-batch
Exabot
Express WebPictures
ExtLinksBot
-Extractor
-ExtractorPro
-EyeNetIE
-Ezooms
facebookscraper
-FlashGet
-Flunky
+FDM
+FHscan
+Fimap
+Foobot
+Fyrebot
GalaxyBot
Genieo
+GermCrawler
+Getintent
GetRight
-Gigablast
-Gigabot
-G-i-g-a-b-o-t
-Go-Ahead-Got-It
+GetWeb
+Gotit
Go!Zilla
Grafula
-GridBot
-GT::WWW
-Haansoft
+GrapeFX
+GrapeshotCrawler
Harvest
HEADMasterSEO
Heritrix
-HMView
+Hloader
HTTP::Lite
+HTTrack
Humanlinks
-HybridBot
Iblog
-IDBot
Id-search
-Image Fetch
+IlseBot
Indy Library
+InfoTekies
instabid
-InterGET
-Internet Ninja
+Intelliseek
+InternetSeer
internetVista monitor
-ips-agent
Iria
-Iskanie
-IstellaBot
+IRLbot
JennyBot
-JetCar
-Jetty
Joomla
-Jorgee
-Keyword Density
-Lanshanbot
+JustView
+Kenjin Spider
+Kozmosbot
Larbin
LeechFTP
-LexiBot
Libwhisker
+Lightspeedsystems
+Likse
+Linkdexbot
LinkextractorPro
LinkpadBot
-LinkWalker
+LinkScan
+LinqiaMetadataDownloaderBot
LinqiaRSSBot
Lipperhey
-Lipperhey Spider
-Lmspider
-LWP::Simple
lwp-trivial
-Mag-Net
-Mail.RU_Bot
-Masscan
+Majestic SEO
+MarkMonitor
+MarkWatch
+Mata Hari
Meanpathbot
Mediatoolkitbot
-Microsoft URL Control
+Microsoft Data Access
MIDown tool
Mister PiX
-Mojeek
-Morfeus Fucking Scanner
+MJ12bot
+Msrabot
MS Web Services Client Protocol
muhstik-scan
Musobot
-Nameprotect
Navroad
-NetAnts
+Needle
netEstate NE Crawler
-NetMechanic
NetSpider
-Nettrack
Netvibes
-NICErsPRO
+NetZIP
Niki-bot
-Ninja
+Nikto
+NimbleCrawler
+NPbot
Nutch
-Octopus
-Offline Explorer
-Offline Navigator
+oBot
+OnCrawl
Openvas
OpenVAS
+OrangeSpider
+OutclicksBot
+PageAnalyzer
+page scorer
PageScorer
-Papa Foto
+Pandalytics
+Pavuk
+pcBrowser
PECL::HTTP
PeoplePal
PHPCrawl
-Picscout
-PictureFinder
-Pimonster
+Picsearch
Pi-Monster
-PleaseCrawl
-POE-Component-Client-HTTP
-ProPowerBot
-Pump
+Pixray
+ProWebWalker
+PyCurl
RankFlex
-RankingBot2
+Rankivabot
RankurBot
RealDownload
-Reaper
-RebelMouse
Recorder
-RepoMonkey
+RedesScrapy
+ReGet
Ripper
+Rogerbot
SalesIntelligent
+Scanbot
ScoutJet
+Searchestate
+SearchmetricsBot
+SEOkicks
SEOlyticsCrawler
+Seomoz
SEOprofiler
seoscanners
-sexsearcher
-Siphon
-SISTRIX
-Sitebeam
-SiteLockSpider
-Site Sucker
+SeoSiteCheckup
+SEOstats
+Shodan
+SiteExplorer
+SiteSucker
+Sitevigil
SlySearch
-SmartDownload
SMTBot
-Snapbot
-Snoopy
-SocialRankIOBot
-Sottopop
-SpaceBison
-Spammen
-SpankBot
-sp_auditbot
Spinn3r
-SputnikBot
-Sqlworm
+spyfu
+Sqworm
Steeler
Stripper
-SurveyBot
-Szukacz
+Suzuran
+T0PHackTeam
+T8Abot
tAkeOut
-Teleport
TeleportPro
-Telesoft
+Telesphorep
The Intraformant
-TheNomad
-TightTwatBot
+Thumbor
Titan
-Toata
-Toweyabot
-Tracemyfile
-Trendiction
-Trendictionbot
trendiction.com
trendiction.de
-True_Robot
-Turingos
-Turnitin
-TwengaBot
-Upflow
+Twice
+Typhoeus
+UnisterBot
Vacuum
+Vagabondo
+VCI
VeriCiteCrawler
-Virusdie
-VoidEYE
-Voltron
-WallpapersHD
-Webalta
+VidibleScraper
+Wallpapers/3.0
+WASALive-Bot
Web Auto
Web Collage
+WebEnhancer
Web Enhancer
WebFetch
+Web Fetch
+WebFuck
Web Fuck
-WebLeacher
-WebmasterWorldForumBot
-webmeup-crawler
-WebPix
-WebReaper
+WebGo IS
+WebSauger
Webshag
+WebsiteQuester
Website Quester
Webster
WebStripper
-WebWhacker
+WebSucker
+Web Sucker
WeSEE
-WPScan
-WWW-Collector-E
-WWW-Mechanize
+Widow
+WinHTTrack
+WiseGuys Robot
+Woobot
+Wotbox
+Wprecon
+WWW::Mechanize
+x09Mozilla
+x22Mozilla
+Zade
+zauba.io
+Zitebot
+ZumBot
+ZyBorg
diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list
index c71f22dd2..fe775b54a 100644
--- a/.dev-tools/test_units/random-referrers-for-test-quick.list
+++ b/.dev-tools/test_units/random-referrers-for-test-quick.list
@@ -1,10 +1,10 @@
-brains2.biz
-citysecurity.nu
-img.wallpaperstock.net
-long-beach-air-conditioning.com
-mowser.com
-nordstar.pro
-shemalegalls.blogporn.in
-stpicks.com
-stretchmate.net
-usacasino.com
+canacopegdl.com
+dogbreedspicture.net
+fickblock18.com
+growshop.es
+justprofit.xyz
+knogg.net
+larger.io
+meet-flirt-dating.com
+privat-girl.net
+toloka.hurtom.com
diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list
index 949a687ae..15a3c23e4 100644
--- a/.dev-tools/test_units/random-referrers-for-test.list
+++ b/.dev-tools/test_units/random-referrers-for-test.list
@@ -1,500 +1,500 @@
-00it.com
-101billion.com
-123any.com
-2x2fan.ru
-40cg.com
-4kepics.com
-4kplayer.pl
-7minuteworkout.com
-8si.ru
-a1security.com.ua
-aa08daf7e13b6345e09e92f771507fa5f4.com
-abiente.ru
-acc.eu.org
-addray.pro
-adelly.bg
-admeasures.com
+12masterov.com
+1-free-share-buttons.com
+300richmond.co.nz
+3weekdiet.com
+4istoshop.com
+5ws.dating-app.ru
+6128786.com
+72-news.com
+a14download.com
+a96527gi.beget.tech
+aa81bf391151884adfa3dd677e41f94be1.com
+abbanreddy.com
+actices.com
+actulite.com
+adconscious.com
+adimmix.com
+adnotbad.com
+adprotect.net
ads.gold
-adsland.men
-adtech.fr
+adssafeprotected.com
adultmeetonline.info
-adultnet.in
-advokateg.xyz
-affordablewebsitesandmobileapps.com
-afftrack001.com
-ahrntal.verymes.xyz
-akvamaster.dp.ua
-albumsuper.info
-allblogroll.com
-alphaforum.ru
-amyfoxfitness.com
-anabolics.shop
-anti-virus-removal.info
-apifasterlightin-a.akamaihd.net
-apiholdingmypage-a.akamaihd.net
-apimegabrowsebiz-a.akamaihd.net
-apimountainbikei-a.akamaihd.net
-api.stathat.com
-app5.letmacworkfaster.world
-apps-analytics.net
-arabseyes.com
-arenda-avtoprokat-krasnodar.ru
-arendas.net
-arkartex.ru
-arkkivoltti.net
-arraty.altervista.org
+afora.ru
+agecheckadult.com
+agroeconom.kz
+aihelen.net
+airlandsea.info
+albuteroli.com
+alert-hgd.xyz
+alert-jdh.xyz
+alfa-medosmotr.ru
+all4wap.ru
+amehdaily.com
+ampower.me
+appartement-stumm.at
+appiq.mobi
+ap.senai.br
+arcadepages.com
+arclk.net
+articlesdirectoryme.info
+artparquet.ru
+asdfg.pro
+asdfz.pro
asia-forum.ru
-asrv-a.akamaihd.net
-asrvvv-a.akamaihd.net
-autowebmarket.com.ua
-aviapanda.ru
-avramstroy.ru
-azartmix.com
-azbaseimages.net
-babosas.com
-babs.com.ua
-babyguns.ru
-backgroundpictures.net
-balkanfarma.org
-bannerswap.com
-beatifullandscape.co
-beclean-nn.ru
-bedcapdealers.com
-bestchoice.cf
-bestdooz.com
+audiobangout.com
+audiofree.ru
+autoplate.info
+aviav.eu
+ayodhya.co
+azbukadiets.ru
+baditri.com
+baikaleminer.com
+balla-boo.se
+bambi.ck.ua
+bard-real.com.ua
+bbtec.net
+beachpics.com
+beauby.ru
+becuo.com
+beget.tech
+best-coupon-offer.com
bestplacetobuyeriacta.jw.lt
best-seo-offer.com
-bestserials.com
-bestsexyblog.com
-bestwaystogetpaid.us
-bestwebsiteawards.com
-bestwebsitesawards.com
-betwinservice.com
-bezlimitko.xyz
-bg6s0.com
-bidr.trellian.com
+best-seo-software.xyz
+besttorrentknifta.weebly.com
+betterhdporn.com
bigames.online
-biketank.ga
-binaryoptionscops.info
-bitporno.sx
+bimatoprost-careprost.com.ua
+biographiya.com
+bird1.ru
+blackcurranthumidifiers.site
+blogs.rediff.com
+blpmovies.com
+bluerobot.info
bocoarchives.org
-bonkers.name
-bookmark4you.com
-brains2.biz
-brainsandeggs.blogspot.com
-bravegirlsclub.com
-buyclomidonlaine.com
-buyessaynow.biz
-buyk.host.sk
-buzzzg.men
-bvlgaribracelet.xyz
-bytimedance.ru
-camdenmemorials.com
-canvas.gloverid.site
-carstrends2015.com
-casinorewards.info
+boeuklad.com
+bonzibuddi.com
+bosman.pluto.ro
+braindaily.xyz
+b-styles.xyz
+bugof.gq
+buntube.net
+businescoop.men
+buytizanidineonline.blogspot.com
+buzzurl.jp
+camdolls.com
+cand.jp
+capture-room.com
+cartierbraceletsreplica.pw
+cartierlove2u.xyz
+cartiertrinity.xyz
+caulderwoodkennels.com
+cbcseward.com
c-gainsbourg.com
-cheapestjerseysonwholesale.com
-checktext.org
-chorus.terakeet.com
-chuckguilford.com
-cialgenisrx.com
-cialischstgerts.com
-citizenclsdriveri7.pen.io
-clickbanksites.info
-clickpapa.com
-clickzzs.nl
-coindirect.io
-coldfilm.ru
-cometorussia.net
+chatroulette.life
+checkm8.com
+chidporn.com
+cialischsrx.com
+cibpenonptib.flu.cc
+circlesl.com
+cityadspix.com
+citysecurity.nu
+cjs.com.ru
+clmforexeu.com
comissionka.net
-compliance-brian.xyz
-compliance-george.xyz
-compliance-irvin.top
-concordexoticrentals.com
-contextualyield.com
-coolgamechannel.com
-copyrightclaims.org
-covadhosting.biz
-crest-poloski.ru
-custodieva.ru
-dangphoto.trade
-dengi-pod-zalog-nedvizhimosti.ru
-digilander.libero.it
-dignitasdata.se
-divku.ru
-dogsrun.net
-dosug-lux.ru
-drunkmoms.net
-dyt.net
-eandsgallery.com
-educontest.net
-ege-essay.ru
+compliance-alex.top
+compliance-barak.top
+compliance-donald.xyz
+compliance-ivan.xyz
+compliance-john.top
+cookielawblog.wordpress.com
+cookie-law-enforcement-ee.xyz
+cordstrap.cc
+cottageofgrace.com
+country-chic.ru
+dalavia.ru
+darkages.info
+data-mining.tk
+dawlenie.com
+degerlund.net
+dellalimov.com
+dfwu1019.info
+digitalassetmanagement.site
+dildofotzen.net
+dirtpics.pw
+divisioncore.com
+djonwatch.ru
+dms-sw.ru
+dnsrsearch.com
+dodlive.mil
+domaincheck.io
+domznaniy.ru
+downloadmefiranaratb1972.xpg.com.br
+doyouknowtheword-flummox.ml
+drhomes.biz
+drpornogratisx.xxx
+dvrlists.com
+easync.io
+ekn-art.se
+eldiariodeguadalajara.com
+elektir.ru
+elektrischezigarette1.blog.pl
+elektrischezigaretten1.blogse.nl
+elektronischezigarettekaufen1.myblog.de
+elmacho.xyz
elmifarhangi.com
-elvenar.com
-en.home-task.com
-ero-advertising.com
-erotikchat-24.com
-euronis-free.com
-evaashop.ru
-extlinks.com
-extrabot.com
-ezigarettekaufen2.mpbloggar.se
-ezigaretteshop2.mywapblog.com
-ezigaretteshop2.vefblog.net
-factorynightclub.com
-falopicm.pw
-fanoboi.com
-fatfasts-4tmz.com
-feel-planet.com
-filmetricsasia.com
-fitness-video.net
-flash4fun.com
-flauntyoursite.com
-formseo.com
-forums.toucharcade.com
-foxweber.com
-freenode.info
-free-share-buttons.top
-freeuploader.com
-front.ru
-fxgallery.com
-gabeshop.ru
-gamebackyard.com
-gameplexcity.com
-games.kolossale.ru
-gateway.zscalerone.net
-gay-file.com
-gazobeton-p.com.ua
-gdebestkupit.ru
+el-nation.com
+eshop4u.jp
+essay-edu.biz
+everypony.ru
+excaliburfilms.com
+exchangeit.gq
+executehosting.com
+facecup.top
+fba-mexico.com
+ff30236ddef1465f88547e760973d70a.com
+findsexguide.com
+firma-legion.ru
+flex4launch.ru
+forum69.info
+forum.poker4life.ru
+fotoxxxru.com
+foxydeal.com
+frbizlist.com
+free-traffic.xyz
+freshberry.com.ua
+frivgame250.com
+frustrated-favorable.gq
+fuckmill.com
+gambarkatabaru.com
+gamblingnerd.com
+gay.adultgalls.com
+gaygalls.net
+gazporno.com
geckoandfly.com
-gelezki.com
+gemgrab-a.akamaihd.net
get-free-social-traffic.com
-getoutofdebtfree.org
-gheus.altervista.org
-girlgamerdaily.com
-given2.com
-gkvector.ru
-go2album.com
-gojiberriess.apishops.ru
-goldadpremium.com
-goodhousekeeping.com
-graphicwe.org
-grupografico-pilar.com.ar
-gta-top.ru
-guge.io
-hdwalls.xyz
-health-medical-portal.info
-heartofbeijing.blogspot.com
-hem.passagen.se
-hermesreplica.win
-hesteel.pl
-hit-men.men
-hkladys.com
-hmywwogw.bloger.index.hr
-howtoclean.club
-howtowhitenteethfast.xyz
-hulfingtonpost.com
-humanelydrew.com
-husky-shop.cz
-id-forex.com
-imagui.eu
+getmiro.com
+ghernnqr.skyrock.com
+gigablast.com
+gogalleryawesome.com
+golmau.host.sk
+good-mummy.ru
+goodnightjournal.com
+googlepositions.com
+goo.ne.jp
+goosefishpost.bid
+graphid.com
+greatzip.com
+gsasearchenginerankerhelp.com
+gsasearchengineranker.xyz
+guarrasdelporno.xxx
+gwagka.com
+h2monline.com
+handicapbathtubarea.com
+hayate.biz
+hdpixent.com
+hdxnxxtube.mobi
+headpharmacy.com
+headpress.ru
+healing-dysplasia.ru
+hermesreplica.pw
+homemature.net
+hoste.octopis.com
+hotblog.top
+housekuba.org
+hptwaakw.blog.fc2.com
+https-legalrc.biz
+hubbble.com
+huimin764128.com
+hyhj.info
+ido3.com
+ifirestarter.ru
+images-free.net
imgata.com
-import-sales.com
-insider.pro
-investmac.com
-invest-pamm.ru
-investsuccess.org
-iscblog.info
-istanbulit.com
-itbc.kiev.ua
-iusstf.org
-ivoiretechnocom.ci
-jav-way.site
-jerseychinabizwholesale.com
-jerseysbizwholesalecheap.com
-jongose.ninja
-jose.mulinohouse.co
-jwss.cc
-kalandranis.gr
-kandidos.com
-kareliatobacco.ru
-kazan.zrus.org
-kazrent.com
-keywesthideaways.co
-kinosed.net
-kiwe-analytics.com
-klejonka.info
-konpax.com
-kustanay.kz
-kvartiry-remont.ucoz.ru
-lacapilla.info
-lacasamorett.com
-laserpen.club
-laudit.ru
-letsart.ru
-letslowbefast.site
-li-er.ru
-likrot.com
-linkbuddies.com
-linksharingt.com
-lion.playtap.us
-lirunet.ru
-lisque.batcave.net
-livejasmin.com
-liver-chintai.org
-lsex.xyz
-luxup.ru
-m3gadownload.pl
-make-money-online.com
-makis.nu
-mantramusic.ru
-maranbrinfo.com.br
-marmotstore.online
-matb3aa.com
-mb140.ru
-med-zdorovie.com.ua
-megahdporno.net
-member-quiz.com
-mensandals.xyz
-menstennisforums.com
-metabar.ru
-methodsmarketing.com
-m-google.xyz
-minecraft-rus.org
-miniads.ca
-missvietnam.org
-mmgq.ru
-mojpregled.com
-moroccosurfadventures.com
-moscow-clining.ru
-mouselink.co
-mrbojikobi4.biz
-mrlmedia.net
-muschisexbilder.com
-muycerdas.xxx
-myplaycity.com
-nac-bearings.ru
-nagdak.ru
-naperehresti.info
+impisr.edunsk.ru
+indiasourcemart.in
+infokonkurs.ru
+informatiecentro.be
+inome.com.ua
+insomniagamingfestival.com
+intelhdgraphicsgtdrive6w.metroblog.com
+intellektmedia.at
+ipornox.xxx
+iqbazar.ru
+iweblist.info
+izismile.com
+james13prix.info
+jamiembrown.com
+javrip.net
+jaxcube.info
+justprofit.xyz
+kabbalah-red-bracelets.com
+kartiniresto.com
+keyhantercume.com
+keywordglobal.co.za
+keywords-monitoring-your-success.com
+khafre.us
+kinobaks.com
+kinoplen.ru
+kinopolet.net
+kiprinform.com
+kladrus.ru
+koleso24.com.ua
+kollesa.ru
+konoplisemena.com
+kosova.de
+krasnodar-avtolombards.ru
+kurbappeal.info
+labelwater.se
+lafourmiliaire.com
+languagecode.com
+laparfumotec.com
+larose.jb2c.me
+law-check-two.xyz
+ledpolice.ru
+lenvred.org
+lernur.net
+letolove.ru
+linkrr.com
+liquimondo.com
+longadventure.com
+lottospring.com
+low-format.ru
+magicdiet.gq
+mahnwachen-helfen.info
+malwareremovalcenter.com
+manve.info
+meccadumps.net
+medictube.ru
+medtherapy.ru
+megaindex.ru
+meltwater.com
+menetie.ru
+mole.pluto.ro
+monetizationking.net
+moscow-region.ru
+moviemail-online.co.uk
+moyaterapiya.ru
+msk-diplomat.com
+mxgetcode.com
+myftpupload.com
+mylesosibirsk.ru
+naturtreenspicerx.pw
+nero-us.com
+newstaffadsshop.club
+nextbackgroundcheck.gq
+nextconseil.com
nicovideo.jp
-niroo.info
-nmrk.ru
-nottyu.xyz
-nvssf.com
-odoratus.net
-okayimage.com
-olympescort.com
-onemactrckr.com
-online247.ml
-ons-add.men
-oralsexfilme.net
-orhonit.com
-ossmalta.com
-outpersonals.com
-outrageousdeal-a.akamaihd.net
-owathemes.com
-oxotl.com
+nodding-passion.tk
+nootrino.com
+nordvpn.com
+northfacestore.online
+notfastfood.ru
+noumeda.com
+novgorod.xrus.org
+novosti-hi-tech.ru
+nvformula.ru
+oballergiya.ru
+ochistka-stokov.ru
+offergroup.info
+okmedia.sk
+oneshotdate.com
+onetravelguides.com
+onlineporno.site
+onstrapon.purplesphere.in
+ooo-gotovie.ru
+osb.se11.ru
+oz-offers.com
+ozoz.it
+paidonlinesites.com
panamaforbeginners.com
-paparazzistudios.com.au
-partnerads.men
-pathwhelp.org
-perl.dp.ua
+partnerline.men
+pcimforum.com
+pdn-4.com
+perfection-pleasure.ru
perso.wanadoo.es
-pestomou.info
-photo-clip.ru
-pictures-and-images.com
-pistonclasico.com
-plastgranar.nu
-pl-top.pl
-pons-presse.com
-pooleroadmedicalcentre.co.uk
-popugaychiki.com
-popup-hgd.xyz
-pornosemki.info
-pornoslive.net
-pr0fit-b0x.com
-pro-okis.ru
-prostitutki-astana.org
-pxhdwsm.com
-qcstrtvt.bloger.index.hr
-qitt.ru
-qpypcx.com
-qualitymarketzone.com
-quickloanbank.com
-ranksonic.info
-r-control.ru
-rcpmda.ikan1080.xyz
-realnye-otzyvy.info
-rebuildermedical.com
-reckonstat.info
-regdefense.com
-registry-cleaner.net
-registrydomainservices.com
-relayblog.com
-remont-comp-pomosh.ru
-remybutler.fr
-repeatlogo.co.uk
-reyel1985.webnode.fr
-ric.info
-ringtonepartner.com
-robot-forex.biz
-rock-cafe.info
-rocketchange.ru
-royal-betting.net
-rumamba.com
-runofilms.ru
-saecsa.co
-salut-camp.ru
-santechnik.jimdo.com
-sbetodiodnye-lampy.ru
-sbricur.com
+pewit.pw
+pigrafix.at
+pihl.se
+piter.xrus.org
+pizda.lol
+play-mp3.com
+pl-vouchers.com
+pochtovyi-index.ru
+podshipniki-ntn.ru
+popup-jdh.xyz
+porndl.org
+pornmania.pl
+porno-dojki.net
+pornoforadult.com
+porodasobak.net
+postclass.com
+potolokelekor.ru
+powc.r.ca.d.sendibm2.com
+pozvonim.com
+pricheskaonline.ru
+privatefx.all4invest.info
+prizesbook.online
+prointer.net.ua
+promalp-universal.ru
+pronorm.fr
+pro-poly.ru
+prostitutki-rostova.org
+psvita.ru
+pts163.ru
+pushdata.sendpulse.com
+quebec-bin.com
+raavidesigns.com
+rapidgator-porn.ga
+reeyanaturopathy.com
+reklamuss.ru
+remmling.de
+remontbiz.ru
+rentalcarnavi.info
+rentehno.ru
+replicalouboutin.xyz
+retreatia.com
+rniaeba.ga
+running-line.ru
+russkie-gorki.ru
+rvtv.ru
+sabaapress.com
+sajatvelemeny.com
+sa-live.com
+sanatorrii.ru
+saneyes.com
+satellite.maps.ilovevitaly.com
+savingsslider-a.akamaihd.net
+sax-sex.com
+scanner-irvin.top
scanner-jack.top
-scanner-margo.top
-searchencrypt.com
-searchwebknow-a.akamaihd.net
-seasaltwithfood.com
-seeresultshub-a.akamaihd.net
-sell-fb-group-here.com
-seminarykansas904.ml
-sexad.net
-sex-foto.pw
-sexuria.net
-sexvideo-sex.com
-sharebutton.to
-ship-marvel.co.ua
-shoplvlv.us
+scanner-mary.top
+scrnet.biz.ua
+seansonline24.pl
+search.alot.com
+segol.tv
+selectads.men
+seo-smm.kz
+sevendays.com.ua
+shohanb.com
+shopcheermakeup.info
shopperifymac.com
-shopwme.ru
-signal03.ru
-silktide.com
-siteaero.com
+signoredom.com
+simply.net
site-analyzer.com
-siteonomy.com
-site-speed-check.site
-sitevaluation.com
-sitiz.club
-sixcooler.de
-sizeplus.work
+site-auditor.online
+si-unique.com
skladvaz.ru
-slavic-magic.ru
+sky-mine.ru
slayerlife.com
-snsdeainavi.info
-soc-econom-problems.ru
-social-s-hhh.xyz
-social-vestnik.ru
-solicita.info
-soserfis.com
-spaceshipad.com
-spammen.de
-sstroy44.ru
-staff.prairiesouth.ca
-stard.shop
-staticfs.host
-stauga.altervista.org
-staynplay.net
-stpicks.com
-stroicol.net
-studiofmp.com
-svarbit.com
-swagbucks.com
-swinginwithme.ru
-takeflyte.com
-talant-factory.ru
-tattomedia.com
-td-33.ru
-tech4master.com
-template-kid.com
-thefarmergame.com
-thejournal.ru
-thetardistimes.ovh
-thewebsitetemplate.info
+slowmac.tech
+snjack.info
+socialseet.ru
+sockshare.net
+soc-proof.su
+sohoindia.net
+spravka130.ru
+stair.registrydomainservices.com
+stardevine.com
+stockspmb.info
+stroydetali.ru
+stroyplus.ru
+studiokamyk.com.pl
+sugvant.ru
+suhanpacktech.com
+sukirgenk.dvrlists.com
+superlist.biz
+svolze.com
+swimpool.ca
+swinger-mobil.net
+tam-gde-more.ru
+technopellet.gr
+terrafootwear.us
+thebestphotos.eu
+thecoolimages.net
+the-usa-games.blogspot.com
thewomenlife.com
-tizanidine4mgstreetprice.blogspot.com
-tizanidinedrugtest.blogspot.com
-tizanidinehclsideeffects.blogspot.com
-tizanidinerecreationaluse.blogspot.com
-tizanidinestreetvalue.blogspot.com
-tizanidineusedfor.blogspot.com
-tizanidinezanaflex.blogspot.com
-tomck.com
-torrentdownloadhub.com
+tizanidine4mguses.blogspot.com
+tizanidinevscyclobenzaprine.blogspot.com
+toolsky.com
+toonfamilies.net
+top250movies.ru
+torontoplumbinggroup.com
torrent-newgames.com
-torrents.cd
-tour-line.net
-tozup.com
-trackzapper.com
-tradedeals.biz
-trafficmp.com
-traffpartners.com
-trahic.ru
-trailer.cinemaflix.website
+torrents-tracker.com
+tracksz.co
+trafficstars.com
+trahvid.com
+treasuretrack-a.akamaihd.net
+trion.od.ua
trkdf.com
-trumpetedextremes.com
-trustaffs.com
-tsan.net
-turbabitload.weebly.com
-twinderbella.com
-tytoona.com
-tzritel.tk
-uamtrk.com
-uchebavchehii.ru
-uchetunet.su
-uhod-za-sobakoj.ru
-unimodemhalfduplefw.pen.io
-unrealcommander.biz
-upupa.net
-urengoy.pro
-ussearche.cf
-ustion.ru
-v24s.net
-v-doc.co
-videochat.life
-videochat.tv.br
-videogamesecrets.com
-videtubs.pl
-viktoria-center.ru
-villakohlanta.nu
-vintontech.info
-vksaver-all.ru
+truck-land.life
+truck-rece.life
+truemfilelj.gq
+trustl.life
+try-rx.com
+ttrraacckkrr.com
+tupper-shop.ru
+turn-up-life.life
+tvnewsclips.info
+twu.com.ua
+ua.tc
+uggsale.online
+ukkelberg.no
+uk-zheu20.ru
+ultimateclassicrock.com
+urlopener.blogspot.com.au
+venerologiya.com
+ventelnos.com
+via-energy-cumpara.com
+via-energy-order.com
+video-camer.com
+videochat.ph
+vielporno.net
+vikistars.com
+vipromoffers.com
+visionwell.com.cn
+visitcambridge.org
+vizag.kharkov.ua
+vkak.ru
vladhistory.com
-voprosotvet24.ru
-vote-up.ru
-w3data.co
-watchmyfb.pl
-weboptimizes.com
-website-speed-up.top
-weclipart.com
-whatzmyip.net
+vpnmouse.com
+walkme.com
+wareseeker.com
+warezaccess.com
+we-are-gamers.com
+webpromotion.ae
+website-analytics.online
+website-speed-check.site
+website-speed-up.site
+wedding0venues.tk
+westen-z.life
+wholesalecheapjerseysfree.com
wholesalejerseychinaoutlet.com
-wholesalejerseychinashop.com
-wholesalejerseysgaa.com
-wildworld.site
-wineration.com
-winwotgold.pl
-wordpresscore.com
-worldis.me
-wormix-cheats.ru
-wrona.it
-wufak.com
-xcombear.ru
-xdoza.com
-xfluro.com
-xitjw.info
-xlolitka.com
-xn------7cdbapdecfd4ak1bn0amjffj7afu3y.xn--p1ai
-xn--80aeahghtf8ac5i.xn--p1ai
-xn--c1acygb.xn--p1ai
-xn----itbeirbjbi7bc6bh2d.xn--p1ai
-xnxxandxvideos.com
-xtube.com
-xvideosbay.com
-xxx-treker.ru
-xxxtube69.com
-yaderenergy.ru
-yaoguangdj.com
-youbrainboost.asia
-youtubedownload.org
+wieseversa.no
+wleuaprpxuvr.ga
+writingservices17.blogspot.ru
+www.get-free-traffic-now.com
+x-lime.net
+xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai
+xn--80ahvj9e.xn--p1ai
+xuki.us
+xxlargepop.com
+xxx-cam.webcam
+xxxnatelefon.ru
+yeartwit.com
+yogamatsexpert.com
+youandcredit.ru
+yousense.info
+zakazvzloma.com
+zarabotat-na-sajte.ru
zastroyka.org
-zona-aqua.ru
-zooggames.com
+zebramart.ru
+zero1.it
+zeroredirect11.com
+zeroredirect7.com
+zhuravlev.info
+znakomstva-moskva77.ru
+zoogdisany.com
+zoompegs.com
diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list
index 044e3178b..2b22b37b6 100644
--- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list
+++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list
@@ -1,250 +1,250 @@
-03e.info
+123movies.love
+2ads.co.uk
+2drittel.de
3rbseyes.com
-991mostfm.co.id
-aa9d046aab36af4ff182f097f840430d51.com
-affrh2015.com
-agahinameh.com
-alienwheel.es
-angel1777.kz
-animebox.com.ua
-apiportalmorecom-a.akamaihd.net
-approved.su
-aproposde.com
-ap.senai.br
-arcteryxstore.online
-atout-energie-69.com
-azazaporn.com
-bambi.ck.ua
-bannerads.de
-battleforupdating.site
-bestssaker.com
-bidr.trellian.com
-biketank.ga
-bitcoinremote.com
-bloglag.com
-blogqpot.com
-blog.yam.com
-bodybuilding-shop.biz
-brus-vsem.ru
-budilneg.xyz
-businescoop.men
-buy-meds24.com
-buypanicdisorderpill.com
-buypillsorderonline.com
-buzzzg.men
-cactussoft.biz
-cartierbracelet.xyz
-cartierreplica.pw
-casinorewards.info
-cdnanalytics.xyz
-cheapjerseysap.com
-cheapmoncler.pw
-chimiver.info
-cleanallvirus.com
-cocyq.inwtrade.com
-compliance-margo.top
-compliance-olga.top
+3w24.com
+4webmasters.com
+999webdesign.com
+9icmzvn6.website
+agysya.ru
+animal-drawings.com
+apilinkswiftco-a.akamaihd.net
+arabsexxxtube.com
+ardimobileinfo.ml
+artel116.ru
+artpicso.com
+avek.ru
+avtocredit-legko.ru
+ayodhya.co
+azbaseimages.net
+bannerpower.com
+beachpics.com
+bestbrainprod.win
+betune.onlinewebshop.net
+bezlimitko.xyz
+bigames.online
+binomo.kz
+biznesluxe.ru
+bloggers.nl
+blog.koorg.ru
+boobsimge.com
+brainboostingsupplements.org
+braintobe.win
+brainzpod.win
+business-suggested.tk
+buyessay3.blogspot.ru
+buyessaynow.biz
+buyfriend.ru
+bvlgaribracelet.xyz
+bwlx.prepedu.cn
+cakemediahq.com.au
+callcafe.info
+call-of-duty.info
+candlewooddentalcentre.com.au
+castingbank.ru
+caveavins.fr
+cerev.info
+chinacheapelitejerseys.com
+choice-credit.ru
+cleanpcnow.com
+clickaider.com
+collegerentals.com
+compiko.info
+compliance-alexa.xyz
coolyarddecorations.com
-copro.pw
-customboxes4less.com
-dating-app.ru
-dayibiao.com
-dealwifi.com
-degerlund.net
-delfin-aqua.com.ua
-derevesendeco.com
-designdevise.com
+coop-gamers.ru
+cordstrap.cc
+counciltally.com
+creditnation.ru
+custom-electric-guitar.com
+cutalltheshit.com
+daymusam.com
+db.speedup.gdn
+descargar-musica-gratis.net
dieta-personalna.pl
dirtpics.pw
-divisioncore.com
-dogclothing.store
-domain2008.com
-donvito.unas.cz
-dotnetdotcom.org
-download-wallpaper.net
-eas-seo.com
+domaincheck.io
+dosugrostov.site
+dtm-spain.com
+earn-from-articles.com
+easytuningshop.ru
+e-c.al
+ecommerce-seo.com
+ekosmetyki.net.pl
+elektir.ru
elektrischezigarette2.devhub.com
-elektrischezigaretten1.blogse.nl
-elektroniksigaraankara.info
-eonpal.com
-errorfixing.space
-evogarage.com
-expertblog.info
-extstat.com
-ezigarettekaufen.myblog.de
-failingmarriege.blogspot.com
-fenoyl.batcave.net
-ff30236ddef1465f88547e760973d70a.com
-financehint.eu
-findthe.pet
+erotik0049.com
+eupornstar.info
+extrabot.com
+extremepornos.net
+fearcrow.com
+filkhbr.com
+filmgo.ru
+flashlarevista.com
+flipper.top
+florida-tourism.net
flowwwers.com
-fok.nl
footbalive.org
-frbizlist.com
-free-fbook-traffic.com
-free-traffic.xyz
-gamingspark.com
-gayxperience.com
+forex-indextop20.ru
+forex.osobye.ru
+gamedayhouse.com
+getdot.ru
get-free-traffic-now.com
-gnuetella.com
-golaya.pw
-googlsucks.com
-googst2.ru
-gotomontenegro.net
-gsasearchengineranker.space
-guildebzh.info
-hanwei.us
-hermesreplica.pw
-highspeed5.net
-hotkeys.com
-iamsport.org
-ilovevitaly.com
-incanto.in.ua
-innodgfdriverhm.aircus.com
-insta-add.pro
-inzn.ru
-ipostroika.ru
-irunfar.com
-iscblog.info
-jamiembrown.com
-jaxcube.info
-junglenet-a.akamaihd.net
-kino2018.cc
-kiskinhouse.com
-klikbonus.com
-konkursov.net
-kumuk.info
-lafourmiliaire.com
-letsart.ru
-lietaer.com
-light.ifmo.ru
-linkbux.ru
-linkredirect.biz
-m5home.ru
-magda-gadalka.ru
-magnetpress.sk
-makemoneyonline.com
-makis.nu
-marktforschung-stuttgart.com
-master-muznachas.ru
-masturbate.co.uk
-mega-bony2017.pl
-menstennisforums.com
-mesbuta.info
-metascephe.com
-mikrobiologies.ru
-mmgq.ru
-mncrftpcs.com
-mole.pluto.ro
-mustat.com
-muz-shoes.ru
-myftpupload.com
-naturalbreakthroughsresearch.com
-negociosdasha.com
-neobux-bg.info
-netfacet.net
-newtechspb.ru
-nicovideo.jp
+gg-arena.ru
+gkvector.ru
+glopages.ru
+gogalleryawesome.com
+graphid.com
+gratis-sexkontakte.com
+gsasearchenginerankerdiscount.com
+havepussy.com
+hit-men.men
+hothor.se
+igru-xbox.net
+i-hobot.ru
+internetproviderstucson.com
+ipornox.xxx
+istripper.com
+jerseywholesaleelitestore.com
+job.icivil.ir
+kakablog.net
+karachev-city.ru
+keyhantercume.com
+kino2018.club
+kinoflux.net
+kochanelli.com
+kosova.de
+kostenlos-sexvideos.com
+kredit-pod-zalog-krasnodar.ru
+kvartir-remont.biz
+laexotic.com
+linkpulse.com
+lkbennettoutlet.online
+lostfilm-online.ru
+louboutinreplica.pw
+macrotek.ru
+mansiondelrio.co
+matsdale.com
+mcadamssupplyco.com
+meccadumps.net
+message-warning.net
+mielec.pl
+minegam.com
+mmog-play.ru
+mobplayer.ru
+monclercoats.xyz
+monclervests.xyz
+moscow-clining.ru
+multgo.ru
+muztops.ru
+mvpicton.co.uk
+nalogovyy-kodeks.ru
+nash-krym.info
+nfljerseyscheapbiz.us
+nikhilbahl.com
+nikitabuch.com
+ningessaybe.me
+noclegonline.info
+normalegal.ru
november-lax.com
-ozas.net
-partybunny.ru
-paydayonlinecom.com
-picsforkeywordsuggestion.com
-pics-group.com
-pinwallpaper.top
-pizza-imperia.com
-porndl.org
-pornhive.org
-pornoxxx.com.mx
+online-templatestore.com
+onstrapon.purplesphere.in
+openstat.com
+palocco.it
+panamaforbeginners.com
+paradontozanet.ru
+perm-profnastil.ru
+php-market.ru
+pictures-and-images.com
+pinstake.com
+plastgranar.nu
+pogosh.com
+pojdelo.weebly.com
+pornogratisdiario.com
+pornohd1080.online
+pornokorol.com
+porno-transsexuals.ru
+porodasobak.net
+posible.net
pretty-mart.com
-privatefx.all4invest.info
-prostitutki-belgoroda.org
-pussyspace.net
-qwesa.ru
-regionshop.biz
-roma-kukareku.livejournal.com
+prizefestival.mobi
+prodvigator.ua
+professionalwritingservices15.blogspot.ru
+prostitutki-tolyatti.org
+qor360.com
+rainbowice.ru
+razamicroelectronics.com
+razorweb-a.akamaihd.net
+refudiatethissarah.info
+remont-fridge-tv.ru
+romhacking.ru
royal-betting.net
-sape.top
-sarf3omlat.com
-sarm.tk
-scripted.com
-searchwebknow-a.akamaihd.net
-seorussian.ru
-serpstat.com
-sevendays.com.ua
-share-buttons-for-free.com
-ship-marvel.co.ua
-shitmovs.com
-signal03.ru
-simpoed.ufop.br
-skutecznetabletkinaporostwlosow.pl
-skytraf.xyz
-smstraf.ru
-sovinsteel.ru
-sport-video-obzor.ru
-startufa.ru
-stauga.altervista.org
-sticken.co
-stickers-market.ru
-streetfire.net
-stuff-about-money.com
-suggest-keywords.com
-susanholtphotography.com
-svtrd.com
-syvertsen-da.no
-takethatad.com
-tengohydar.tk
+rutor.vip
+sakhboard.ru
+saveindex.xyz
+sbornik-zakonov.ru
+scansafe.net
+search.1and1.com
+seofied.com
+sexcamamateurchat.com
+sexvideo-sex.com
+sexyali.com
+siha.de
+site3.free-share-buttons.com
+siteonomy.com
+site-speed-checker.site
+sledstvie-veli.net
+slowmacfaster.trade
+social-button.xyz
+socialmadesimple.com
+social-s-hhh.xyz
+sortthemesitesby.com
+soviet-portal.do.am
+spabali.org
+spaceshipad.com
+sptslmtrafms.com
+ssconstruction.co
+stairliftsarea.com
+strag-invest.ru
+streha-metalko.si
+studiokamyk.com.pl
+sugarkun.com
+supermama.top
+svarbit.com
+swinon.site
+takeflyte.com
+tamada69.com
thecoolimages.net
+thegolfclub.info
+thetoiletpaper.com
timdreby.com
-tk-assortiment.ru
-tongkatmadura.info
-top10-online-games.com
-top250movies.ru
-torontoplumbinggroup.com
-touchmods.fr
-track2.shop
-tracxn.com
-trailer.cinemaflix.website
+tizanidine6mg.blogspot.com
+tizanidineshowupondrugtest.blogspot.com
+tnctrx.com
+topcar-krasnodar.ru
+torrent-newgames.com
trichizobswiv.agddns.net
-ttrraacckkrr.com
-twittruth.com
-unionmarkt.de
-upstore.me
-valoresito.com
-vavilone.com
-videochat.life
-videochat.ph
-videokrik.net
-vikistars.com
-vip2ch.com
-vipps.com.my
-vipsexfinders.com
-vizag.kharkov.ua
-vkonche.com
-vpnmouse.com
-wallpapers-best.com
-wcb.su
-webscouter.net
-website-audit.com.ua
-wesharepics.info
-wholesalejerseys-cheapest.com
-word-vorlagen.xyz
-writingservices17.blogspot.ru
-www.888.com
-xdoza.com
-xgames-04.com
-xingzi-vision.com
-xkaz.org
-xn-----7kcabaipgeakzcss7bjdqdwpfnhv.xn--p1ai
-xn----itbkqkfiq.xn--p1ai
-xtube.com
+trustedhealthtips.com
+unitygame3d.com
+urlcut.ru
+userequip.com
+vashsvet.com
+vclicks.net
+video-chat.in
+virtuagirl.com
+voditeltrezviy.ru
+vozbujdenie.com
+wapsite.me
+websitevaluebot.com
+wedding0venues.tk
+whatistizanidinehclusedfor.blogspot.com
+whereiskentoday.com
+win-spy.com
+wladimirpayen.com
+wosik-dach.service-for-web.de
+xn--80aebbcbcdemfkhba4byaehoejh8dza3v.xn--p1ai
+xnxx-n.com
xxart.ru
-xxx-treker.ru
-xz618.com
-youandcredit.ru
-yourporngay.com
-youtoner.it
-ytmnd.com
-zaobao.com.sg
-zhongwenlink.com
-zverokruh-shop.cz
+yellowstonesafaritours.com
+zeg-distribution.com
+zeroredirect7.com
diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt
index f371d9e5f..525a76fd6 100644
--- a/.dev-tools/test_units/ratelimittest-beta.txt
+++ b/.dev-tools/test_units/ratelimittest-beta.txt
@@ -1,3 +1,24 @@
+
+503 Service Temporarily Unavailable
+
+503 Service Temporarily Unavailable
+
nginx/1.14.2
+
+
+
+503 Service Temporarily Unavailable
+
+503 Service Temporarily Unavailable
+
nginx/1.14.2
+
+
+
+503 Service Temporarily Unavailable
+
+503 Service Temporarily Unavailable
+
nginx/1.14.2
+
+
@@ -8,35 +29,7 @@
Welcome to the NginX Ultimate Bad Bot Blocker
This is merely used for testing !!!
-
-503 Service Temporarily Unavailable
-
-503 Service Temporarily Unavailable
-
nginx/1.14.2
-
-
-
-503 Service Temporarily Unavailable
-
-503 Service Temporarily Unavailable
-
nginx/1.14.2
-
-
-
-503 Service Temporarily Unavailable
-
-503 Service Temporarily Unavailable
-
nginx/1.14.2
-
-
-
-503 Service Temporarily Unavailable
-
-503 Service Temporarily Unavailable
-
nginx/1.14.2
-
-
-
+