diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
index 61a5d54bc..314f39712 100644
--- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1727
-### Updated: Tue Jul 2 08:55:06 SAST 2019
+### Version: V3.2019.07.1728
+### Updated: Tue Jul 2 09:44:26 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 465ec1626..eabb2e30b 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 1;
+104.198.131.58 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index 290c48e19..6bad13743 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 0;
+104.198.131.58 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index 61a5d54bc..314f39712 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1727
-### Updated: Tue Jul 2 08:55:06 SAST 2019
+### Version: V3.2019.07.1728
+### Updated: Tue Jul 2 09:44:26 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 465ec1626..eabb2e30b 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 1;
+104.198.131.58 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index 290c48e19..6bad13743 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 0;
+104.198.131.58 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
index 61a5d54bc..314f39712 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1727
-### Updated: Tue Jul 2 08:55:06 SAST 2019
+### Version: V3.2019.07.1728
+### Updated: Tue Jul 2 09:44:26 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
index 465ec1626..eabb2e30b 100644
--- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 1;
+104.198.131.58 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
index 290c48e19..6bad13743 100644
--- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 0;
+104.198.131.58 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
index 61a5d54bc..314f39712 100644
--- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1727
-### Updated: Tue Jul 2 08:55:06 SAST 2019
+### Version: V3.2019.07.1728
+### Updated: Tue Jul 2 09:44:26 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/distribution_tests_missing_includes/arch-install.log b/.dev-tools/distribution_tests_missing_includes/arch-install.log
index e69de29bb..fdcb57196 100644
--- a/.dev-tools/distribution_tests_missing_includes/arch-install.log
+++ b/.dev-tools/distribution_tests_missing_includes/arch-install.log
@@ -0,0 +1,32 @@
+[1m[35m----------------------------------------------------------
+[1m[35mExecute install-ngxblocker - Arch Linux (Missing includes)
+[1m[35m----------------------------------------------------------
+
+sending incremental file list
+
+sent 350 bytes received 14 bytes 728.00 bytes/sec
+total size is 47.02K speedup is 129.17
+Checking url: https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/include_filelist.txt
+
+Creating directory: /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/conf.d
+Creating directory: /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/bots.d
+
+REPO = https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master
+
+Downloading [FROM]=> [REPO]/conf.d/globalblacklist.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/conf.d/globalblacklist.conf...OK
+Downloading [FROM]=> [REPO]/conf.d/botblocker-nginx-settings.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/conf.d/botblocker-nginx-settings.conf...OK
+
+REPO = https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master
+
+Downloading [FROM]=> [REPO]/bots.d/blockbots.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/bots.d/blockbots.conf...OK
+Downloading [FROM]=> [REPO]/bots.d/ddos.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/bots.d/ddos.conf...OK
+Downloading [FROM]=> [REPO]/bots.d/custom-bad-referrers.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/bots.d/custom-bad-referrers.conf...OK
+Downloading [FROM]=> [REPO]/bots.d/bad-referrer-words.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/bots.d/bad-referrer-words.conf...OK
+Downloading [FROM]=> [REPO]/bots.d/blacklist-ips.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/bots.d/blacklist-ips.conf...OK
+Downloading [FROM]=> [REPO]/bots.d/blacklist-user-agents.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/bots.d/blacklist-user-agents.conf...OK
+Downloading [FROM]=> [REPO]/bots.d/whitelist-domains.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/bots.d/whitelist-domains.conf...OK
+Downloading [FROM]=> [REPO]/bots.d/whitelist-ips.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/bots.d/whitelist-ips.conf...OK
+Nothing to update for directory: /usr/local/sbin
+Setting mode: 700 => /usr/local/sbin/install-ngxblocker
+Setting mode: 700 => /usr/local/sbin/setup-ngxblocker
+Setting mode: 700 => /usr/local/sbin/update-ngxblocker
diff --git a/.dev-tools/distribution_tests_missing_includes/arch-setup.log b/.dev-tools/distribution_tests_missing_includes/arch-setup.log
index e69de29bb..6845c48fe 100644
--- a/.dev-tools/distribution_tests_missing_includes/arch-setup.log
+++ b/.dev-tools/distribution_tests_missing_includes/arch-setup.log
@@ -0,0 +1,23 @@
+
+[1m[35m--------------------------------------------------------
+[1m[35mExecute setup-ngxblocker - Arch Linux (Missing includes)
+[1m[35m--------------------------------------------------------
+
+Checking url: https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/include_filelist.txt
+
+inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/conf.d/globalblacklist.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/nginx.conf
+inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/conf.d/botblocker-nginx-settings.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/nginx.conf
+
+Whitelisting ip: 34.66.178.120 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/bots.d/whitelist-ips.conf
+
+Checking for missing includes:
+
+Checking url: https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/include_filelist.txt
+
+Nothing to update for directory: /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/conf.d
+Nothing to update for directory: /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/bots.d
+Nothing to update for directory: /usr/local/sbin
+Setting mode: 700 => /usr/local/sbin/install-ngxblocker
+Setting mode: 700 => /usr/local/sbin/setup-ngxblocker
+Setting mode: 700 => /usr/local/sbin/update-ngxblocker
+[1m[32mUpdating bots.d path[0m: [1m[37m/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/bots.d => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/conf.d/globalblacklist.conf[0m
diff --git a/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/nginx.conf b/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/nginx.conf
index 2f2cf8820..8f1ee9534 100644
--- a/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/nginx.conf
+++ b/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/nginx.conf
@@ -69,6 +69,12 @@ http {
# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
# include fastcgi_params;
#}
+
+
+# Nginx Bad Bot Blocker Includes
+ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/arch/etc/nginx/conf.d/globalblacklist.conf;
+
# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
diff --git a/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/nginx.conf b/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/nginx.conf
index 01a4a2187..f67a73969 100644
--- a/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/nginx.conf
+++ b/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/nginx.conf
@@ -58,8 +58,13 @@ http {
# Virtual Host Configs
##
- include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
+
+
+
+# Nginx Bad Bot Blocker Includes
+ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/conf.d/botblocker-nginx-settings.conf;
+ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/conf.d/globalblacklist.conf;
}
diff --git a/.dev-tools/distribution_tests_missing_includes/ubuntu1604-install.log b/.dev-tools/distribution_tests_missing_includes/ubuntu1604-install.log
index e69de29bb..9d86a9441 100644
--- a/.dev-tools/distribution_tests_missing_includes/ubuntu1604-install.log
+++ b/.dev-tools/distribution_tests_missing_includes/ubuntu1604-install.log
@@ -0,0 +1,32 @@
+[1m[35m------------------------------------------------------------------
+[1m[35mExecute install-ngxblocker - Ubuntu 16.04.6 LTS (Missing includes)
+[1m[35m------------------------------------------------------------------
+
+sending incremental file list
+etc/nginx/nginx.conf
+
+sent 1.49K bytes received 41 bytes 3.05K bytes/sec
+total size is 19.83K speedup is 12.99
+Checking url: https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/include_filelist.txt
+
+Creating directory: /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/bots.d
+
+REPO = https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master
+
+Downloading [FROM]=> [REPO]/conf.d/globalblacklist.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/conf.d/globalblacklist.conf...OK
+Downloading [FROM]=> [REPO]/conf.d/botblocker-nginx-settings.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/conf.d/botblocker-nginx-settings.conf...OK
+
+REPO = https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master
+
+Downloading [FROM]=> [REPO]/bots.d/blockbots.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/bots.d/blockbots.conf...OK
+Downloading [FROM]=> [REPO]/bots.d/ddos.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/bots.d/ddos.conf...OK
+Downloading [FROM]=> [REPO]/bots.d/custom-bad-referrers.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/bots.d/custom-bad-referrers.conf...OK
+Downloading [FROM]=> [REPO]/bots.d/bad-referrer-words.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/bots.d/bad-referrer-words.conf...OK
+Downloading [FROM]=> [REPO]/bots.d/blacklist-ips.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/bots.d/blacklist-ips.conf...OK
+Downloading [FROM]=> [REPO]/bots.d/blacklist-user-agents.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/bots.d/blacklist-user-agents.conf...OK
+Downloading [FROM]=> [REPO]/bots.d/whitelist-domains.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/bots.d/whitelist-domains.conf...OK
+Downloading [FROM]=> [REPO]/bots.d/whitelist-ips.conf [TO]=> /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/bots.d/whitelist-ips.conf...OK
+Nothing to update for directory: /usr/local/sbin
+Setting mode: 700 => /usr/local/sbin/install-ngxblocker
+Setting mode: 700 => /usr/local/sbin/setup-ngxblocker
+Setting mode: 700 => /usr/local/sbin/update-ngxblocker
diff --git a/.dev-tools/distribution_tests_missing_includes/ubuntu1604-setup.log b/.dev-tools/distribution_tests_missing_includes/ubuntu1604-setup.log
index e69de29bb..85cc41cd2 100644
--- a/.dev-tools/distribution_tests_missing_includes/ubuntu1604-setup.log
+++ b/.dev-tools/distribution_tests_missing_includes/ubuntu1604-setup.log
@@ -0,0 +1,23 @@
+
+[1m[35m----------------------------------------------------------------
+[1m[35mExecute setup-ngxblocker - Ubuntu 16.04.6 LTS (Missing includes)
+[1m[35m----------------------------------------------------------------
+
+Checking url: https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/include_filelist.txt
+
+inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/conf.d/globalblacklist.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/nginx.conf
+inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/conf.d/botblocker-nginx-settings.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/nginx.conf
+
+Whitelisting ip: 35.193.14.140 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/bots.d/whitelist-ips.conf
+
+Checking for missing includes:
+
+Checking url: https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/include_filelist.txt
+
+Nothing to update for directory: /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/conf.d
+Nothing to update for directory: /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/bots.d
+Nothing to update for directory: /usr/local/sbin
+Setting mode: 700 => /usr/local/sbin/install-ngxblocker
+Setting mode: 700 => /usr/local/sbin/setup-ngxblocker
+Setting mode: 700 => /usr/local/sbin/update-ngxblocker
+[1m[32mUpdating bots.d path[0m: [1m[37m/home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/bots.d => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-16.04/etc/nginx/conf.d/globalblacklist.conf[0m
diff --git a/.dev-tools/distribution_tests_missing_includes/ubuntu1804-install.log b/.dev-tools/distribution_tests_missing_includes/ubuntu1804-install.log
index 6a9048ae3..8662937b2 100644
--- a/.dev-tools/distribution_tests_missing_includes/ubuntu1804-install.log
+++ b/.dev-tools/distribution_tests_missing_includes/ubuntu1804-install.log
@@ -5,8 +5,8 @@
sending incremental file list
etc/nginx/nginx.conf
-sent 2.01K bytes received 43 bytes 4.10K bytes/sec
-total size is 20.46K speedup is 9.99
+sent 2.04K bytes received 43 bytes 4.17K bytes/sec
+total size is 20.46K speedup is 9.81
Checking url: https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/include_filelist.txt
Creating directory: /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/bots.d
diff --git a/.dev-tools/distribution_tests_missing_includes/ubuntu1804-setup.log b/.dev-tools/distribution_tests_missing_includes/ubuntu1804-setup.log
index 8e6903795..5bf03da9b 100644
--- a/.dev-tools/distribution_tests_missing_includes/ubuntu1804-setup.log
+++ b/.dev-tools/distribution_tests_missing_includes/ubuntu1804-setup.log
@@ -8,7 +8,7 @@ Checking url: https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-ba
inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/conf.d/globalblacklist.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/nginx.conf
inserting: include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/conf.d/botblocker-nginx-settings.conf; => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/nginx.conf
-Whitelisting ip: 104.154.120.187 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/bots.d/whitelist-ips.conf
+Whitelisting ip: 35.188.73.34 => /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/.dev-tools/distribution_tests_missing_includes/ubuntu-18.04/etc/nginx/bots.d/whitelist-ips.conf
Checking for missing includes:
diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template
index 61a5d54bc..314f39712 100644
--- a/.dev-tools/globalblacklist-testing.template
+++ b/.dev-tools/globalblacklist-testing.template
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1727
-### Updated: Tue Jul 2 08:55:06 SAST 2019
+### Version: V3.2019.07.1728
+### Updated: Tue Jul 2 09:44:26 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template
index 3a9a088e9..4d1c5803f 100755
--- a/.dev-tools/globalblacklist.template
+++ b/.dev-tools/globalblacklist.template
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1727
-### Updated: Tue Jul 2 08:55:04 SAST 2019
+### Version: V4.2019.07.1728
+### Updated: Tue Jul 2 09:44:24 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
index c0fc69668..2b15f263d 100644
--- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
@@ -45,4 +45,4 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.192.85.2 0;
+34.68.144.114 0;
diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
index 3a9a088e9..4d1c5803f 100644
--- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1727
-### Updated: Tue Jul 2 08:55:04 SAST 2019
+### Version: V4.2019.07.1728
+### Updated: Tue Jul 2 09:44:24 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 465ec1626..4dddc08a6 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 1;
+35.193.14.140 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index 290c48e19..40cb4d7a8 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 0;
+35.193.14.140 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index 3a9a088e9..4d1c5803f 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1727
-### Updated: Tue Jul 2 08:55:04 SAST 2019
+### Version: V4.2019.07.1728
+### Updated: Tue Jul 2 09:44:24 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 465ec1626..4dddc08a6 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 1;
+35.193.14.140 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index 290c48e19..40cb4d7a8 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 0;
+35.193.14.140 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
index 3a9a088e9..4d1c5803f 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1727
-### Updated: Tue Jul 2 08:55:04 SAST 2019
+### Version: V4.2019.07.1728
+### Updated: Tue Jul 2 09:44:24 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
index 465ec1626..4dddc08a6 100644
--- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 1;
+35.193.14.140 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
index 290c48e19..40cb4d7a8 100644
--- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 0;
+35.193.14.140 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
index 3a9a088e9..4d1c5803f 100644
--- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1727
-### Updated: Tue Jul 2 08:55:04 SAST 2019
+### Version: V4.2019.07.1728
+### Updated: Tue Jul 2 09:44:24 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
index 503b1de7a..f0d7ffecc 100644
--- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
@@ -45,4 +45,4 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.224.112.202 0;
+35.192.187.174 0;
diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
index def4050db..78bf42a1c 100644
--- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1727
-### Updated: Tue Jul 2 08:55:04 SAST 2019
+### Version: V4.2019.07.1728
+### Updated: Tue Jul 2 09:44:24 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
index 478d9d03f..137da33fd 100644
--- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
@@ -45,5 +45,5 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.202.145.110 0;
-35.192.85.2 0;
+34.66.178.120 0;
+34.68.144.114 0;
diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
index 3a9a088e9..4d1c5803f 100644
--- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1727
-### Updated: Tue Jul 2 08:55:04 SAST 2019
+### Version: V4.2019.07.1728
+### Updated: Tue Jul 2 09:44:24 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
index e3688dbe0..403c2e793 100644
--- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
@@ -45,5 +45,5 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.184.96.71 0;
-35.202.145.110 0;
+35.193.14.140 0;
+104.198.131.58 0;
diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
index 3a9a088e9..4d1c5803f 100644
--- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1727
-### Updated: Tue Jul 2 08:55:04 SAST 2019
+### Version: V4.2019.07.1728
+### Updated: Tue Jul 2 09:44:24 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf
index 465ec1626..eabb2e30b 100644
--- a/.dev-tools/test_units/blacklist-ips.conf
+++ b/.dev-tools/test_units/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 1;
+104.198.131.58 1;
127.0.0.1 1;
diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list
index a2a0583f4..576310bd8 100644
--- a/.dev-tools/test_units/random-bots-for-test-quick.list
+++ b/.dev-tools/test_units/random-bots-for-test-quick.list
@@ -1,10 +1,10 @@
-Boardreader
-com.plumanalytics
-CrunchBot
-Firefox/7.0
-Humanlinks
-Indy Library
-Pump
-RankActiveLinkBot
-TurnitinBot
-WebGo IS
+DIIbot
+RankFlex
+SearchmetricsBot
+seoscanners
+Sqlmap
+Sqlworm
+TeleportPro
+WebLeacher
+Webshag
+WWW-Collector-E
diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list
index 401f29b9e..c5450e0c4 100644
--- a/.dev-tools/test_units/random-bots-for-test.list
+++ b/.dev-tools/test_units/random-bots-for-test.list
@@ -1,250 +1,250 @@
-360Spider
-404enemy
+404checker
Aboundex
-ADmantX
+Acunetix
+AfD-Verbotsverfahren
AhrefsBot
AIBOT
+Aipbot
Alexibot
Alligator
+AllSubmitter
AlphaBot
Anarchie
-Apexoo
-Asterias
Attach
-BackDoorBot
+Backlink-Ceck
backlink-check
-BackStreet
+BackWeb
Badass
+Barkrowler
BBBike
-BDCbot
BDFetch
+BetaBot
Bitacle
-Black Hole
-BlackWidow
-BlowFish
+Blackboard
+BLEXBot
+Blow
+Boardreader
Bolt
BotALot
-BunnySlippers
+Brandwatch
+Buddy
+BuiltBotTough
+Bullseye
BuzzSumo
-Calculon
CATExplorador
-CCBot
-CheeseBot
+Cegbfeieh
+CheTeam
ChinaClaw
-Chlooe
-Cloud mapping
-coccocbot-web
+Cliqzbot
+Cogentbot
cognitiveseo
-Collector
-com.plumanalytics
-Craftbot
+CopyRightCheck
+Copyscape
+Cosmos
+crawler4j
crawler.feedback
-Crescent
-DatabaseDriverMysqli
+crawl.sogou.com
+CrazyWebCrawler
+CSHttp
+Curious
DBLBot
+demandbase-bot
Deusu
+Digincore
+DIIbot
Dirbuster
-DnyzBot
-DomainAppender
+Discobot
+Dispatch
DomainSigmaCrawler
Dotbot
Download Wonder
-DSearch
EasyDL
+Ebingbong
ECCP/1.0
-EMail Siphon
-Exabot
+Ecxi
+EroCrawler
Express WebPictures
Extractor
-EyeNetIE
-facebookscraper
+Extreme Picture Finder
+Ezooms
FHscan
-Firefox/7.0
-Freeuploader
-GetRight
-Go-Ahead-Got-It
-Gotit
+Fimap
+Foobot
+FrontPage
+GalaxyBot
+GermCrawler
+Getintent
+GetWeb
+Gigabot
+Go!Zilla
Grabber
-GrabNet
Grafula
-GrapeFX
-GrapeshotCrawler
GridBot
+Haansoft
+HaosouSpider
Harvest
-HEADMasterSEO
-Heritrix
Hloader
-HMView
-HTMLparser
HTTP::Lite
HTTrack
+Id-search
IlseBot
-InternetSeer
+IndeedBot
+InfoNaviRobot
+InfoTekies
+Intelliseek
+InterGET
+Internet Ninja
+internetVista monitor
+ips-agent
Iria
-Iskanie
+IRLbot
IstellaBot
+JamesBOT
JennyBot
-Jetty
JikeSpider
-Jorgee
-JustView
+JOC Web Spider
+Joomla
+Jyxobot
Kenjin Spider
-Larbin
+LeechFTP
LeechGet
-LexiBot
LibWeb
-Libwhisker
-Lightspeedsystems
-Likse
-LinkScan
+LinksManager
LinkWalker
LinqiaMetadataDownloaderBot
LinqiaRSSBot
-Litemage_walker
+Lipperhey
Lmspider
LNSpiderguy
-Ltx71
-lwp-request
lwp-trivial
-Mail.RU_Bot
+Magnet
+magpie-crawler
+Majestic SEO
Masscan
+Mass Downloader
meanpathbot
-MeanPath Bot
+Mediatoolkitbot
Metauri
Microsoft URL Control
-MIIxpc
+MIDown tool
+Mister PiX
MJ12bot
-MSIECrawler
-Nameprotect
-NearSite
+Mojeek
+Morfeus Fucking Scanner
+MS Web Services Client Protocol
+Musobot
+Nessus
+NetAnts
Netcraft
netEstate NE Crawler
-NetMechanic
-Ninja
+NetZIP
+Nibbler
+Niki-bot
+Nikto
+NimbleCrawler
Nmap
-Nutch
+NPbot
Octopus
-OnCrawl
+Offline Explorer
+Offline Navigator
Openfind
-PageAnalyzer
-Page Analyzer
+OrangeBot
+OutfoxBot
PageGrabber
-Pandalytics
Panscient
+Papa Foto
PeoplePal
-Picscout
-PleaseCrawl
+Picsearch
+PictureFinder
+Pi-Monster
plumanalytics
-Pockey
POE-Component-Client-HTTP
ProPowerBot
ProWebWalker
Psbot
-Quick-Crawler
-RankingBot
+Pump
+PxBroker
+PyCurl
+QueryN Metasearch
+RankActive
+RankFlex
RankingBot2
-Rankivabot
RankurBot
-RealDownload
Reaper
+RebelMouse
Recorder
-ReGet
RepoMonkey
-Ripper
-RocketCrawler
-s1z.ru
+Rogerbot
SalesIntelligent
-SBIder
-ScanAlert
+Scanbot
scan.lol
+ScoutJet
ScreenerBot
-Searchestate
-SearchmetricsBot
-Semrush
-SemrushBot
SEOkicks
-SEOkicks-Robot
-SEOlyticsCrawler
-seoscanners
-Shodan
+SEOprofiler
+SeoSiteCheckup
+sexsearcher
Siphon
SISTRIX
-Sitebeam
-Siteimprove
-SiteLockSpider
-SiteSucker
-Site Sucker
SlySearch
+SMTBot
Snapbot
-Snoopy
-SocialRankIOBot
-Sociscraper
-sogouspider
+Sottopop
+SpaceBison
+Spammen
SpankBot
-Spanner
-sp_auditbot
Spbot
SputnikBot
spyfu
-Steeler
Sucker
Sucuri
SuperBot
SuperHTTP
-SurveyBot
-Swiftbot
-Szukacz
-T8Abot
-Telesoft
-Telesphorep
+sysscan
The Intraformant
-TightTwatBot
+TheNomad
+Titan
Toata
Trendiction
-trendiction.de
-TurnitinBot
-TwengaBot
-Twice
+Trendictionbot
+True_Robot
Typhoeus
-UnisterBot
+Upflow
Vacuum
-Vagabondo
-VB Project
-VCI
-VeriCiteCrawler
-VidibleScraper
VoidEYE
-Voltron
Wallpapers/3.0
-WASALive-Bot
+WBSearchBot
+Web Auto
WebCollage
Web Collage
-WebCopier
+WEBDAV
WebEnhancer
Web Enhancer
WebFetch
-Web Fetch
-Web Fuck
-WebLeacher
-webmeup-crawler
-Web Sauger
+WebFuck
+WebImageCollector
+Web Pix
+WebReaper
+WebsiteExtractor
+Website Quester
WebStripper
-WebSucker
-WeSEE
-Whatweb
+Web Sucker
+WebWhacker
+WebZIP
Who.is Bot
-WiseGuys Robot
-Woobot
-Wprecon
+Widow
+WISENutbot
+Wonderbot
WPScan
-WWW-Mechanize
+WWW-Collector-E
+WWW::Mechanize
x09Mozilla
Xaldon WebSpider
Xaldon_WebSpider
-Xenu
-xpymep1.exe
-Zade
+YoudaoBot
Zauba
zauba.io
+Zeus
+Zitebot
+ZmEu
ZumBot
-ZyBorg
diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list
index 7d12923a3..135354b1b 100644
--- a/.dev-tools/test_units/random-bots-for-whitelist-test.list
+++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list
@@ -1,250 +1,250 @@
360Spider
+404checker
404enemy
Abonti
Aboundex
-ADmantX
-AfD-Verbotsverfahren
-AIBOT
-Aipbot
-Alexibot
+Acunetix
+AhrefsBot
+AiHitBot
+Alligator
+AlphaBot
+Apexoo
+archive.org_bot
Asterias
+autoemailspider
+BackDoorBot
+backlink-check
BackStreet
-BackWeb
+Badass
+Barkrowler
BatchFTP
-Battleztar Bazinga
+BBBike
BDCbot
BDFetch
-Black Hole
-BlackWidow
-BLEXBot
+Bigfoot
+Bitacle
BlowFish
Bolt
-BotALot
-Brandprotect
+Buddy
BuiltBotTough
-CATExplorador
+BunnySlippers
+BuzzSumo
CazoodleBot
CCBot
+Cegbfeieh
+Chlooe
+Claritybot
Cliqzbot
-Copyscape
-Cosmos
+coccocbot-web
+cognitiveseo
+Collector
crawler4j
-crawler.feedback
crawl.sogou.com
CrazyWebCrawler
Crescent
+CrunchBot
CSHttp
-Custo
+DBLBot
demandbase-bot
+Demon
Devil
+Dirbuster
Disco
-DittoSpyder
-DomainStatsBot
-Dragonfly
+Dispatch
+DnyzBot
+DomainAppender
+DomainCrawler
+DomainSigmaCrawler
+Drip
DSearch
DTS Agent
EasyDL
Ebingbong
-Ecxi
-EMail Wolf
-ExtLinksBot
-Extreme Picture Finder
+EroCrawler
+Express WebPictures
+Extractor
EyeNetIE
+Ezooms
FDM
-FemtosearchBot
+FHscan
+Fimap
Firefox/7.0
-FlashGet
-Flunky
-Freeuploader
-FyberSpider
GalaxyBot
Genieo
GermCrawler
Getintent
-GetWeb
-Gotit
+GetRight
+G-i-g-a-b-o-t
+Go-Ahead-Got-It
+Go!Zilla
Grabber
GrabNet
Grafula
-GrapeshotCrawler
+GT::WWW
+Havij
Heritrix
+HMView
+HTMLparser
+HTTP::Lite
HTTrack
Humanlinks
-IDBot
+Iblog
+Id-search
IlseBot
Image Fetch
Image Sucker
IndeedBot
-Indy Library
InfoTekies
-instabid
InterGET
-internetVista monitor
ips-agent
Iria
IRLbot
-Iskanie
IstellaBot
Jbrofuzz
-JennyBot
JetCar
-Jetty
JikeSpider
-Joomla
-Jorgee
-JustView
-Jyxobot
-Kenjin Spider
Keyword Density
-Lanshanbot
Larbin
-Likse
-LinkpadBot
-LinkScan
-LinkWalker
-LinqiaScrapeBot
+LeechFTP
+Lftp
+Libwhisker
+Lightspeedsystems
+LinksManager
+LinqiaRSSBot
Lipperhey
-Lmspider
-LWP::Simple
-lwp-trivial
-Magnet
-Mag-Net
-Mail.RU_Bot
-Majestic SEO
-MarkMonitor
+Litemage_walker
+Ltx71
+lwp-request
+magpie-crawler
+MarkWatch
Mass Downloader
-MauiBot
+Mata Hari
Meanpathbot
-MeanPath Bot
-Mediatoolkitbot
mediawords
-MFC_Tear_Sample
+MegaIndex.ru
Microsoft URL Control
+MIDown tool
MIIxpc
Mojeek
Morfeus Fucking Scanner
-Msrabot
-MS Web Services Client Protocol
-muhstik-scan
-Musobot
-Nameprotect
+Mr.4x3
+MSFrontPage
+Name Intelligence
Navroad
-NearSite
-Nessus
+Needle
+Netcraft
NetMechanic
NetSpider
+Nettrack
Netvibes
-NextGenSearchBot
-Nibbler
-Niki-bot
-NimbleCrawler
+NetZIP
+NICErsPRO
+Nikto
Ninja
-oBot
-Offline Navigator
+Nmap
+Nutch
+Octopus
+Offline Explorer
OnCrawl
-Openfind
OpenLinkProfiler
-OpenVAS
-OrangeSpider
OutclicksBot
OutfoxBot
+Page Analyzer
PageGrabber
-PageScorer
-Pandalytics
-Panscient
-Pavuk
-PECL::HTTP
-Picscout
+page scorer
+pcBrowser
+PeoplePal
Picsearch
Pimonster
-Pi-Monster
-plumanalytics
-Pockey
-ProWebWalker
+Pixray
+PleaseCrawl
+Probethenet
+ProPowerBot
+Pump
PxBroker
-PyCurl
+QueryN Metasearch
RankActive
-RankActiveLinkBot
RankingBot2
-RankurBot
+RealDownload
Reaper
RebelMouse
-RedesScrapy
ReGet
-s1z.ru
+RepoMonkey
+Ripper
+Rogerbot
SBIder
ScanAlert
ScoutJet
-Screaming
+Scrapy
ScreenerBot
+Searchestate
SearchmetricsBot
-SEOstats
-Shodan
-Siphon
-Sitebeam
+Semrush
+SemrushBot
+SEOkicks
+SeoSiteCheckup
+serpstatbot
SiteExplorer
SiteLockSpider
+SiteSnagger
+SiteSucker
+Sitevigil
+SlySearch
Snake
-Snapbot
-Sosospider
-SpaceBison
+SocialRankIOBot
+Sociscraper
+Spammen
Spanner
-sp_auditbot
+Spbot
+Spinn3r
Sqlmap
-Surfbot
-Suzuran
-Swiftbot
+Sqworm
+SuperBot
+SuperHTTP
+SurveyBot
+sysscan
+T0PHackTeam
T8Abot
tAkeOut
-Teleport
TeleportPro
-Telesphorep
-Thumbor
-TightTwatBot
+TheNomad
+Titan
Tracemyfile
-Trendiction
-Trendictionbot
-True_Robot
-Turingos
+trendiction.com
Turnitin
-TurnitinBot
Upflow
-URLy Warning
-Vacuum
-VeriCiteCrawler
-VidibleScraper
-Virusdie
+VB Project
VoidEYE
-Wallpapers/3.0
-Web Auto
+Voil
+Voltron
WebCollage
-Web Collage
-WebCopier
-WEBDAV
-WebFetch
+WebEnhancer
Web Fetch
+Web Fuck
WebGo IS
WebImageCollector
WebLeacher
-webmeup-crawler
+WebmasterWorldForumBot
+WebPix
WebReaper
WebSauger
-Webshag
-WebsiteExtractor
-Website Quester
+WebsiteQuester
+WebStripper
Web Sucker
-Whatweb
-WISENutbot
+WebWhacker
+WeSEE
+Widow
+WiseGuys Robot
Wonderbot
-Wotbox
Wprecon
-WPScan
WWW-Collector-E
-WWW::Mechanize
x09Mozilla
-Xaldon_WebSpider
+Xaldon WebSpider
Xenu
+YoudaoBot
+Zauba
Zermelo
-Zeus
+zgrab
ZmEu
ZumBot
+ZyBorg
diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list
index fc29d20fa..75a40a63d 100644
--- a/.dev-tools/test_units/random-referrers-for-test-quick.list
+++ b/.dev-tools/test_units/random-referrers-for-test-quick.list
@@ -1,10 +1,10 @@
-amazon-adsystem.com
-bestpriceninja.com
-dogbreedspicture.net
-enhand.se
-free-deals.faith
-kartiniresto.com
-neobux-bg.info
-recipedays.ru
-space-worry.ml
-voloomoney.com
+adssafeprotected.com
+apisecretsaucebi-a.akamaihd.net
+countercrazy.com
+db.speedup.gdn
+drpornogratisx.xxx
+ecommerce-seo.com
+gameonasia.com
+page2rss.com
+park.above.com
+pro-okis.ru
diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list
index 26271fd81..a74cc177d 100644
--- a/.dev-tools/test_units/random-referrers-for-test.list
+++ b/.dev-tools/test_units/random-referrers-for-test.list
@@ -1,500 +1,500 @@
-00author.com
-11pikachu.ru
-123locker.com
-1webmaster.ml
-2345.com
-24-ak.ru
-3dracergames.com
-4pp13.com
-5-steps-to-start-business.com
-777-club.ru
-aa14ab57a3339c4064bd9ae6fad7495b5f.com
-above.com
-academia-nsk.org
-acads.net
-actices.com
-actulite.com
-adrunnr.com
-adult-shop.com.ua
+00webcams.com
+0912701309f8ce.com
+1000n1.ru
+13tabs.com
+1stat.ru
+21h2o.com
+2ads.co.uk
+3weekdiet.com
+76brighton.co.uk
+83net.jp
+8lufu.com
+8si.ru
+999webdesign.com
+a1security.com.ua
+a96527gi.beget.tech
+aanapa.ru
+aaronabel.com
+accessoires-mode.in
+admanaerofoil.com
+adobereader-free.ru
+adssafeprotected.com
+adultgalls.com
advancedcleaner.com
-agahinameh.com
-air-edem.ru
-akama.com
-alborzan.com
+affordablewebsitesandmobileapps.com
+aghanyna.com
+ahrntal.verymes.xyz
+aibolita.com
+akvamaster.dp.ua
+alevinefesleri.com
+alf-img.com
alienwheel.es
-allwomen.info
-alphaforum.ru
-alt-servis.ru
-alveris.ru
-amateurmatch.com
+allbizne.co.ua
+allboard.xobor.de
+alldownload.pw
+alldubai.biz
+allstatesugarbowl.org
+alltheviews.com
+alphahoverboards.com
+amazingninja.com
amt-k.ru
-animal36.com
-anniemation.com
-aosheng-tech.com
-apibetweenlinesn-a.akamaihd.net
-apidiamondatanet-a.akamaihd.net
-api.stathat.com
+androids-store.com
+apper.de
apptester.tk
-arenda-avtoprokat-krasnodar.ru
+aramaicmedia.org
+arate.ru
+arcadepages.com
+arendas.net
artparquet.ru
-arvut.org
-asrvvv-a.akamaihd.net
-atley.eu.pn
-autobudpostach.club
-auto-moto-elektronika.cz
-autoplate.info
-azartniy-bonus.com
-b2b-lounge.com
-bambi.ck.ua
-bayareaaeroclub.org
-beclean-nn.ru
-benchmarkcommunications.co.uk
-bestempresas.es
-besttorrentknifta.weebly.com
-bidr.trellian.com
-bildsuche.ru
-bimatoprost-careprost.com.ua
-bingo8888.com
-bioca.org
-bio-market.kz
+asiengirls.net
+aussie-prizes.com
+autotop.com.ua
+avek.ru
+aversis.net
+aviav.co
+aviav.ru
+azbukafree.com
+balook.com
+baoxaydung.com.vn
+bausparen.bz.it
+bbtec.net
+bensbargains.net
+best-mam.ru
+best-seo-offer.com
+bestsexyblog.com
+betterhdporn.com
+big-cash.net
+biketank.ga
bitcoins-live.ru
-bizcheapjerseyswholesalechina.com
-bizlist.com.de
-bmwhighperformers.com
-board.f00d.de
-boole.onlinewebshop.net
-brainboostingsupplements.org
+blog.f00kclan.de
+bloggedporn.com
+bloggers.nl
+bluesman.nu
+bmw-ark.ru
+boeuklad.com
braindaily.xyz
-brus.city
-buron.pw
-buyessay3.blogspot.ru
+brendbutik.ru
+browserprotecter.com
+bryansk.zrus.org
+builtwith.com
+burkesales.com
+buybest1.biz
buynorxx.com
-cablecar.us
-callawaygolfoutlet.online
-candytiner.com
-canyougethighofftizanidine.blogspot.com
-carapics.com
-cartierlove2u.xyz
-cartierreplica.top
-cashonads.com
-celebsopics.com
-cfacarrosserie74.com
-cfjump.com
-chastnoeporno.com
-chatseo.com
-cheapmoncler.pw
-chelyabinsk.xrus.org
-chocoslim.pro
+bvlgaribracelet.xyz
+bvlgariring.xyz
+bytimedance.ru
+camdenmemorials.com
+cam-kontakte.org
+capacitacionyaprendizaje.com
+carstrends2015.com
+cartierlovebracelet.xyz
+casablancamanor.co.za
+cctva.tv
+cejewelry.xyz
+centraletermice.us
+cezartabac.ro
+cgi2.nintendo.co.jp
+cheapestjerseysonwholesale.com
+cheapsergiorossi.online
+checkhit.com
christianlouboutinreplica.pw
-christianlouboutinreplica.win
+ciceros.co
+clash-clans.ru
cleanallvirus.com
-clickhype.com
-clothesforcash.com
-clubfashionista.com
-club-lukojl.ru
-coin-hive.com
-comeondog.info
-compliance-alex.top
+clickintext.net
+clicksor.net
+cmhomestayagency.com
+compliance-checker.info
compliance-george.top
-compliance-jess.top
-concealthyself.com
-counciltally.com
-countbertwistdisp26.soup.io
-cpabegins.ru
-cs-passion.pl
-customerpromos-a.akamaihd.net
-cutalltheshit.com
-cvety24.by
-cyse.tk
-dame-ns.kz
-daydream-studio.ru
+compliance-julianna.top
+compliance-julia.top
+compliance-willy.top
+compucelunlock.net
+com-secure.download
+coolingoods.com
+copyrightclaims.org
+cpajunkies.com
+crd.clan.su
+custom-electric-guitar.com
+cypernhuset.se
+czat.wp.pl
dbmkfhqk.bloger.index.hr
-deanmoore.ie
decorationspcs.com
-demenageur.com
-destinationrealestate.com
-dev-seo.blog
-dfwu1019.info
+delayreferat.ru
+deplim.com
+designdevise.com
+detki-opt.ru
+dev33.dioniqlabb.se
+diamond-necklace.info
diarioaconcagua.com
digilander.libero.it
-directrev.com
-dirtpics.pw
-docsportal.net
-doeco.ru
-domaincheck.io
-dosugrostov.site
-earl-brown.info
-easytuningshop.ru
-e-commerce-seo.com
-ecookna.com.ua
-eduinfosite.com
-ekto.ee
-elektroniksigaraankara.info
-elektronischezigarettekaufen1.myblog.de
-elektrozigaretten1.webs.com
-erectile.bid
-erotische-geschichten-xxl.com
-eshop4u.jp
-euronis-free.com
-ezb.elvenmachine.com
-ezigarettekaufen1.hpage.com
-face.hostingx.eu
-famix.xyz
-favoritemoney.ru
-fenoyl.batcave.net
-filesdatabase.net
-filkhbr.com
-filmetricsasia.com
-financepoints.eu
-findacheaplawyers.com
+diminishedvalueoforegon.com
+dkvorota.ru
+doctissimo.fr
+documentsite.net
+domaincrawler.com
+doyouknowtheword-flummox.ml
+dstroy.su
+dverimegapolis.ru
+dvr.biz.ua
+earthmagic.info
+e-buyeasy.com
+ekobata.ru
+elvel.com.ua
+en.home-task.com
+envaseslotusama.com
+eropornosex.ru
+evaashop.ru
+excaliburfilms.com
+eyemagination.com
+ezigarettekaufen2.mpbloggar.se
+familyholiday.ml
+fatmaelgarny.com
+favornews.com
+femdom.twiclub.in
+fetishinside.com
flagcounter.me
-flashbannernow.com
-flowwwers.com
-forex.osobye.ru
-forodvd.com
-forumprofi.de
-forums.toucharcade.com
-fotopop.club
-fotzen-ficken.com
-free-share-buttons.com
-freesitetest.com
-front.ru
-fuckmill.com
-fxtips.ru
-gamerextra.com
-gavuer.ru
-gay-file.com
-gazporno.com
-gentamicineyedrops.blogspot.com
-getridofstretchmarks.org
-gg.zzyjxs.com
-gilsonchiro.xyz
-glall.ru
-glcomputers.ru
-globetrotting-culture.ru
-gogalleryawesome.com
-gojiberriess.apishops.ru
-gonextmedia.com
+flauntyoursite.com
+flipper.top
+flprog.com
+footballfarrago.com
+fortevidyoze.net
+forum20.smailik.org
+free-deals.faith
+freenom.link
+free-share-buttons.top
+free-social-buttons.com
+freetangodownload.com
+fr.netlog.com
+fungamelands.com
+funnel.co.za
+funtoonez.com
+fyxabomiw.ru
+gael-s.ru
+galeon.com
+gallictures.com
+game300.ru
+gamedayassist.com
+gameprimary.com
+garciniaxt.us
+gateway.zscalerone.net
+gdcentre.ru
+gemgrab-a.akamaihd.net
+gilbertbanda.net
+girlporn.ru
+glass-msk.ru
+globalsurfari.com
+gloverid.site
+go2mike.ru
+golden-catalog.pro
good-mummy.ru
-googlsucks.com
-gotwebsite1.com
-gov.yanao.ru
-graphicwe.org
-greenzaim.ru
-grizzlysgrill.com
-gwebtools.com
+google-liar.ru
+googlepositions.com
+gourcy.altervista.org
+growboxbl.ru
+g.starmoe.xyz
+guruofcasino.com
+hamptonoaks.ca
+hauleddes.com
hd-filmy.net
-hdxnxxtube.mobi
-herehloadibs.cf
-hermesbelts.xyz
-hexpilot.com
-histats.com
-hoholikik.club
-homedecorpicture.us
-hostcritique.com
-hostsshop.ru
-hotshoppymac.com
-housediz.com
-hscsscotland.com
-idegenvezeto.eu
-iideaidekonkatu.info
-imabase.com
-images-graphics-pics.com
-img.wallpaperstock.net
-incest-ru.com
-infazavr.ru
-intellektmedia.at
-internetartfair.com
+hdhc.ru
+healgastro.com
+healing-dysplasia.ru
+heartofpayne.xyz
+helpmymacfaster.trade
+helvetia.com.ua
+howmuchdoestizanidinecost.blogspot.com
+hoztorg-opt.ru
+hspline.com
+humbmosquina.tk
+husky-shop.cz
+hybrid.ru
+ibb.com.ua
+ibmdatamanagement.co
+imagefinder.site
+i-midias.net.br
+imperia31.ru
+incanto.in.ua
+incomekey.net
+insomniagamingfestival.com
+instabid.tech
internetproviderstucson.com
-iplogger.org
-iqoption.com
-it-max.com.ua
+ipornox.xxx
+i-service.kz
+isotoner.com
izhevsk.zrus.org
-izoll.ru
-j33x.com
-jagg.info
-jamiembrown.com
+james13prix.info
+japfm.com
jasonpartington.com
-jbl-charge.info
-jewelryandfiligree.com
-jillepille.com
-jmat.cn
+jennyfire.ru
+jerseychinabizwholesale.us
+jikoman.info
jose.mulinohouse.co
-kaliningrad.zrus.org
-kamen-e.ru
-kazan.xrus.org
-kazinogames.lv
-kaz.kz
-keywords-monitoring-your-success.com
-keywordteam.net
-kinocccp.net
-kinostorm.net
-klikbonus.com
-kndxbkdx.bloger.index.hr
-komp-pomosch.ru
-kredit-blog.ru
-kursy-ege.ru
+jpcycles.com
+jrpmakati.com
+kamorel.com
+karpun-iris.ru
+kasino-money.pw
+katushka.net
+keywordhut.com
+kidskunst.info
+kimcurlrvsms.com
+kino-key.info
+kol-energo.ru
+kosynka-games.ru
+krasnodar.zrus.org
+kretpicf.pw
kwzf.net
-laexotic.com
+lafourmiliaire.com
lakomka.com.ua
-larose.jb2c.me
-law-check-eight.xyz
-lazymanyoga.com
-lazy-z.com
-likesdesign.com
-likrot.com
-limtu.ifmo.ru
-linkrr.com
-lion.playtap.us
-lion.snapmap.us
-littlesunraiser.com
-liupis.com
-livetsomudvekslingstudent.bloggersdelight.dk
-livingroomdecor.info
-loadopia.com
+lankarns.com
+laptoper.net
+law-check-nine.xyz
+le-clos-des-alouettes.com
+letslowbefast.site
+lexaprogeneric.link
+lietaer.com
+life-instyle.com
+light.ifmo.ru
+lignofix.ua
+linkwithin.com
+lion.conboy.us
+livingcanarias.com
+lizace.com
+localflirtbuddies.com
locationdesaison.com
-loftdigital.eu
+lol-smurfs.com
louboutinreplica.pw
-lovasszovetseg.hu
-lsex.xyz
-luxup.ru
-lyngdalhudterapi.no
+louboutinshoes.xyz
+m4ever.net
macdamaged.tech
-macfix.life
-magda-gadalka.ru
-magicalfind-a.akamaihd.net
-marketingvici.com
-matpre.top
-maximpartnerspr.com
-mbiologi.ru
-mebel-alait.ru
-mediaoffers.click
-medizinreisen.de
-medosmotr-ufa.ru
-med-zdorovie.com.ua
-mega-bony-2017.pl
-methodsmarketing.com
-migente.com
-mindbox.co.za
-mirmedinfo.ru
-mmostrike.ru
-moesen-ficken.com
-monetizationking.net
-moviezbonkerssk.cf
-mp3downloadhq.com
-msk.afora.ru
-muizre.ru
-mustwineblog.com
-my-aladin.com
-mydeathspace.com
-my-floor.in.ua
-mygameplus.ru
-netallergy.ru
+makemoneyonline.com
+makis.nu
+marktforschung-stuttgart.com
+medicinacom.ru
+medictube.ru
+medi-fitt.hu
+missis.top
+mnogolok.info
+mojowhois.com
+monclerjacketsoutlet.win
+money-for-placing-articles.com
+montredemarque.nl
+morefastermac.trade
+mototsikl.org
+mriyadh.com
+mrmoneymustache.com
+mrsdalloways.com
+muz-baza.net
+muznachas-service.ru
+mydirtyhobby.com
+myindospace.com
+mylida.org
+myshopmatemac.com
+nacap.ru
+nagdak.ru
+nastydollars.com
net-radar.com
-nikkiewart.ru
-ningessaybe.me
-nodding-passion.tk
-normalegal.ru
-noscrapleftbehind.co
-nowtorrents.com
-nuker.com
-nzfilecloud.weebly.com
-obnal.org
-oda.as
-odywpjtw.bloger.index.hr
-offergroup.info
-offf.info
-oksrv.com
-onlineslotmaschine.com
-onlinewritingjobs17.blogspot.ru
-onload.pw
-onlyforemont.ru
-onstrapon.purplesphere.in
-oops-cinema.ru
-openlibrary.org
-openmultipleurl.com
-open-odyssey.org
-optitrade24.com
-ourville.info
-palvira.com.ua
-paramountmarble.co.uk
-partner-high.men
-partnerline.men
-pavlodar.xkaz.org
-pcads.ru
-pensplan4u.com
-php-market.ru
-piccdata.com
-piccshare.com
-picmoonco.pw
-pihl.se
-poppen-nw.net
-porndroids.com
-pornhubforum.tk
-pornofiljmi.com
-porno-video-chati.ru
-priceg.com
-producm.ru
-professionaldieselcare.com
-profitfx.online
-progress-upakovka.ru
-projectforte.ru
-promalp-universal.ru
-promodj.com
-promoheads.com
-pron.pro
-proposal-engine.com
-pukaporn.com
-pushdata.sendpulse.com
-quickloanbank.com
-randki-sex.com
-rankingchart.de
-razorweb-a.akamaihd.net
-r-control.ru
-real-time-analytics.com
-reyel1985.webnode.fr
-rfid-locker.co
-rn-to-bsn.com
-rocketchange.ru
-roll123.com
-royalvegascasino.com
-rozalli.com
-rubbed.us
-russia-tao.ru
-rybalka-opt.ru
-salutmontreal.com
+netvouz.com
+networkad.net
+nextrent-crimea.ru
+nfljerseyscheapchinabiz.com
+ngps1.ru
+no-rx.info
+novosibirsk.xrus.org
+npoet.ru
+nvformula.ru
+nyfinance.ml
+obsessionphrases.com
+offers.bycontext.com
+one-gear.com
+onlinadverts.com
+online-podarki.com
+online-x.ru
+onlythegames.com
+o-o-6-o-o.ru
+osb.se11.ru
+oxford-book.com.ua
+paydayloanslocal.com
+penisvergrotendepillennl.ovh
+pensplan.com
+petedrummond.com
+picphotos.net
+picquery.com
+picsforkeywordsuggestion.com
+picture-group.com
+pills24h.com
+pintattoos.com
+pio.polytopesexempt.com
+pirateday.ru
+pix-hd.com
+pizda.lol
+pl.aasoldes.fr
+poem-paying.gq
+poisk-zakona.ru
+porn555.com
+pornodojd.ru
+pornonik.com
+pornoxxx.com.mx
+primedice.com
+princeadvantagesales.com
+professionalwritingservices15.blogspot.ru
+promoforum.ru
+prostitutki-almata.org
+prostitutki-rostova.org
+pro-tec.kz
+quebec-bin.com
+qwarckoine.com
+rapidgator-porn.ga
+realmonte.net
+remont-fridge-tv.ru
+remote-dba.de
+remybutler.fr
+renecaovillasale.online
+ricorsogiustizia.org
+rockingclicks.com
+rubanners.com
+rvtv.ru
+safe-app.net
+sale-japan.com
+sanatorrii.ru
sape.top
-sbdl.no
-scanner-alex.top
-scanner-george.top
-scanner-julianna.top
-scat.porn
-scripted.com
-scrnet.biz.ua
-sdsjweb.com
-seadragonherbery.com
-se.bnt-team.com
-semaltmedia.com
-seorank.info
-sergiorossistore.online
-sexblog.pw
-sexobzor.info
-sexphoto.site
-sexs-foto.top
-sexuria.net
-sex-watch.com
-sexy-screen-savers.com
-s-forum.biz
-share-buttons-for-free.com
-shlyahten.ru
-shop.acim.org
-shopcheermakeup.info
-shopvilleroyboch.com.ua
-sibvitr.ru
-siteaero.com
-sitiz.club
+saugatuck.com
+savefrom.com
+scanspyware.net
+searchinquire.com
+seminarygeorgia59.ga
+seocheckupx.com
+seofied.com
+seojokes.net
+seonetwizard.com
+servethis.com
+service.adtech.us
+shakhtar-doneck.ru
+sherlock.se
+shop-electron.ru
+sim-service.net
+site-speed-checker.site
+s.lollypopgaming.com
slomm.ru
-slujbauborki.ru
-smartadserver.com
-snip.to
-snip.tw
+smart-balancewheel.com
+social-buttons.xyz
+socialtrade.biz
sockshare.net
-soft1.ru
-softonicads.com
-softwaretrend.net
-sophang8.com
-spasswelt.net
-spitfiremusic.com
-statistici.ro
-steame.ru
-stjamesschool.info
-stockquotes.wooeb.com
-stop-gepatit.te.ua
-strongsignal-a.akamaihd.net
-suhanpacktech.com
-sundrugstore.com
-svtrd.com
-swsociety.se
-tagil.zrus.org
-tattooreligion.ru
-t-bygg.com
-tedxrj.com
-telefonsex-ohne0900.net
-telefonsexsofort.tk
-teslathemes.com
-tetracsaudi.com
-thaismartloan.com
-thecoolimages.net
-thecoral.com.br
-thefarmergame.com
-thefotosgratis.eu
-themestotal.com
-thewomenlife.com
-tizanidine6mg.blogspot.com
-tizanidinedosage.blogspot.com
-tizanidineduringpregnancy.blogspot.com
-tizanidineinfo.blogspot.com
+songplanet.ru
+speedup-my.site
+srecorder.com
+staticfs.host
+steamoff.net
+stpicks.com
+strag-invest.ru
+streetfire.net
+stroiminsk.com
+sugarkun.com
+superlist.biz
+surfbuyermac.com
+susanholtphotography.com
+svolze.com
+t3chtonic.com
+technika-remont.ru
+technopellet.gr
+tedy.su
+tehngr.ru
+telsis.com
+terrafootwear.us
+theguardlan.com
+thetattoohut.com
+thfox.com
+tiandeural.ru
+tiens2010.ru
+titan-ads.life
+tizanidinepain.blogspot.com
tizanidinesideeffectsweightloss.blogspot.com
-tizanidinevscyclobenzaprine.blogspot.com
-tmtrck.com
-tomck.com
-topkarkas.com
-touchmods.fr
-tracklead.net
-track-rankings.online
-trackzapper.com
-trafficnetzwerk.de
-truck-addzilla.life
-tvorozhnaja-zapekanka-recept.ru
-twojebook.pl
-uchetunet.su
-ucoz.ru
-ukrobstep.com
-umekana.ru
-unmaroll.ya.ru
-uptime-delta.net
-utrolive.ru
-vacuumscleaner.com
-valkiria-tk.ru
-vapeface.club
-vapomnoncri.tk
-vapsy.com
-vchulkax.com
-vekzdorov.ru
-vellings.info
-viagra-soft.ru
-vksaver-all.ru
-voloslove.ru
-voluumtrk.com
-vzube.com
-walkme.com
-webmonetizer.net
-website-datenbank.de
-website-speed-up.site
-weddingdresses.xyz
+tizanidinesleepaid.blogspot.com
+tizanidinestreetprice.blogspot.com
+tobeyouday.win
+token-lab.org
+tomatis.gospartner.com
+tongkatmadura.info
+topshef.ru
+top-study.work
+torrents-tracker.com
+tour-line.net
+toyota.7zap.com
+track112.site
+tradgardspartner.se
+traffic100.com
+trafficfactory.biz
+trafficjunky.net
+traffmonster.info
+trucri.me
+tsan.net
+tube8.com
+tuckpointingmasonrysystems.com
+twiclub.in
+tzritel.tk
+ua-company.ru
+u-cheats.ru
+udsgame.online
+uhodzalijami.ru
+ultramart.biz
+umnovocaminho.com
+unacittaconte.org
+uptime-as.net
+uptime.com
+varbergsvind.se
+vektorpress.ru
+vezuviy.su
+via-energy-acquistare.com
+via-gra.webstarts.com
+videokrik.net
+video-production.com
+vigrx-original.ru
+virtuagirl.com
+vitoriacabos.com
+viven.host.sk
+vltai.com
+vut.com.ru
+vysigy.su
+w3data.co
+wahicbefa31.soup.io
+watchinf.com
+wdrake.com
+webmasters.stackexchange.com
+websiteexplorer.info
+weburlopener.com
+wedding-salon.net
welcomeauto.ru
-whatistizanidine2mg.blogspot.com
-wholinkstome.com
-wineitudes.wordpress.com
-w-journal.ru
-wordkeyhelper.com
-worldoffiles.ru
+wildworld.site
+williamrobsonproperty.com
+windowssearch-exp.com
+witherrom55.eklablog.fr
+wnhjavlhezp.gq
+woman-tampon.ru
+wonderfulflowers.biz
+worst-sites.online
writingservices17.blogspot.ru
-wzgyyq.com
-xboxster.ru
-xn----7sbbahaq9bb5afgiqfliv4m.xn--p1ai
+wtsindia.in
+xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai
xn--80aaaks3bbhabgbigamdr2h.xn--p1ai
-xn--d1aifoe0a9a.top
-xn--e1afanlbnfckd7c3d.xn--p1ai
-x-true.info
-xxxguitars.com
-yellowads.men
-yes-com.com
+xn--80aikhbrhr.net
+xpresscare.ru
+xtube.com
+xxlargepop.com
+xxxrus.org
+yaderenergy.ru
+yadro.ru
+yebocasino.com
yhit.press
-yougetsignal.com
-zakonobosago.ru
-zdesoboi.com
-zeroredirect5.com
-zlatnajesen.com
-zojirushi-products.ru
-zootravel.com
+youtoner.it
+youtuhe.com
+yuweng.info
+zaimhelp.ru
+zarabotok--doma.ru
+zarajbuilders.com
+zeroredirect11.com
+zeroredirect7.com
+zeroredirect8.com
+zoompegs.com
+zrelaya.pw
+zscalerone.net
diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list
index 41a7c81a4..4500b3c1d 100644
--- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list
+++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list
@@ -1,250 +1,250 @@
-1-free-share-buttons.com
-1kinobig.ru
-273-fz.ru
-3weekdiet.com
-40cg.com
-4kepics.com
-4replicawatch.net
-5tey7463.weebly.com
-aab94f698f36684c5a852a2ef272e031bb.com
-acheterviagrafr24.com
-adloads.net
-adzerg.com
-akvamaster.dp.ua
-algerianembassy.co.in
-allpdfmags.net
-alphaforum.ru
-alphahoverboards.com
-alvtank.se
-amt-k.ru
-android-vsem.org
-apiappenableinfo-a.akamaihd.net
-apidigidocketnet-a.akamaihd.net
-apispringsmartne-a.akamaihd.net
-aquarium-pro.ru
-arenda-avtoprokat-krasnodar.ru
-artparquet.ru
-autobrennero.it
-avtovolop.ru
-babs.com.ua
-baersaratov.ru
-balkanfarma.org
-bannerspace.com
-beachfix.co
-beauty-things.com
-best-deals-products.com
-bestimagecoollibrary.com
-bestserials.com
-betwinservice.com
-big-cash.net
-bimatoprost-careprost.com.ua
-bitnote.co
-blogos.kz
-brainxs.us
-bryansk.zrus.org
-burkesales.com
-buy-loft.ru
-callcafe.info
-calvet.altervista.org
-camel-beach.com
-cardsharp1.ru
-cerev.info
-cheapestjerseysonwholesale.com
+1stat.ru
+34.gs
+883zy.com
+9med.net
+aa81bf391151884adfa3dd677e41f94be1.com
+aanapa.ru
+academiacsmendoza.org
+adhome.biz
+agadelha.com.br
+alfa-medosmotr.ru
+alot.com
+am-se.com
+apidiamondatanet-a.akamaihd.net
+applicationg29.com
+apxeo.info
+articlesdirectoryme.info
+audiofree.ru
+aviav.co
+avtolombard-krasnodar.com
+ayanaspa.com
+baladur.ru
+bala.getenjoyment.net
+balitouroffice.com
+bang-hotties.com
+belinvestdom.by
+bestofpicture.com
+best-way.men
+bg6s0.com
+blogsfunky672.weebly.com
+bolitgorlo.net
+boltalko.xyz
+bupropion-sr-150-mg.us
+buy-forum.ru
+buyk.host.sk
+cactussoft.biz
+camdenmemorials.com
+cand.jp
+candy-glam-hp.com
+capture-room.com
+carabela.com.do
+carloans.com
+cartierlovestore.xyz
+celejihad.info
cheapmoncler.pw
-chimiver.info
-chocolateslim-original.com
-cialischmrx.com
-compliance-alexa.top
-compliance-checker-7.info
-compliance-jane.top
-country-chic.ru
-cpajunkies.com
-creams.makeforum.eu
-csgo4.win
-customchocolate.business-for-home.com
-cymbaltaandtizanidine.blogspot.com
-dailyrank.net
-dangphoto.trade
-dating-time-now.com
-dev.citetick.com
-dirtpics.pw
-divci-hry.info
-djihispano.com
-doska-vsem.ru
-dustyorate.com
-e2click.com
-earn-from-articles.com
-elektrischeziga.livejournal.com
-elektrozigarette1.dreamwidth.org
-eloconcream.blogspot.com
-engines-usa.com
-eshop4u.jp
-ezigarettekaufen1.hpage.com
-favornews.com
-filesclub.net
-forexunion.net
-foto-sisek.porngalleries.top
-freeuploader.com
-fx-brokers-review.com
-gael-s.ru
-gameplexcity.com
-gayxperience.com
-geileweiber.tk
-getridofstretchmarks.org
-gheus.altervista.org
-go2mike.ru
-googlemare.com
-gosmeb.ru
-gotwebsite1.com
-hdxnxxtube.mobi
-hitsbox.info
-hmywwogw.bloger.index.hr
-homemade.gq
-hotdl.in
-houdom.net
-hoverboard360.de
-hyiphunter.org
-inome.com.ua
-iomoio.net
-iqoption.pro
-ivoiretechnocom.ci
-jacago.com
-janavibekken.no
-javxxx18.com
-jbl-charge.info
-justbcause.com
-kazinogames.lv
-keywords-monitoring-success.com
-keywordteam.net
-kinofak.net
-kino-fun.ru
-klin3952.ru
-kotaku.com
-krasivoe-hd.net
-labplus.ru
-laexotic.com
-lawyers.cafe
-letolove.ru
-lifebyleese.com
-lignofix.ua
-live-cam6.info
-live-sexchat.ru
-lost-alpha.ru
-lovelycraftyhome.com
-m5home.ru
-machicon-ueno.info
-makemoneyonline.com
-mansiondelrio.co
-master-muznachas.ru
-mcadamssupplyco.com
-medicineseasybuy.com
-medictube.ru
-meendo-free-traffic.ga
-metabar.ru
-mhi-systems.ru
-middlerush-a.akamaihd.net
-midst.eu
-monclercheap.xyz
-musezone.ru
-musicdaddy.net
+cialiswithoutadoctor.net
+clipartnew.com
+com-onlinesupport.site
+compliance-george.top
+compliance-george.xyz
+compliance-walt.top
+conboy.us
+copblock.org
+corta.co
+cottageofgrace.com
+crest3d.ru
+czat.wp.pl
+daneshetabiat.com
+daydream-studio.ru
+delfin-aqua.com.ua
+detki-opt.ru
+dfwu1013.info
+djstools.com
+dollartree.info
+druzhbany.ru
+e-collantes.com
+elektrischezigarette1.blog.pl
+elitedollars.com
+epicbrogaming.com
+eshop.md
+euromasterclass.ru
+ezaz.info
+farmprofi.net
+fashion-stickers.ru
+filunika.com.ru
+findthe.pet
+forex-procto.ru
+foto-basa.com
+franklinfire.co
+fr-bearings.ru
+free411games.com
+freshmac.space
+galeria-zdjec.com
+gazporno.com
+generic-pills-online.com
+getyourimage.club
+ghostvisitor.com
+girlporn.ru
+girls-ufa.ru
+googglet.com
+greenidesign.co
+guardlink.org
+hamacapty.com
+hasshe.com
+hdapp1008-a.akamaihd.net
+hentai-manga.porn
+hobbyhuren24.net
+hoporno.com
+hplaserjetpdriver8y.pen.io
+hydropump.su
+iamsport.org
+ideashome.id
+igtools.club
+ilbe.club
+import-sales.com
+infektsii.com
+irkutsk.online-podarki.com
+istripper.com
+itsdp3.com
+j33x.com
+jerseywholesaleelitestore.com
+jyvopys.com
+kamalsinha.com
+kemerovo.zrus.org
+kerwinandcariza.com
+keywordsking.com
+kinopolet.net
+konturkrasoty.ru
+kreativperlen.ch
+letto.by
+light.ifmo.ru
+link.ac
+locatellicorretor.com.br
+lol-smurfs.com
+lotto6888.com
+louboutinbooties.xyz
+louisvuittonoutletstore.net
+luxup.ru
+m1media.net
+m292.info
+mackeeper-land-672695126.us-east-1.elb.amazonaws.com
+madot.onlinewebshop.net
+magazintiande.ru
+mainlinehobby.net
+meget.co.za
+mesbuta.info
+methodsmarketing.com
+montazhnic.ru
+morocco-nomad-excursions.com
+moroccosurfadventures.com
+most.gov.iq
myfreecams.com
-nelc.edu.eg
-nikitsyringedrivelg.pen.io
-nl.netlog.com
-nofreezingmac.click
-npoet.ru
-ok.ru
-optom-deshevo.ru
-orsonet.ru
-partners-ship.pro
-perl.dp.ua
-picscout.com
-picsforkeywordsuggestion.com
-pictures-and-images.net
-popmarker.com
-popunder.net
-popunder.ru
-pornhive.org
-pornoklad.ru
-pornosee.info
-portadd.men
-powc.r.ca.d.sendibm2.com
-powenlite24.ru
-pozdrawleniya.com
-privetsochi.ru
-prodess.ru
-quit-smoking.ga
+myhealthcare.com
+myyour.eu
+obnal.org
+okeinfo.online
+old-rock.com
+omoikiri-japan.ru
+online-podarki.com
+ontargetseo.us
+oops-cinema.ru
+openlibrary.org
+optitrade24.com
+ostrovtaxi.ru
+palvira.com.ua
+paramountmarble.co.uk
+pawli.eu
+pb-dv.ru
+pechikamini.ru
+petedrummond.com
+photochki.com
+picturesfrom.com
+pkr1hand.com
+pk-services.ru
+pochtovyi-index.ru
+polybuild.ru
+pons-presse.com
+pornobrazzers.biz
+pornotubexxx.name
+produkto.net
+pushdata.sendpulse.com
+putevka24.ru
+quickbuck.com
quizzitch.net
+raavidesigns.com
randki-sex.com
-ranksonic.org
-razamicroelectronics.com
-referencemoi.com
-renewablewealth.com
-robertefuller.com
-rockprogblog.com
-royalads.net
-runofilms.ru
-saddiechoua.com
-sady-urala.ru
-sbprabooks.com
-seo-platform.com
-serialsway.ucoz.ru
-sex-tracker.de
-shell-pmr.ru
-shikiso.info
-shop.acim.org
-signal03.ru
-sildenafilcitratemed.com
-simple-image.com.ua
-simply.net
+rankings-analytics.com
+razleton.com
+remont-comp-pomosh.ru
+rezeptiblud.ru
+rixpix.ru
+ruscopybook.com
+russia-today-video.ru
+sa-live.com
+samara.rosfirm.ru
+savingsslider-a.akamaihd.net
+scalerite.co.za
+scanspyware.net
+sdelatmebel.ru
+seeingmeerkat.com
+sexkontakteao.info
+sexs-foto.com
+sherlock.se
+shopwme.ru
+sigmund-freud.co.uk
+sildenafil-tadalafil.info
sisi-go.ru
-site3.free-share-buttons.com
-sk.golden-praga.ru
-smichovbike.cz
-sobecjvuwa.com.ru
-soblaznu.net
-solicita.info
-solution4u.com
-spaceshipad.com
-spidtest.org
-spywarebegone.com
-staticfs.host
-sugarlyflex.pw
-superlist.biz
-superoboi.com.ua
-tacbelarus.ru
-taraz.xkaz.org
-telegraf.by
-tfxiq.com
-thedownloadfromwarez.blogspot.com
-tizanidine4mgtablets.blogspot.com
-tksn.ru
-tmm-kurs.ru
-torontoplumbinggroup.com
-tramadolandtizanidine.blogspot.com
-tubo360.com
-tula.mdverey.ru
-turbodsp.com
-uchil.net
-umityangin.net
-unrealcommander.com
-vedomstvo.net
-venta-prom.ru
-villakohlanta.nu
-vitalads.net
-vkgaleria.com
-vksaver-all.ru
-vmnmvzsmn.over-blog.com
-webshoppermac.com
-weclipart.com
-worldhistory.biz
-wormix-cheats.ru
+slayerlife.com
+sluganarodu.ru
+social-buttons.xyz
+spyfu.com
+spymac.net
+spywareit.com
+stanthonyscatholicchurch.org
+startufa.ru
+stefanbakosab.se
+strv.se
+stuff-about-money.com
+sukirgenk.dvrlists.com
+swiped.su
+syndicate.fun
+tako3.com
+tecspb.ru
+thin.me.pn
+threecolumnblogger.com
+tizanidineduringpregnancy.blogspot.com
+token-lab.org
+topmira.com
+track2.shop
+traffic100.com
+trafficjunky.net
+twittrading.com
+universals.com.ua
+unrealcommander.biz
+viddyoze.com
+video-chat.in
+videosbox.ru
+vip.51.la
+vonradio.com
+vrnelectro.ru
+warningwar.ru
+weekes.biz.tc
+wholesalejerseys-cheapest.com
+winterclassichockeyjerseys.com
+winx-play.ru
+wladimirpayen.com
+word-vorlagen.net
wpsecurity.website
-x-musics.com
-xn-----7kcabaipgeakzcss7bjdqdwpfnhv.xn--p1ai
-xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai
-xnxxandxvideos.com
-yougetsignal.com
-youporn-forum.ga
-youraticles.pl
-your-bearings.com
+xfire.com
+xn--80ajjbdhgmudixfjc8c5a9df8b.xn--p1ai
+xn--80ak6aa92e.com
+xn--e1aggki3c.xn--80adxhks
+xtube.com
+xvideospornoru.com
+xxx-treker.ru
+yopoint.in
+yoshkarola.zrus.org
+youtubedownload.org
+ytmnd.com
+zebradudka.com
+zolotoy-lis.ru
diff --git a/.dev-tools/test_units/ratelimittest-master.txt b/.dev-tools/test_units/ratelimittest-master.txt
index a7215fb74..525a76fd6 100644
--- a/.dev-tools/test_units/ratelimittest-master.txt
+++ b/.dev-tools/test_units/ratelimittest-master.txt
@@ -5,13 +5,6 @@
nginx/1.14.2