diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
index 7ff156874..b6d7e67e9 100644
--- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1708
-### Updated: Mon Jul 1 10:19:47 SAST 2019
+### Version: V3.2019.07.1709
+### Updated: Mon Jul 1 10:37:05 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 465ec1626..6893ce0fc 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 1;
+35.192.136.167 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index 290c48e19..879967dc9 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 0;
+35.192.136.167 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index 7ff156874..b6d7e67e9 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1708
-### Updated: Mon Jul 1 10:19:47 SAST 2019
+### Version: V3.2019.07.1709
+### Updated: Mon Jul 1 10:37:05 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 465ec1626..6893ce0fc 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 1;
+35.192.136.167 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index 290c48e19..879967dc9 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 0;
+35.192.136.167 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
index 7ff156874..b6d7e67e9 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1708
-### Updated: Mon Jul 1 10:19:47 SAST 2019
+### Version: V3.2019.07.1709
+### Updated: Mon Jul 1 10:37:05 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
index 465ec1626..6893ce0fc 100644
--- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 1;
+35.192.136.167 1;
127.0.0.1 1;
diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
index 290c48e19..879967dc9 100644
--- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 0;
+35.192.136.167 0;
127.0.0.1 0;
diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
index 7ff156874..b6d7e67e9 100644
--- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1708
-### Updated: Mon Jul 1 10:19:47 SAST 2019
+### Version: V3.2019.07.1709
+### Updated: Mon Jul 1 10:37:05 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template
index 7ff156874..b6d7e67e9 100644
--- a/.dev-tools/globalblacklist-testing.template
+++ b/.dev-tools/globalblacklist-testing.template
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.07.1708
-### Updated: Mon Jul 1 10:19:47 SAST 2019
+### Version: V3.2019.07.1709
+### Updated: Mon Jul 1 10:37:05 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template
index d211bcd2c..f26e52add 100755
--- a/.dev-tools/globalblacklist.template
+++ b/.dev-tools/globalblacklist.template
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1708
-### Updated: Mon Jul 1 10:19:46 SAST 2019
+### Version: V4.2019.07.1709
+### Updated: Mon Jul 1 10:37:04 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
index 896ffe576..2b15f263d 100644
--- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf
@@ -45,4 +45,4 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-104.154.120.187 0;
+34.68.144.114 0;
diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
index d211bcd2c..f26e52add 100644
--- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1708
-### Updated: Mon Jul 1 10:19:46 SAST 2019
+### Version: V4.2019.07.1709
+### Updated: Mon Jul 1 10:37:04 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 5e7c0ed8e..b9d24324a 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.184.226.236 1;
+34.68.144.114 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index 3e60bde86..ea117da9f 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.184.226.236 0;
+34.68.144.114 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index d211bcd2c..f26e52add 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1708
-### Updated: Mon Jul 1 10:19:46 SAST 2019
+### Version: V4.2019.07.1709
+### Updated: Mon Jul 1 10:37:04 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 5e7c0ed8e..b9d24324a 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.184.226.236 1;
+34.68.144.114 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index 3e60bde86..ea117da9f 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.184.226.236 0;
+34.68.144.114 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
index d211bcd2c..f26e52add 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1708
-### Updated: Mon Jul 1 10:19:46 SAST 2019
+### Version: V4.2019.07.1709
+### Updated: Mon Jul 1 10:37:04 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
index 5e7c0ed8e..b9d24324a 100644
--- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.184.226.236 1;
+34.68.144.114 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
index 3e60bde86..ea117da9f 100644
--- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.184.226.236 0;
+34.68.144.114 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
index d211bcd2c..f26e52add 100644
--- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1708
-### Updated: Mon Jul 1 10:19:46 SAST 2019
+### Version: V4.2019.07.1709
+### Updated: Mon Jul 1 10:37:04 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
index 0135b296e..403c2e793 100644
--- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
@@ -45,5 +45,5 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.192.85.2 0;
-104.154.120.187 0;
+35.193.14.140 0;
+104.198.131.58 0;
diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
index 47b0c4a59..71b996828 100644
--- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1708
-### Updated: Mon Jul 1 10:19:46 SAST 2019
+### Version: V4.2019.07.1709
+### Updated: Mon Jul 1 10:37:04 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
index 8ecd49951..92e5004d6 100644
--- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
@@ -45,6 +45,6 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.184.226.236 0;
-35.224.112.202 0;
-35.193.7.13 0;
+104.154.113.151 0;
+34.66.178.120 0;
+35.192.187.174 0;
diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
index d211bcd2c..f26e52add 100644
--- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1708
-### Updated: Mon Jul 1 10:19:46 SAST 2019
+### Version: V4.2019.07.1709
+### Updated: Mon Jul 1 10:37:04 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
index 430e746ee..cf6daef34 100644
--- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
@@ -45,6 +45,6 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-104.154.120.187 0;
-35.184.226.236 0;
-35.202.145.110 0;
+104.198.131.58 0;
+35.192.136.167 0;
+34.66.178.120 0;
diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
index d211bcd2c..f26e52add 100644
--- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.07.1708
-### Updated: Mon Jul 1 10:19:46 SAST 2019
+### Version: V4.2019.07.1709
+### Updated: Mon Jul 1 10:37:04 SAST 2019
### Bad Referrer Count: 6717
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf
index 465ec1626..6893ce0fc 100644
--- a/.dev-tools/test_units/blacklist-ips.conf
+++ b/.dev-tools/test_units/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.188.1.99 1;
+35.192.136.167 1;
127.0.0.1 1;
diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list
index dff6c7a0b..489325858 100644
--- a/.dev-tools/test_units/random-bots-for-test-quick.list
+++ b/.dev-tools/test_units/random-bots-for-test-quick.list
@@ -1,10 +1,10 @@
-Alligator
-Apexoo
-ChinaClaw
-DomainCrawler
-Gigablast
-GrapeFX
-MSFrontPage
-Msrabot
-Toata
-WebSucker
+Aboundex
+BetaBot
+ECCP/1.0
+Iria
+JetCar
+Kozmosbot
+MauiBot
+Mojeek
+OpenLinkProfiler
+WebBandit
diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list
index eb8d8806e..bed60124e 100644
--- a/.dev-tools/test_units/random-bots-for-test.list
+++ b/.dev-tools/test_units/random-bots-for-test.list
@@ -1,250 +1,250 @@
-360Spider
-404checker
404enemy
80legs
Abonti
-Aboundexbot
Acunetix
-AhrefsBot
+ADmantX
AIBOT
-Aipbot
+AiHitBot
Alexibot
AllSubmitter
-Attach
-autoemailspider
-BackDoorBot
-Backlink-Ceck
-backlink-check
-BackStreet
+AlphaBot
+Apexoo
+Asterias
Badass
Barkrowler
BatchFTP
Battleztar Bazinga
-BDCbot
Bigfoot
-Blackboard
-BlackWidow
+Bitacle
+Black Hole
+BLEXBot
+Boardreader
Bolt
-BotALot
Brandwatch
-Buddy
+BuiltBotTough
BunnySlippers
-CazoodleBot
+BuzzSumo
Cegbfeieh
+CheeseBot
+CherryPicker
CheTeam
-ChinaClaw
Chlooe
Claritybot
+Cliqzbot
coccocbot-web
+Cogentbot
cognitiveseo
-Copier
+Collector
+com.plumanalytics
+CopyRightCheck
Copyscape
Cosmos
-Craftbot
crawler4j
-CrazyWebCrawler
-DataCha0s
-DBLBot
-Deusu
+crawler.feedback
+CSHttp
+demandbase-bot
+Devil
DigitalPebble
-DIIbot
Dirbuster
-Disco
-Discobot
-Dispatch
-DomainAppender
-DomainCrawler
-Download Wonder
-DSearch
-eCatch
+Discoverybot
+Drip
+DTS Agent
+ECCP/1.0
+Ecxi
+EirGrabber
evc-batch
Evil
Exabot
-Express WebPictures
ExtLinksBot
-Extractor
EyeNetIE
-Ezooms
-facebookscraper
-FHscan
-Fimap
Firefox/7.0
-FlashGet
Flunky
-Fyrebot
-Getintent
+FyberSpider
+GalaxyBot
+GermCrawler
GetRight
+GetWeb
+Gigabot
G-i-g-a-b-o-t
-Go!Zilla
+Go-Ahead-Got-It
Grabber
-Grafula
-GrapeshotCrawler
+GrapeFX
+GT::WWW
Harvest
-HEADMasterSEO
+Havij
Heritrix
-Hloader
HTTP::Lite
-HTTrack
-Iblog
-Image Fetch
-Indy Library
+Humanlinks
+HybridBot
+IDBot
+Image Sucker
+IndeedBot
InfoTekies
-instabid
-Intelliseek
InterGET
+internetVista monitor
ips-agent
-Iria
-IstellaBot
-JamesBOT
-JetCar
-JOC Web Spider
-Joomla
-JustView
-Kenjin Spider
+Jbrofuzz
+JikeSpider
+Keyword Density
Kozmosbot
-LeechFTP
-Lftp
+LeechGet
Libwhisker
+Lightspeedsystems
+Likse
LinkextractorPro
-LinkScan
-LinqiaMetadataDownloaderBot
+LinkWalker
+Lipperhey
Lipperhey Spider
-Litemage_walker
+Lmspider
+LNSpiderguy
+lwp-request
+lwp-trivial
+Magnet
+Mag-Net
+magpie-crawler
MarkMonitor
MarkWatch
-Mata Hari
+Mass Downloader
+MauiBot
Meanpathbot
+MeanPath Bot
mediawords
-Metauri
-MFC_Tear_Sample
+MegaIndex.ru
+Microsoft Data Access
+Microsoft URL Control
+MIIxpc
+MSFrontPage
+MSIECrawler
Msrabot
-MS Web Services Client Protocol
muhstik-scan
-Musobot
-Nameprotect
-Navroad
-Needle
-NetAnts
-Netcraft
+Name Intelligence
+Nessus
NetMechanic
NetSpider
-NetZIP
-NextGenSearchBot
+Nettrack
+Net Vampire
Nibbler
NICErsPRO
+Niki-bot
Nimbostratus
-Ninja
+Nmap
+NPbot
Nutch
+oBot
Offline Explorer
-Offline Navigator
-OnCrawl
-Openfind
OpenLinkProfiler
-OpenVAS
OrangeSpider
-OutclicksBot
OutfoxBot
+PageAnalyzer
Page Analyzer
PageGrabber
page scorer
+Pandalytics
+Panscient
pcBrowser
-PECL::HTTP
-PeoplePal
-PictureFinder
+PHPCrawl
+Picscout
+Picsearch
+Pixray
+PleaseCrawl
plumanalytics
Pockey
+POE-Component-Client-HTTP
Probethenet
+ProWebWalker
+Psbot
+Pump
QueryN Metasearch
+Quick-Crawler
RankActive
+RankActiveLinkBot
RankFlex
RankingBot
-Recorder
-Scanbot
-scan.lol
-ScoutJet
+RankingBot2
+RealDownload
+Reaper
+RepoMonkey
+s1z.ru
Scrapy
-Screaming
+ScreenerBot
Searchestate
-Semrush
+SemrushBot
+SEOkicks
SEOkicks-Robot
-SEOlyticsCrawler
Seomoz
-SEOprofiler
seoscanners
+SeoSiteCheckup
SEOstats
serpstatbot
-sexsearcher
Shodan
+Siphon
SISTRIX
-Sitebeam
-SiteLockSpider
-Sitevigil
-SMTBot
-Snapbot
-SocialRankIOBot
+Site Sucker
+Snake
+Snoopy
sogouspider
-Sosospider
-Sottopop
+Spammen
SpankBot
+Spanner
sp_auditbot
spyfu
Sqlworm
Sqworm
-Sucuri
+Stripper
+Sucker
SuperHTTP
Swiftbot
sysscan
+Szukacz
+T8Abot
+tAkeOut
Teleport
Telesoft
Telesphorep
-The Intraformant
-TheNomad
TightTwatBot
-Toweyabot
Trendiction
-trendiction.de
-Turingos
-Twice
-UnisterBot
+Turnitin
+Typhoeus
Upflow
+VB Project
VCI
-VeriCiteCrawler
-VidibleScraper
Virusdie
-Voltron
+Voil
+Wallpapers/3.0
WallpapersHD
WBSearchBot
+WebAuto
+Web Auto
WebCollage
WebCopier
+WEBDAV
WebEnhancer
Web Enhancer
-WebFetch
-WebFuck
-Web Fuck
+Web Fetch
+WebImageCollector
+WebmasterWorldForumBot
+webmeup-crawler
WebPix
Web Pix
-WebReaper
-WebSauger
Web Sauger
-WebsiteExtractor
-Webster
-WebStripper
-WebWhacker
+Web Sucker
WebZIP
+WeSEE
Whack
-Whatweb
-WiseGuys Robot
-Wotbox
-WWW-Collector-E
+Who.is Bot
+Widow
+Wonderbot
+Wprecon
WWW-Mechanize
-WWW::Mechanize
+WWWOFFLE
x09Mozilla
+Xaldon WebSpider
Xenu
xpymep1.exe
YoudaoBot
Zade
-Zauba
-zauba.io
+Zermelo
Zeus
-ZmEu
+zgrab
+Zitebot
ZyBorg
diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list
index 997bb0821..75f38a09b 100644
--- a/.dev-tools/test_units/random-bots-for-whitelist-test.list
+++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list
@@ -1,250 +1,250 @@
-404checker
-404enemy
-Aboundex
Aboundexbot
Acunetix
-AfD-Verbotsverfahren
-AhrefsBot
-AIBOT
+ADmantX
AiHitBot
+Alexibot
+AllSubmitter
Anarchie
archive.org_bot
-Asterias
+ASPSeek
+Attach
autoemailspider
-backlink-check
-BackWeb
+BacklinkCrawler
+BackStreet
Badass
+Barkrowler
BatchFTP
BDCbot
-Bigfoot
+BetaBot
Bitacle
-Blackboard
Black Hole
+BlackWidow
BLEXBot
+Blow
+BlowFish
Boardreader
BotALot
Brandprotect
-Brandwatch
-Buddy
BuiltBotTough
+Bullseye
CATExplorador
-CherryPicker
-Chlooe
Cliqzbot
-Cloud mapping
-Cogentbot
-cognitiveseo
+Collector
+com.plumanalytics
+Copyscape
Cosmos
Craftbot
crawler4j
CrazyWebCrawler
-CrunchBot
Curious
-DBLBot
+Custo
demandbase-bot
+Demon
Deusu
-Digincore
-Dirbuster
+DigitalPebble
+DIIbot
Disco
-Discobot
+Dispatch
DittoSpyder
+DomainAppender
DomainCrawler
+DomainSigmaCrawler
DomainStatsBot
-DTS Agent
-EasyDL
+Dotbot
+Download Wonder
+Dragonfly
+eCatch
Ecxi
EirGrabber
EMail Siphon
-evc-batch
+EMail Wolf
+EroCrawler
Evil
+Exabot
Express WebPictures
ExtLinksBot
-Extractor
-Ezooms
-facebookscraper
-FDM
+EyeNetIE
+Fimap
+FlashGet
+Flunky
Foobot
-FrontPage
FyberSpider
+Fyrebot
GalaxyBot
-Genieo
-GermCrawler
-GetRight
-Go-Ahead-Got-It
-Gotit
+Getintent
+Gigabot
+G-i-g-a-b-o-t
+Go!Zilla
+Grabber
+GrabNet
GrapeFX
-GrapeshotCrawler
GridBot
HaosouSpider
+Harvest
Havij
-Heritrix
-HMView
-HTMLparser
-HTTP::Lite
-HybridBot
-Iblog
-IDBot
-IlseBot
-Indy Library
-InfoNaviRobot
-Intelliseek
-InternetSeer
+HTTrack
+Humanlinks
+Id-search
+Image Fetch
+Image Sucker
+IndeedBot
+InterGET
+Internet Ninja
internetVista monitor
ips-agent
-IRLbot
-Jbrofuzz
-JetCar
-JustView
-Jyxobot
+Iskanie
+IstellaBot
+JamesBOT
+Jetty
+JOC Web Spider
+Keyword Density
Kozmosbot
-LexiBot
+Lftp
Libwhisker
-Lightspeedsystems
Likse
+Linkdexbot
LinkextractorPro
+LinkScan
+LinksManager
LinqiaMetadataDownloaderBot
LinqiaRSSBot
Lipperhey Spider
+Litemage_walker
Lmspider
LNSpiderguy
-Ltx71
+lwp-request
lwp-trivial
-Magnet
-Majestic-SEO
-MauiBot
-MegaIndex.ru
+Mag-Net
+magpie-crawler
+Mail.RU_Bot
+Majestic12
+Majestic SEO
+MarkMonitor
+Masscan
+Mass Downloader
+Mata Hari
+Meanpathbot
+MeanPath Bot
Metauri
-MFC_Tear_Sample
Microsoft Data Access
Microsoft URL Control
MIDown tool
MIIxpc
+Mister PiX
+MJ12bot
Mojeek
-Morfeus Fucking Scanner
-MSFrontPage
-Musobot
+Mr.4x3
+MSIECrawler
Name Intelligence
Nameprotect
Navroad
-NearSite
-NetAnts
-NetLyzer
-Nettrack
-Net Vampire
+Nessus
+netEstate NE Crawler
+NetMechanic
+NetSpider
Netvibes
-NetZIP
NextGenSearchBot
-Nimbostratus
+NimbleCrawler
Nmap
-Nutch
-Octopus
-Offline Explorer
+oBot
Openfind
+OpenLinkProfiler
OpenVAS
+OrangeBot
OrangeSpider
OutfoxBot
-Page Analyzer
-Pandalytics
+PageGrabber
+page scorer
+PageScorer
Panscient
Papa Foto
PECL::HTTP
-Picscout
-Picsearch
-POE-Component-Client-HTTP
-Probethenet
-Psbot
+PeoplePal
+PictureFinder
+ProWebWalker
PyCurl
-Quick-Crawler
RankActive
-RankActiveLinkBot
-RankingBot
+RankFlex
RankingBot2
-RankurBot
-ReGet
-RocketCrawler
-s1z.ru
+Rankivabot
+Reaper
+Recorder
+RedesScrapy
+Rogerbot
ScanAlert
scan.lol
-Scrapy
+ScoutJet
Screaming
-ScreenerBot
Searchestate
SEOkicks-Robot
Seomoz
SEOprofiler
seoscanners
SEOstats
-sexsearcher
Siphon
SISTRIX
-Siteimprove
-SiteSnagger
-Site Sucker
-Sitevigil
-Snoopy
+SMTBot
+Snake
+sogouspider
+Sogou web spider
Spammen
Spanner
-Spbot
SputnikBot
+Sqlmap
Sqlworm
Sqworm
-Steeler
Stripper
-Sucker
SuperHTTP
Surfbot
-sysscan
+Swiftbot
+T8Abot
tAkeOut
-Teleport
-Telesphoreo
+Telesoft
+Telesphorep
Thumbor
TightTwatBot
Titan
+Toata
+Trendiction
+Trendictionbot
trendiction.com
-True_Robot
-Turnitin
-Upflow
+Turingos
+TurnitinBot
+TwengaBot
+URLy Warning
URLy.Warning
Vacuum
-VB Project
-VeriCiteCrawler
-VoidEYE
-Voltron
-WASALive-Bot
+Vagabondo
+VCI
+Virusdie
+Wallpapers/3.0
+Webalta
WebAuto
-WebBandit
+Web Auto
WebCollage
-WebCopier
-WebEnhancer
+Web Enhancer
WebFetch
WebFuck
+Web Fuck
WebGo IS
+WebLeacher
+WebmasterWorldForumBot
+Web Pix
+WebReaper
WebSauger
-Web Sauger
-Webshag
-Website Quester
-WebStripper
-WebWhacker
-WebZIP
+Web Sucker
Whack
-Whacker
-Whatweb
-Who.is Bot
+Widow
WinHTTrack
-Wonderbot
+WiseGuys Robot
Woobot
-Wprecon
WPScan
WWW-Collector-E
WWW-Mechanize
x09Mozilla
x22Mozilla
-Xenu
Zade
Zauba
Zermelo
-Zeus
-zgrab
Zitebot
ZmEu
-ZumBot
diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list
index 4e8cdf219..dca9731b3 100644
--- a/.dev-tools/test_units/random-referrers-for-test-quick.list
+++ b/.dev-tools/test_units/random-referrers-for-test-quick.list
@@ -1,10 +1,10 @@
-4webmasters.org
-htmlcorner.com
-pinwallpaper.top
-prostitutki-yaroslavlya.org
-scanner-walter.top
-shodanhq.com
-tizanidinehydrochloride4mgstreetvalue.blogspot.com
-tizanidinemusclerelaxant.blogspot.com
-usswrite.com
-videochaty.ru
+angigreene.com
+animal36.com
+gaz-voshod.ru
+gorabagrata.ru
+kinopolet.net
+net-profits.xyz
+ru-dety.ru
+stackthatbucks.com
+ublaze.ru
+vkonche.com
diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list
index 29ac04401..ff43645b9 100644
--- a/.dev-tools/test_units/random-referrers-for-test.list
+++ b/.dev-tools/test_units/random-referrers-for-test.list
@@ -1,500 +1,500 @@
-000free.us
-03p.info
-08800.top
0lovespells0.blogspot.com
-1000n1.ru
-1pamm.ru
-256bit.by
-28n2gl3wfyb0.ru
-2ads.co.uk
-2x2fan.ru
-34.gs
-4inn.ru
-4k-player.pl
-4rent.ru
-5forex.ru
-5-steps-to-start-business.com
-6hopping.com
-7a2rnuey1tw9ar.ru
-7zap.com
-8si.ru
-aa8780bb28a1de4eb5bff33c28a218a930.com
-absolutelyanalog.com
-abwa.tk
-actices.com
-activepr.ru
-adbetclickin.pink
-adclickthru.net
+100searchengines.com
+101raccoon.ru
+123kuma.com
+12bet.com
+14b.info
+1bet.com
+2drittel.de
+3dracergames.com
+3w24.com
+45en.ru
+4webmasters.com
+55wmz.ru
+a14download.com
+a1security.com.ua
+abiente.ru
+academia-nsk.org
+acheterviagrafr24.com
+adelly.bg
+adinterax.com
ads-seo.men
-ads.tremorhub.com
-adultfriendfinder.com
-adult-shop.com.ua
-adventureparkcostarica.com
-adverster.com
-advokateg.ru
-adzpower.com
-aerogo.com
-agapovdg.ru
-akita.kz
-alert-fdm.xyz
-alert-hgd.xyz
-alevinefesleri.com
-alfa-medosmotr.ru
-allbizne.co.ua
-allcredits.su
+agreda.pluto.ro
+alfapro.ru
+alienwheels.de
+allblogroll.com
alldubai.biz
-alot.com
-alphaforum.ru
-altamayoztourism.com
-alt-servis.ru
+allmarketsnewdayli.gdn
+aloofly.com
+alpinism.ru
amateurgalls.com
-amospalla.es
-animalrank.com
-anonymous-redirect.com
-apiqualitinknet-a.akamaihd.net
-appartement-stumm.at
-appfastplay.com
-arabsexxxtube.com
-atleticpharm.org
-australia-opening-times.com
-autolombard-krasnodar.ru
-autovideobroadcast.com
-azbuka-mo.ru
-babespcs.com
-bagcionderlab.com
-basedecor.ru
-beclean-nn.ru
-berlininsl.com
-berrymall.ru
-bestmobilityscooterstoday.com
-bigcareer.info
-biglistofwebsites.com
-bimatoprost-careprost.com.ua
-bioscorp.ru
-bizlist.com.de
-bkns.vn
-blogporn.in
-blog.remote-computer.de
+amazingpic.net
+anal-acrobats.com
+anapa-inns.ru
+anlimebel.kiev.ua
+apiallgeniusinfo-a.akamaihd.net
+apiportalmorecom-a.akamaihd.net
+apishops.ru
+appsaurus.com
+aramaicmedia.org
+aruplighting.com
+asiengirls.net
+asscenihotosidea.blogspot.co.za
+astana.xxxkaz.org
+automatic-seo.com
+avek.ru
+avtochehli.by
+avtointeres.ru
+azazu.ru
+azbaseimages.net
+balkanfarma.org
+bamo.xsl.pt
+beachpics.com
+beauby.ru
+best-mam.ru
+bestpriceninja.com
+betterscooter.com
+beyan.host.sk
+bioca.org
+blagovest-med.ru
+blog-hits.com
+blog.koreadaily.com
bloke.com
-boazpower.com
-brainxs.us
-braip.com.br
+bookmark4you.com.biz
+boole.onlinewebshop.net
+break-the-chains.com
+breget74.ru
briomotor.co
-bscodecs.com
-businesxxl.com
-buydissertation.net
-buyessayonline19.blogspot.ru
-buypillsorderonline.com
-bwlx.prepedu.cn
-cablecar.us
-californianews.cf
-call-of-duty.info
+bumascloset.com
+burn-fat.ga
+buron.pw
+business-made-fun.com
+buypuppies.ca
+buzzonclick.com
+bvlgariring.xyz
+cacheimages.com
callstevens.com
-canacopegdl.com
-carsdined.org
-cartechnic.ru
-casite-513049.cloudaccess.net
-cezartabac.ro
+candytiner.com
+capture-room.com
+cardsdumps.com
+cartiertrinity.xyz
+casablancamanor.co.za
+cejewelry.xyz
+centraletermice.us
+centrdebut.ru
+centre-indigo.org.ua
+ceotrk.com
+cfsrating.sonicwall.com
cgi2.nintendo.co.jp
-chatroulette.si
-chinajerseyswholesalecoupons.com
-chlooe.com
-choice-credit.ru
-cibpenonptib.flu.cc
-cigarpass.com
-clevernt.com
-cntravelre.com
-coaching-netz.info
-compliance-barak.top
-compliance-checker-7.info
-compliance-don.xyz
-compliance-george.xyz
-compliance-jess.top
-compliance-walter.top
-copypaste.traderzplanet.in
-counciltally.com
-cowlmash.com
-cubook.supernew.org
-d2jsp.org
-dalavia.ru
-damasarenai.info
-darkages.info
-dayibiao.com
-d-black.bz
-demenageur.com
-destinationrealestate.com
-deutschland123.de
+cheapmoncler.win
+chiblackhawks-jerseys.com
+chidporn.com
+chimiver.info
+chocoslim.pro
+christianlouboutinoutlet.win
+cicaki.net
+ciekawinki.pl
+citysecurity.nu
+cjs.com.ru
+cleanallvirus.com
+clickbank.net
+clicksor.net
+clips.ua.ac.be
+club-lukojl.ru
+compliance-brian.top
+conciergegroup.org
+coslab.club
+cowblog.fr
+csgo4.win
+culturevie.info
+custom-electric-guitar.com
+cyse.tk
+dailyfinancefix.com
+data-mining.tk
+dating-time-now.com
+dealwifi.com
+designdevise.com
+dev-seo.blog
digitalfaq.com
-dimaka.info
-discountbarbour.online
-divatraffic.com
-domaincheck.io
+dignitasdata.se
+dinkolove.ya.ru
+djekxa.ru
+dlya-android.org
+docsportal.net
+documentbase.net
+dosugrostov.site
+dotnetdotcom.org
downloads-whatsapp.com
-drillsaw.ru
-drugs-no-rx.info
+doyouknowtheword-flummox.ml
dyt.net
-easycommerce.cf
-easyshoppermac.com
-editors.choice6912650.hulfingtonpost.com
-elidelcream.weebly.com
-elmifarhangi.com
-enge-fotzen.info
+ecookna.com.ua
+editmedios.com
+ekspertmed.com
+ekto.ee
+electrik-avenue.com
+elektronischezigarettekaufen1.myblog.de
+elektronischezi.livejournal.com
+elvenmachine.com
+engines-usa.com
escort-russian.com
-essay-writing.work
-excitacion.info
+eshop.md
+esnm.ru
executehosting.com
-exmasters.com
-expertblog.info
-extremal-blog.com
-extremez.net
ezigarettenkaufen1.deviantart.com
-ezigaretteshop2.mywapblog.com
-fandlr.com
fermersovet.ru
-fickkontaktehobbyhuren.com
-filkhbr.com
-filmetricsasia.com
-flooringinstallation-edmonton.com
-forms-mtm.ru
-fotoxxxru.com
-fotzen-ficken.com
-foxydeal.com
-freelotto.com
-free-share-buttons.top
-freetangodownload.com
-fungamelands.com
+ferrotodo.com
+filesdatabase.net
+fitfloponline.store
+flavors.me
+f-loaded.de
+floating-share-buttons.com
+florida-tourism.net
+forums.toucharcade.com
+foto-sisek.porngalleries.top
+fototravel.eu
+frcls.fr
+freelifetimefuckbook.com
+fullgirl.ru
+fun-mobi.pl
+fym.com.ru
+g33.org
+gagrasector.ru
+gallerily.com
gallerylisting.com
-gallictures.com
-gamedayassist.com
-gamerscorps.com
+gamebackyard.com
+gamedayhouse.com
+gameprimary.com
gateway.zscalerone.net
-gcup.ru
-geile-lelly.eu
-genetworx.com
-getdot.ru
-getmiro.com
-getoutofdebtfree.org
-get-your-social-buttons.info
-gidonline.one
-girls-ufa.ru
-gloverid.site
-golaya.pw
+gdebestkupit.ru
+gearcraft.us
+getlamborghini.ga
+gettpromos.com
+ggiaro.com
+girlsatgames.ru
+glcomputers.ru
+glicol.kz
+globalscam.ga
+gnuetella.com
golfresa.lucania.se
-gopixdatabase.com
-gotcher.us
-gowreckdiving.com
-gsasearchengineranker.site
-gyffu.com
-hdimagegallery.net
-headpress.ru
-healing-dysplasia.ru
-helpmymacfaster.trade
-hidemyass.com
-high-speed1.net
-hildinghr.se
+gorabagrata.ru
+gwhwpxbw.bloger.index.hr
+hanwei.us
+hdhc.ru
+healgastro.com
+healmytrauma.info
+hebr.myddns-flir.com
+helvetia.com.ua
+hifidesign.ru
+holidaypics.org
+homeart.space
+homedecoguide.info
+honyaku.yahoofs.jp
+hop.clickbank.net
hopeonthestreet.co.uk
-horoshieokna.com
-hostingclub.lk
-hotenergy.ru
-hotloans.ru
-hoverboardforsaledirect.com
-hubbble.com
-humanelydrew.com
-hybrid.ru
+hotxnights.info
+housemilan.ru
+houseofrose.com
+hut1.ru
+hyhj.info
+i4track.net
+idealtits.net
ifmo.ru
-igrovyeavtomaty777.ru
-iideaidekonkatu.info
-imagerydatabase.com
-imediadesk.com
-immobiliaremassaro.com
-impisr.edunsk.ru
-impresagaia.it
-incest-ru.com
-indianmedicaltourismshop.com
+ilte.info
+img.wallpaperstock.net
+iminent.com
+imk.com.ua
+impisr.ru
infektsii.com
-innoslicon.com
+intellektmedia.at
invest-pamm.ru
-iqoption.com
-iscblog.info
-janettabridal.com
-kartiniresto.com
-keyhantercume.com
-keywordblocks.com
-khafre.us
-kichenaid.ru
-kinotorka.ru
-klumba55.ru
-knigonosha.net
-knowyournextmove.com
+iqbazar.ru
+itsdp3.com
+jackwolfskinoutlet.online
+japfm.com
+jimmychoosale.online
+juliadiets.com
+jus0wil.pp.ua
+justucalling32211123456789.tk
+kaidalibor.de
+kashubadesign.ru
+keenoutlet.online
+keki.info
+keyword-suggestions.com
+kiinomaniak.pl
+kino-fun.ru
+kinohall.ru
+kinopolet.net
+klikbonus.com
konkursov.net
-kountrylife.com
-kuzinsp.ru
-kwzf.net
-lacave.ntic.fr
+koronirealestate.gr
+kosmetyki.tm.pl
+kovesszucs.atw.hu
+krasivoe-hd.net
+krasnodar-avtolombards.ru
+krasnodar.ru
+kreativperlen.ch
+kumuk.info
laexotic.com
-lankarns.com
-laptoper.net
-laulini.soclog.se
-law-check-seven.xyz
+lafriore.ru
+languagecode.com
+larger.io
+law-check-nine.xyz
lawyers.cafe
-lechenie-gemorroya.com
-lennatin.info
-lescinq.com
-lflash.ru
+leadwayau.com
+lesbian.xyz
+lietaer.com
+light.ifmo.ru
+linkarena.com
+linkbux.ru
linkredirect.biz
-lion.bolegapakistan.com
-liveinternet.ro
-live-sexcam.tk
-liveshoppersmac.com
-livetsomudvekslingstudent.bloggersdelight.dk
-lkbennettstore.online
-locatellicorretor.com.br
-lombia.com
+liran-locks.com
+lirunet.ru
+loftdigital.eu
+lombardfinder.ru
lonerangergames.com
-losangeles-ads.com
-low-format.ru
+lostfilm-online.ru
ltvperf.com
-lyrics.home-task.com
-lyrster.com
-magazin-pics.ru
-mainlinehobby.net
-make-money-online.com
-makis.nu
-maladot.com
+machicon-akihabara.info
+macotool.com
+mactechinfo.info
+madot.onlinewebshop.net
+makemoneyonline.com
manipulyator-peterburg.ru
-manuscript.su
-masterseek.com
-masthopehomes.com
-mearns-tractors.co.uk
-meds-online24.com
+map028.com
+marcogrup.com
+marcoislandvacations.net
+marketingvici.com
+marmotstore.online
+matematikus.info
+matsdale.com
+mcar.in.ua
+mebel-alait.ru
+medicinacom.ru
+medispainstitute.com.au
mega-bony2017.pl
-members.ghanaweb.com
-menstennisforums.com
-micasainvest.com
-microsoftportal.net
-microstatic.pl
-mindbox.co.za
-minecraft-neo.ru
-mini.7zap.com
-mirmedinfo.ru
-misswell.net
-mister-shop.com
-mistr-x.org
-mixtapetorrent.com
-mixx.com
-mmog-play.ru
-mojaocena.com
-moskva.nodup.ru
+metabar.ru
+microsearch.ru
+misandesign.se
+missvietnam.org
+mnogabukaff.net
+moneyteam24.com
+moscow-clining.ru
+moviezbonkerssk.cf
mowser.com
-mulberryoutletonlineeu.com
-muschisexbilder.com
-musicvidz.ru
-musirc.com
-my-floor.in.ua
-myfreemp3.eu
-myhitmp3.club
-mymobilemoneypages.com
-napalm51.nut.cc
-netpics.org
-nextbackgroundcheck.gq
-nextconseil.com
-ningessaybe.me
-nonews.co
-nordvpn.com
-notebook-pro.ru
-nowtorrents.com
+mrvideospornogratis.xxx
+muzaporn.com
+my-bc.ru
+myftpupload.com
+mylesosibirsk.ru
+myplaycity.com
+nac-bearings.ru
+naperehresti.info
+neks.info
+netanalytics.xyz
+netfacet.net
+netoil.no
+new7ob.com
+news-speaker.com
+nhl09.ru
+nofreezingmac.work
+northfacestore.online
+notasprensa.info
+nvssf.com
nzfilecloud.weebly.com
-odesproperty.com
-oecnhs.info
-office2web.com
-officedocuments.net
-olgacvetmet.com
-oneminutesite.it
-onetravelguides.com
-onlyforemont.ru
-openmultipleurl.com
-opinionreelle.com
-ops.picscout.com
-oyster-green.com
-oz-offers.com
-pacificair.com
-palocco.it
-partner-stop.men
-partnerworkroom.men
-pawli.eu
-p-business.ru
-pearlisland.ru
-peekyou.com
-peugeot-club.org
-pflexads.com
-photostudiolightings.com
-pics-group.com
-pinkduck.ga
+oballergiya.ru
+ohmyrings.com
+okout.ru
+okroshki.ru
+online247.ml
+onload.pw
+o-o-8-o-o.com
+optikremont.ru
+oren-cats.ru
+osagonline.ru
+otbelivanie-zubov.com
+outshop.ru
+papasdelivery.ru
+paramountmarble.co.uk
+partner-pop.men
+pc-services.ru
+pc-test.net
+phobia.us
+photochki.com
+phpdevops.com
+php-market.ru
+picture-group.com
+pintattoos.com
+piratecams.com
+pixelrz.com
+pixgood.com
pizza-tycoon.com
-pk-services.ru
-plastjulgranar.se
-play.leadzupc.com
-play-movie.pl
-polcin.de
-popup-hgd.xyz
-porngalleries.top
-pornofiljmi.com
-pornolook.net
-pornotubexxx.name
-priscilarodrigues.com.br
-qexyfu.bugs3.com
-quickchange.cc
-rankings-analytics.com
-ranksonic.com
-razorweb-a.akamaihd.net
-realmonte.net
+pochtovyi-index.ru
+pons-presse.com
+popunder.net
+pornoforadult.com
+pornoklad.ru
+porno-play.net
+pornosee.info
+porno-video-chati.ru
+pornzone.tv
+pr0fit-b0x.com
+pregnant.guru
+pricheskaonline.ru
+primedice.com
+privhosting.com
+prostitutki-astana.org
+putitin.me
+pxhdwsm.com
+qpypcx.com
+quickcashlimited.com
+quit-smoking.ga
+rankia.com
+rapidgator-porn.ga
realresultslist.com
-rebuildermedical.com
-redirectme.net
-regionshop.biz
-reklama-i-rabota.ru
-remont-fridge-tv.ru
-rewardit.com
-rightenergysolutions.com.au
-rniaeba.ga
-rossmark.ru
-rp9.ru
-ruf777.com
-rus-pornuha.com
-safe-app.net
-samolet.fr
-sanjosestartups.com
-sbf441.com
-scanner-fred.top
-screentoolkit.com
-sdelatmebel.ru
-searchinquire.com
-searchtooknow-a.akamaihd.net
-seasaltwithfood.com
-secretscook.ru
-seonetwizard.com
-seotoolsagency.com
-sexkontakte-seite.com
-sex-pr.net
-sexsaoy.com
-sfj-ror.no
-sharebutton.net
-sherlock.se
-shlyahten.ru
-shopwme.ru
-sirpornogratis.xxx
-sisi-go.ru
-site-speed-checker.site
-sitevaluation.org
+real-time-analytics.com
+rebelmouse.com
+reelheroes.net
+referencemoi.com
+refudiatethissarah.info
+remont-comp-pomosh.ru
+renecaovillasale.online
+renewablewealth.com
+rentalcarnavi.info
+revistaindustria.com
+rewardpoll.com
+ring4rhino.com
+rootandroid.org
+runtnc.net
+saletool.ru
+sanyuprojects.com
+scanmyphones.com
+scanner-olga.top
+scanner-willy.top
+sdrescher.net
+se.bnt-team.com
+seksotur.ru
+semaltmedia.com
+service.adtech.fr
+service.adtech.us
+service-core.ru
+serw.clicksor.com
+sex-dating.co
+sexflirtbook.com
+sextracker.de
+sex-watch.com
+shijian.ac.cn
+shillyourcoins.com
+sibdevice.ru
+sibecoprom.ru
+sientalyric.co
+signx.info
+sinel.info
+sinestesia.host.sk
+sisiynas.ru
+siteaero.com
+siteexpress.co.il
+sitopreferito.it
skylta.com
-skytraf.xyz
-s.lollypopgaming.com
-small-game.com
-snjack.info
+sky-mine.ru
+slavia.info
+slujbauborki.ru
+smallseotools.com
+sms2x2.ru
snjatie-geroinovoy-lomki.ru
-snsdeainavi.info
socialmediasuggest.com
-soda.media
-sofit-dmd.ru
-softonicads.com
-sotechco.co
-sovinsteel.ru
+softxaker.ru
+sonata-arctica.wz.cz
special-porn.com
-spidtest.org
-spylog.com
-spymac.net
-sstroy44.ru
-stauga.altervista.org
+sports-supplements.us
+sprttrack.com
+ssconstruction.co
+stairliftstrue.com
+startufa.ru
+statistici.ro
+statustroll.com
steamoff.net
-stillmiracle.com
-stroyhelp-dv.ru
-sumo.com
-svetodiodoff.ru
-takeprofitsystem.com
-tako3.com
-td-l-market.ru
-tech4master.com
-telefonsexkostenlos.tk
-telesvoboda.ru
-tfxiq.com
-tizanidine4mguses.blogspot.com
-tizanidineandhydrocodone.blogspot.com
-tizanidineoral.blogspot.com
-tizanidinerecreationaluse.blogspot.com
-tizanidineshowupondrugtest.blogspot.com
-tkanorganizma.ru
-todohr.com
-tonerbox.kz
-toondinsey.com
-top10registrycleaners.com
-tradgardspartner.se
-trafficgenius.xyz
+stickers-market.ru
+stjamesschool.info
+strangeduckfilms.com
+strigkaomsk.ru
+studiofmp.com
+superoboi.com.ua
+sygraem.com
+syndicate.fun
+tagil.zrus.org
+taqywu51.soup.io
+taximytishi.ru
+thaismartloan.com
+thebestphotos.eu
+thecounter.com
+the-usa-games.blogspot.com
+thin.me.pn
+tizanidinerestlesslegsyndrome.blogspot.com
+tizanidinesleepaid.blogspot.com
+tizanidinestreetvalue.blogspot.com
+tizanidinevsvalium.blogspot.com
+tmearegion26.com
+tobeyouday.win
+token-lab.org
+tongkatmadura.info
+toolsky.com
+tooplay.com
+torrentgamer.net
+trackzapper.com
traffique.net
-trichizobswiv.agddns.net
-tupper-posuda.ru
-typimga.pw
-tzritel.tk
-uhodzalijami.ru
-unrealcommander.biz
-uptime-alpha.net
-url-img.link
-urlopener.blogspot.com.au
-usadacha.net
-valuado.com
-valueclick.com
-vasileostrovsky-rayon.ru
-vegascosmetics.ru
-vektorpress.ru
-vekzdorov.ru
-vesnatehno.com
-via-energy-order.com
-videojam.tv
-video-production.com
-vikistars.com
-viktoria-center.ru
-vinylvault.co.uk
+traxdom.ru
+truebeauty.cc
+tvteleport.ru
+uamtrk.com
+ukrobstep.com
+ultimateclassicrock.com
+uni.me
+universals.com.ua
+us-america.ru
+usbggettwku.ga
+utiblog.fr
+uytmaster.ru
+via-energy-acquistare.com
+videochat.mx
+villacoloniale.com
+vitanail.ru
viteonlusarezzo.it
-vkak.ru
-vkgaleria.com
-voloslove.ru
-vsdshnik.com
-vtc.pw
-warezaccess.com
-wcb.su
+vksaver-all.ru
+vut.com.ru
+vzlom-na-zakaz.com
webinstantservice.com
-welck.octopis.com
+webix.biz
+webjam.com
+webpromotion.ae
welcomeauto.ru
-wetgames.ru
-while.cheapwebsitehoster.com
-wholesalejerseychinashop.com
-wikes.20fr.com
-wineitudes.wordpress.com
+weprik.ru
+whengirlsgowild.com
+whosonmyserver.com
+williamrobsonproperty.com
+wingsofrefuge.net
wnhjavlhezp.gq
-word-vorlagen.xyz
-wtsindia.in
-xblognetwork.com
-xkaz.org
-x-musics.com
-xn--1-8sbcpb0bdm8k6a.xn--p1ai
-xn------7cdbapdecfd4ak1bn0amjffj7afu3y.xn--p1ai
-xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai
-xn----7sbaphztdjeboffeiof6c.xn--p1ai
-xn--80aagddcgkbcqbad7amllnejg6dya.xn--p1ai
-xn--80ab4aa2g.xn--p1ai
-x-stars.ru
-yachts-cruise.info
+wonderfulflowers.biz
+wosik-dach.service-for-web.de
+wpthemedetector.co.uk
+ww2awards.info
+x-diesel.info
+xerox-douglas.cf
+xn--80ahdheogk5l.xn--p1ai
+xn--h1ahbi.com.ua
+xn----itbeirbjbi7bc6bh2d.xn--p1ai
+xn--lifehacer-1rb.com
+xn--oogle-wmc.com
+xxxtube69.com
yadro.ru
-yaminecraft.ru
-youdesigner.kz
+yebocasino.com
+youandcredit.ru
+youdao.com
+youghbould.wordpress.com
+youjizz.vc
youresponsive.com
-yousense.info
-youthreaders.com
-zaloro.com
-za-music.mymobiplanet.com
-zarajbuilders.com
-zarepta.com
-zetgie.com.pl
-zhacker.net
-znakomstva-moskva77.ru
-zoolubimets.ru
-zoomovies.org
+zakonobosago.ru
+zambini.ru
+zeroredirect11.com
+zeroredirect6.com
+zlatnajesen.com
+zoogdisany.com
diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list
index 90ed5c3a0..b67976ae4 100644
--- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list
+++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list
@@ -1,250 +1,250 @@
-101billion.com
-101raccoon.ru
-1-99seo.com
-1-free-share-buttons.com
-1qingdao.com
-2girls1cup-free.com
-adultfullhd.com
-advokateg.xyz
-agapovdg.ru
-air-edem.ru
-all4wap.ru
-alvtank.se
-amigobulls.com
-anapa-inns.ru
-anmysite.com
-apidiamondatanet-a.akamaihd.net
-apisecretsaucebi-a.akamaihd.net
-appiq.mobi
-automatic-seo.com
-autotop.com.ua
-avkzarabotok.info
-baltgem.com
-bamo.xsl.pt
-batanga.net
-battlecarnival.su
-battleforupdating.site
-battle.net
-bear.gotcher.us
-betune.onlinewebshop.net
-bget.ru
-bmw.afora.ru
-bobinoz.com
-boleznikogi.com
-bonzibuddi.com
+101lesbian.xyz
+11pikachu.ru
+4kplayer.pl
+51.la
+6hopping.com
+7a2rnuey1tw9ar.ru
+8si.ru
+aaronabel.com
+adanih.com
+adultnet.in
+afora.ru
+aliexpresscashback.club
+aloofly.com
+aludecor.info
+amazon-seo-service.com
+animaltoplist.com
+apartamentwroclaw.eu
+apibatbrowsecom-a.akamaihd.net
+apimossnetinfo-a.akamaihd.net
+arenda-yeisk.ru
+auspolice.com
+aversis.net
+axbocz.net
+babyguns.ru
+baersaratov.ru
+bag77.ru
+bankcrediti.ru
+beauty-things.com
+bestadbid.com
+beta.hotkeys.com
+bharatdefencekavach.com
+bitcoinremote.com
+bitnote.co
+blackle.com
+boc.kz
+bombla.org
brillianty.info
-burn-fat.ga
-buzzurl.jp
-californianews.cf
-cartierlove2u.com
-cartierloveringreplica.xyz
-cayado.snn.gr
-ciekawinki.pl
-cineacademy.ru
-cleanpcnow.com
-clicksotrk.com
-collegerentals.com
-compliance-barak.xyz
-compliance-donald.xyz
-compliance-ivan.top
-compliance-nelson.top
-cosmetic.donna7753191.ru
-countercrazy.com
-crd.clan.su
-csgo4.win
-cubook.supernew.org
-czeshop.info
-danceuniverse.ru
-dating-app.ru
-dealwifi.com
-delfin-aqua.com.ua
-den-noch24.ru
-digest-project.ru
-djonwatch.ru
-dneprsvet.com.ua
-dogsrun.net
-dolohen.com
-dosugrostov.site
-easyshoppermac.com
-easyukraine.com
-ecommerce-seo.org
-econom.co
-ekspertmed.com
-electrik-avenue.com
-elektrischezigarette2.devhub.com
-elektronischezigarette2.mex.tl
-elektrozigaretten1.postbit.com
-elitesportsadvisor.com
-eluxer.net
-elvenmachine.com
-eropornosex.ru
-erotiktreff24.info
-eu-cookie-law.blogspot.com
-euromasterclass.ru
-eurosamodelki.ru
-exmasters.com
-fashionavenuegame.com
-feeriaclub.ru
-fikasound.tk
-film-one.ru
-foodcrafts.website
-forpostlock.ru
-fortwosmartcar.pw
-fr-bearings.ru
-funcrushgames.com
-funponsel.com
-gamerextra.com
-gamerscorps.com
-genetworx.com
-gentamicineyedrops.blogspot.com
-getyourimage.club
-gfaq.ru
-go2jump.org
-gov.yanao.ru
-gyffu.com
-hamacapty.com
-health-medical-portal.info
-hem.passagen.se
-hetmanship.xyz
-hkdiiohi.skyrock.com
-hotgirlhdwallpaper.com
-hothot.ru
-housemilan.ru
-huimin764128.com
-hystersister.com
-incanto.in.ua
-ivearchenceinflu.cf
-jav-fetish.site
-jimmychoosale.online
-jumptap.com
-jus0wil.pp.ua
-justucalling32211123456789.tk
-kalandranis.gr
-kerwinandcariza.com
-kinocccp.net
-kinoflux.net
-kmgamex.cf
-kongoultry.net
-ks1234.com
-kurwa.win
-letslowbefast.today
-li-er.ru
-lifebyleese.com
-locimge.pw
-longlifelomilomi.info
-mainhp.net
-mamasuper.prom.ua
-mararoom.ru
-marcogrup.com
-matrixalchemy.com
-mcnamaratech.com
-medosmotr-ufa.ru
-miniature.io
-mir-betting.ru
-mir-business-24.ru
-mokrayakiska.com
-mouselink.co
-mp3films.ru
-mypets.by
-neobux-bg.info
-newsperuse.com
-novosti-hi-tech.ru
-npoet.ru
-oecnhs.info
-offergroup.info
-online-templatestore.com
-ooo-olni.ru
-openfrost.net
-openmediasoft.com
-osoznanie-narkotikam.net
-partnerads.men
-partner-cdn.men
-pinkduck.ga
-pkr1hand.com
-plastjulgranar.se
-poisk-zakona.ru
-pojdelo.weebly.com
+bukleteg.xyz
+buytizanidineonlinenoprescription.blogspot.com
+callcafe.info
+cardsdumps.com
+cfcl.co.uk
+cfsrating.sonicwall.com
+cheapwebsitehoster.com
+christianlouboutinreplica.win
+chuckguilford.com
+ci.ua
+compliance-alex.xyz
+compliance-elena.top
+compliance-elena.xyz
+compliance-jess.top
+compliance-mark.top
+cookielawblog.wordpress.com
+cougfan.info
+dame-ns.kz
+datract.com
+dayibiao.com
+deplim.com
+discovertreasure-a.akamaihd.net
+dms-sw.ru
+doublepimp.com
+downloaddy.net
+dverimegapolis.ru
+earncash.com.ua
+earthmagic.info
+easync.io
+ecommerce-seo.com
+ednorxmed.com
+eduinfosite.com
+eladkarako.github.io
+elektrischezigaretten1.blogse.nl
+el-nation.com
+elstal.com.pl
+enge-fotzen.info
+ereko.ru
+eyemagination.com
+ezaz.info
+fitfloponline.store
+fivedwld.cf
+floridamobilebillboards.com
+flowersbazar.com
+forex-procto.ru
+forodvd.com
+forsex.info
+geilehausfrauen.net
+gemara.com
+germetiki.com.ua
+glasof.es
+goldenggames.com
+googst2.ru
+guge.io
+hccoder.info
+hol.es
+hoste.octopis.com
+hulfingtonpost.com
+ibb.com.ua
+igrovyeavtomaty777.ru
+imagine-ex.co
+i-midias.net.br
+inboxdollars.com
+insurple.com
+internet-apteka.ru
+izhevsk.xrus.org
+james13prix.info
+javlibrary.cc
+kaidalibor.de
+kamen-e.ru
+kemerovo.zrus.org
+keywordglobal.co.za
+kichenaid.ru
+kino2018.club
+kinofak.net
+koleso24.com.ua
+koversite.info
+kozhniebolezni.com
+kuzinsp.ru
+languagecode.com
+lesbian.xyz
+life.biz.ua
+liffa.ru
+liverpool.gsofootball.com
+mackeeper-land-672695126.us-east-1.elb.amazonaws.com
+mahnwachen-helfen.info
+maslenka.kz
+mastershef.club
+meds-online24.com
+mere.host.sk
+mexicosleevegastrectomy.com
+millionare.com
+minneapoliscopiers.com
+mir-limuzinov.ru
+mobplayer.net
+moscow-region.ru
+mostantikor.ru
+muizre.ru
+mycaf.it
+mydirtyhobby.com
+mydirtystuff.com
+myprintscreen.com
+naphukete.ru
+nash-krym.info
+nationalbreakdown.com
+netfacet.net
+next-dentists.tk
+ngps1.ru
+njkmznnb.ru
+nofreezingmac.click
+novatech.vn
+nyfinance.ml
+oconto.ru
+ohmyrings.com
+oil-td.ru
+ossmalta.com
+paphoselectricianandplumber.com
+pathwhelp.org
+perl.dp.ua
+petitions.whitehouse.gov
+pharmacyincity.com
+picturesmania.com
+pistonclasico.com
pokerniydom.ru
-popmarker.com
-porndl.org
-pornoreino.com
-pozvonim.com
-pricheski-video.com
-priscilarodrigues.com.br
-prizestohandle.club
+polybuild.ru
+pornolook.net
+powitania.pl
+printingpeach.com
+profitkode.com
profit-opportunity.com
-puzzleweb.ru
-rcpmda.ikan1080.xyz
-referencemoi.com
-registry-clean-up.net
-registrydomainservices.com
-ric.info
-ruclicks.com
-sbetodiodnye-lampy.ru
-scottbywater.com
-semrush.com
-seoheap.com
-servicecenter.co.ua
-sexad.net
-shtora66.ru
-shymkent.xkaz.org
-signal03.ru
-sildenafilcitratemed.com
-siteonomy.com
-slavia.info
-sodexo.com
-soochi.co
-sosdepotdebilan.com
-sotkal.lark.ru
-spaceshipad.com
-spb-plitka.ru
-starpages.net
-stroy-portal22.ru
-sukirgenk.dvrlists.com
-svensk-poesi.com
-symphonyintegratedhealthcare.com
-takeflyte.com
-tandvardshuset.net
+prokotov.com
+promalp-universal.ru
+py100.ru
+rabotaetvse.ru
+rada.ru
+rank-checker.online
+real-time-analytics.com
+remontbiz.ru
+riders.ro
+rimedia.org
+rniaeba.ga
+royal-investments.net
+sabaapress.com
+sad-torg.com.ua
+scanner-alexa.top
+scanner-nelson.top
+seoholding.com
+seomarketings.online
+sexcamamateurchat.com
+sex-pr.net
+sexsearch.com
+shariki-zuma-lines.ru
+sim-service.net
+sitevaluation.com
+slonechka.ru
+snsdeainavi.info
+socialmediasuggest.com
+soft-terminal.ru
+softtor.com
+solnplast.ru
+stardevine.com
+starwars.wikia.com
+stroydetali.ru
+taaaak.com
tedxrj.com
-thedownloadfromwarez.blogspot.com
-tizanidinehcl2mg.blogspot.com
-tizanidineorflexeril.blogspot.com
+thebluffs.com
+thetoiletpaper.com
+tizanidine4mguses.blogspot.com
+tizanidineforopiatewithdrawal.blogspot.com
+tizanidinehydrochloride4mgstreetvalue.blogspot.com
+tizanidineinfo.blogspot.com
+tizanidinemusclerelaxant.blogspot.com
tizanidineusedfor.blogspot.com
-tizanidinewithdrawalsymptoms.blogspot.com
-tomatis.gospartner.com
-topappspro.com
-torrentdownloadhub.com
-tour-line.net
-toys.erolove.in
-traffique.net
-traffpartners.com
-turbodsp.com
-tvand.ru
-twitlinks.com
-twittrading.com
-uamtrk.com
-uni.me
-vadimkravtcov.ru
-valmetrundan.se
-vapeface.club
-venta-prom.ru
-ventelnos.com
-viagengrarx.com
-visitcambridge.org
-vkontarkte.com
-voloslove.ru
-vote-up.ru
-vseigru.one
-we-are-gamers.com
-welcomeauto.ru
-wesharepics.info
-wheelchairliftsarea.com
-wholesalejerseychinaoutlet.com
-win-spy.com
-w-journal.ru
-worst-sites.online
-xedserver.com
-x-true.info
-yellowads.men
-zhuravlev.info
-z-master.ru
+tizanidinevsvalium.blogspot.com
+toondinsey.com
+toon-families.com
+torontoplumbinggroup.com
+torrent-to-magnet.com
+trackmedia101.com
+tracksz.co
+trafficnetzwerk.de
+try-rx.com
+turizm.bz
+twincitiescarservice.com
+umekana.ru
+unmaroll.ya.ru
+uptime-delta.net
+veles.shop
+videochaty.ru
+videogamesecrets.com
+visa-china.ru
+vzlom-na-zakaz.com
+whereverdesperate.gq
+wurr.voila.net
+x-diesel.biz
+xjlottery.com
+xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai
+xn--80ajjbdhgmudixfjc8c5a9df8b.xn--p1ai
+xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks
+xnxx-n.com
+xportvusbdriver8i.snack.ws
+x-rates.ru
+xtrafficplus.com
+xvideosporn.biz
+yellocloud.be
+yes-com.com
+yougame.biz
+yurgorod.ru
+zagadki.in.ua
+zakon-ob-obrazovanii.ru
+zarenica.net
+zigzog.ru
+zrizvtrnpale.tk
diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt
index 3b5fea028..a7215fb74 100644
--- a/.dev-tools/test_units/ratelimittest-beta.txt
+++ b/.dev-tools/test_units/ratelimittest-beta.txt
@@ -5,7 +5,7 @@
nginx/1.14.2