diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
index 77b3478ad..67e67a4c8 100644
--- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.10.1853
-### Updated: Tue Oct 1 09:44:41 SAST 2019
+### Version: V3.2019.10.1854
+### Updated: Tue Oct 1 11:32:48 SAST 2019
### Bad Referrer Count: 6774
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index 77b3478ad..67e67a4c8 100644
--- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.10.1853
-### Updated: Tue Oct 1 09:44:41 SAST 2019
+### Version: V3.2019.10.1854
+### Updated: Tue Oct 1 11:32:48 SAST 2019
### Bad Referrer Count: 6774
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
index 77b3478ad..67e67a4c8 100644
--- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.10.1853
-### Updated: Tue Oct 1 09:44:41 SAST 2019
+### Version: V3.2019.10.1854
+### Updated: Tue Oct 1 11:32:48 SAST 2019
### Bad Referrer Count: 6774
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
index 77b3478ad..67e67a4c8 100644
--- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.10.1853
-### Updated: Tue Oct 1 09:44:41 SAST 2019
+### Version: V3.2019.10.1854
+### Updated: Tue Oct 1 11:32:48 SAST 2019
### Bad Referrer Count: 6774
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template
index 77b3478ad..67e67a4c8 100644
--- a/.dev-tools/globalblacklist-testing.template
+++ b/.dev-tools/globalblacklist-testing.template
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V3.2019.10.1853
-### Updated: Tue Oct 1 09:44:41 SAST 2019
+### Version: V3.2019.10.1854
+### Updated: Tue Oct 1 11:32:48 SAST 2019
### Bad Referrer Count: 6774
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template
index 03866ef42..0e0b5a154 100755
--- a/.dev-tools/globalblacklist.template
+++ b/.dev-tools/globalblacklist.template
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.10.1853
-### Updated: Tue Oct 1 09:44:39 SAST 2019
+### Version: V4.2019.10.1854
+### Updated: Tue Oct 1 11:32:47 SAST 2019
### Bad Referrer Count: 6774
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
index 03866ef42..0e0b5a154 100644
--- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.10.1853
-### Updated: Tue Oct 1 09:44:39 SAST 2019
+### Version: V4.2019.10.1854
+### Updated: Tue Oct 1 11:32:47 SAST 2019
### Bad Referrer Count: 6774
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
index 0ff8f6817..e22596857 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.192.91.101 1;
+34.66.25.221 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
index 29c63bbe3..e6677be79 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.192.91.101 0;
+34.66.25.221 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
index 03866ef42..0e0b5a154 100644
--- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.10.1853
-### Updated: Tue Oct 1 09:44:39 SAST 2019
+### Version: V4.2019.10.1854
+### Updated: Tue Oct 1 11:32:47 SAST 2019
### Bad Referrer Count: 6774
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
index 0ff8f6817..e22596857 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.192.91.101 1;
+34.66.25.221 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
index 29c63bbe3..e6677be79 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.192.91.101 0;
+34.66.25.221 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
index 03866ef42..0e0b5a154 100644
--- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.10.1853
-### Updated: Tue Oct 1 09:44:39 SAST 2019
+### Version: V4.2019.10.1854
+### Updated: Tue Oct 1 11:32:47 SAST 2019
### Bad Referrer Count: 6774
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
index 0ff8f6817..e22596857 100644
--- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
+++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf
@@ -1,2 +1,2 @@
-35.192.91.101 1;
+34.66.25.221 1;
127.0.0.1 1;
diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
index 29c63bbe3..e6677be79 100644
--- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf
@@ -1,2 +1,2 @@
-35.192.91.101 0;
+34.66.25.221 0;
127.0.0.1 0;
diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
index 03866ef42..0e0b5a154 100644
--- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
+++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.10.1853
-### Updated: Tue Oct 1 09:44:39 SAST 2019
+### Version: V4.2019.10.1854
+### Updated: Tue Oct 1 11:32:47 SAST 2019
### Bad Referrer Count: 6774
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
index 71e2e0bdd..93a98260a 100644
--- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf
@@ -45,5 +45,5 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
+35.202.245.105 0;
34.66.25.221 0;
-104.197.122.201 0;
diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
index b0ddc9537..353fb87d7 100644
--- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.10.1853
-### Updated: Tue Oct 1 09:44:39 SAST 2019
+### Version: V4.2019.10.1854
+### Updated: Tue Oct 1 11:32:47 SAST 2019
### Bad Referrer Count: 6774
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
index f2a1a6b3c..daed34a9d 100644
--- a/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf
@@ -45,5 +45,5 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-35.192.10.37 0;
-35.192.91.101 0;
+34.66.50.208 0;
+35.188.15.155 0;
diff --git a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf
index 93bf21b33..07a809cd2 100644
--- a/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test3_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.10.1853
-### Updated: Tue Oct 1 09:44:39 SAST 2019
+### Version: V4.2019.10.1854
+### Updated: Tue Oct 1 11:32:47 SAST 2019
### Bad Referrer Count: 6774
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
index 28aa6cdd5..825edd288 100644
--- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf
@@ -45,5 +45,6 @@
# - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions.
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-34.66.50.208 0;
+35.188.15.155 0;
+104.197.122.201 0;
34.66.200.49 0;
diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
index 03866ef42..0e0b5a154 100644
--- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.10.1853
-### Updated: Tue Oct 1 09:44:39 SAST 2019
+### Version: V4.2019.10.1854
+### Updated: Tue Oct 1 11:32:47 SAST 2019
### Bad Referrer Count: 6774
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
index a2d4f1546..823bbd202 100644
--- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
+++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf
@@ -46,5 +46,5 @@
# - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
# - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
35.188.15.155 0;
-104.197.122.201 0;
-34.66.25.221 0;
+34.66.200.49 0;
+35.192.91.101 0;
diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
index 03866ef42..0e0b5a154 100644
--- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
+++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf
@@ -4,8 +4,8 @@
### VERSION INFORMATION #
###################################################
-### Version: V4.2019.10.1853
-### Updated: Tue Oct 1 09:44:39 SAST 2019
+### Version: V4.2019.10.1854
+### Updated: Tue Oct 1 11:32:47 SAST 2019
### Bad Referrer Count: 6774
### Bad Bot Count: 556
###################################################
diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list
index 1ccc7f6e0..8c9fd7d66 100644
--- a/.dev-tools/test_units/random-bots-for-test-quick.list
+++ b/.dev-tools/test_units/random-bots-for-test-quick.list
@@ -1,10 +1,10 @@
-Barkrowler
-Brandwatch
-ChinaClaw
-Digincore
-Mag-Net
-Majestic12
-MSFrontPage
-sexsearcher
-Snoopy
-T8Abot
+Anarchie
+DnyzBot
+DomainAppender
+Mail.RU_Bot
+mediawords
+SpaceBison
+Titan
+Trendictionbot
+YoudaoBot
+Zermelo
diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list
index c288ed4ce..0651dc70e 100644
--- a/.dev-tools/test_units/random-bots-for-test.list
+++ b/.dev-tools/test_units/random-bots-for-test.list
@@ -1,250 +1,250 @@
-360Spider
-404checker
404enemy
-Aboundex
-ADmantX
+Aboundexbot
+AhrefsBot
+AiHitBot
Aipbot
-Anarchie
+Alligator
+AllSubmitter
+AlphaBot
Apexoo
-archive.org_bot
ASPSeek
-Asterias
Attach
Backlink-Ceck
-BackWeb
-Bandit
-Barkrowler
+BacklinkCrawler
+BackStreet
BatchFTP
BBBike
-BetaBot
-BLEXBot
+BDCbot
+BDFetch
+Bitacle
+Black Hole
+BlackWidow
Blow
-BlowFish
Boardreader
-BotALot
+Brandprotect
Brandwatch
+Buddy
BuiltBotTough
Bullseye
-BuzzSumo
+BunnySlippers
+CCBot
Cegbfeieh
CheeseBot
-coccocbot-web
+CheTeam
+Cliqzbot
Cogentbot
Collector
-CopyRightCheck
+Cosmos
+Craftbot
crawler4j
crawl.sogou.com
CrazyWebCrawler
-Crescent
-CrunchBot
CSHttp
-Curious
Custo
-DBLBot
-DigitalPebble
-DIIbot
-Dirbuster
+Deusu
Disco
-DomainStatsBot
-Drip
-DTS Agent
+Dispatch
+DittoSpyder
+DomainCrawler
+DomainSigmaCrawler
+Download Wonder
+DSearch
Ebingbong
ECCP/1.0
Ecxi
-Evil
-Express WebPictures
-ExtLinksBot
+EMail Wolf
+EroCrawler
+evc-batch
+Exabot
+Extractor
ExtractorPro
Extreme Picture Finder
EyeNetIE
FDM
+FemtosearchBot
FHscan
-Fimap
-Firefox/7.0
+FlashGet
Flunky
Foobot
+FrontPage
FyberSpider
+Fyrebot
+GalaxyBot
GermCrawler
Getintent
+GetWeb
Gigabot
G-i-g-a-b-o-t
+Go-Ahead-Got-It
Gotit
GoZilla
Go!Zilla
+Grabber
+GrabNet
+Grafula
GrapeFX
-GT::WWW
+GrapeshotCrawler
Haansoft
HaosouSpider
+Heritrix
+Hloader
+HMView
HTMLparser
-HTTrack
-Humanlinks
-HybridBot
-Iblog
-IlseBot
-Image Sucker
-InfoNaviRobot
+IDBot
+Id-search
+Indy Library
+instabid
Intelliseek
+InterGET
+Internet Ninja
InternetSeer
-internetVista monitor
-IRLbot
+Iria
Iskanie
+IstellaBot
JamesBOT
-JennyBot
+JetCar
Jetty
JikeSpider
-JOC Web Spider
Joomla
-Jorgee
-Keyword Density
+Kenjin Spider
Kozmosbot
Larbin
-LeechFTP
-LeechGet
-LexiBot
-Lftp
+LibWeb
+Likse
+LinkextractorPro
+LinkpadBot
LinkScan
-LinkWalker
+LinksManager
LinqiaMetadataDownloaderBot
-LinqiaRSSBot
LNSpiderguy
-LWP::Simple
-Majestic12
+Magnet
+Mag-Net
Majestic SEO
-Majestic-SEO
MarkMonitor
-Masscan
-Mata Hari
+MarkWatch
+Meanpathbot
Mediatoolkitbot
-MegaIndex.ru
+Metauri
Microsoft Data Access
MIDown tool
-Mister PiX
+MJ12bot
Mr.4x3
-MSFrontPage
-MSIECrawler
-Name Intelligence
-Nameprotect
-NetAnts
-Netcraft
-Nettrack
-Netvibes
+Navroad
+Needle
+netEstate NE Crawler
+NetSpider
NextGenSearchBot
NICErsPRO
-Niki-bot
NimbleCrawler
+Nimbostratus
+Nmap
oBot
-Offline Navigator
OnCrawl
-OpenLinkProfiler
OrangeBot
OrangeSpider
-OutclicksBot
OutfoxBot
PageAnalyzer
-PageGrabber
-Pandalytics
Pavuk
-Picsearch
-Pimonster
+pcBrowser
+PECL::HTTP
+Picscout
+PictureFinder
+Pi-Monster
Pixray
PleaseCrawl
-Probethenet
-ProPowerBot
-PyCurl
-QueryN Metasearch
+plumanalytics
+POE-Component-Client-HTTP
+ProWebWalker
+PxBroker
+RankFlex
RankingBot
-Reaper
-Recorder
-RedesScrapy
+RankingBot2
+RankurBot
+RealDownload
+RebelMouse
ReGet
-RepoMonkey
-Ripper
RocketCrawler
-RSSingBot
-ScanAlert
+s1z.ru
Scanbot
-Scrapy
-Screaming
Searchestate
-SearchmetricsBot
-Semrush
SemrushBot
SEOkicks
+SEOkicks-Robot
Seomoz
-SEOstats
-serpstatbot
+SEOprofiler
+SeoSiteCheckup
sexsearcher
-Siphon
-SISTRIX
-Sitebeam
+Shodan
+SiteExplorer
Siteimprove
-Site Sucker
+SiteSnagger
SMTBot
-Snake
Snapbot
-Snoopy
+Sociscraper
+Sogou web spider
Sosospider
Spammen
-Spanner
Spinn3r
SputnikBot
-spyfu
+Sqlmap
Sqlworm
Sqworm
Steeler
+Stripper
Sucker
-Sucuri
+SuperBot
+Surfbot
+SurveyBot
Swiftbot
-sysscan
+Szukacz
+T0PHackTeam
+T8Abot
+tAkeOut
Telesphoreo
Telesphorep
The Intraformant
Thumbor
+TightTwatBot
Titan
-Toata
-Trendiction
+Toweyabot
+Tracemyfile
Trendictionbot
-trendiction.de
True_Robot
Turingos
Twice
-URLy Warning
-Vacuum
+Upflow
VCI
-Virusdie
-Voil
-WallpapersHD
+Voltron
+Wallpapers/3.0
WASALive-Bot
-WBSearchBot
-WebBandit
-WebEnhancer
-WebFetch
+WebCollage
+WEBDAV
Web Fetch
-WebFuck
Web Fuck
WebGo IS
+WebImageCollector
+WebLeacher
+webmeup-crawler
WebPix
Web Pix
+WebReaper
WebSauger
WebsiteQuester
-WebStripper
-WebSucker
-WebWhacker
+Webster
+Web Sucker
WeSEE
-Whack
-Whacker
Whatweb
-Who.is Bot
Widow
Wonderbot
-WPScan
-WWW-Collector-E
-WWW-Mechanize
+Wotbox
WWW::Mechanize
-Xaldon_WebSpider
-YoudaoBot
+x09Mozilla
+x22Mozilla
+Xaldon WebSpider
+Xenu
+xpymep1.exe
Zauba
-zauba.io
+Zeus
Zitebot
-ZmEu
ZumBot
+ZyBorg
diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list
index 4a88a8a4f..9a51f89fb 100644
--- a/.dev-tools/test_units/random-bots-for-whitelist-test.list
+++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list
@@ -1,250 +1,250 @@
-Aboundex
-Acunetix
-ADmantX
-AfD-Verbotsverfahren
-Alligator
+360Spider
+404checker
+404enemy
+Abonti
+AIBOT
+AllSubmitter
+Anarchie
Apexoo
+archive.org_bot
ASPSeek
-Asterias
+autoemailspider
BackDoorBot
+Backlink-Ceck
+BacklinkCrawler
+BackStreet
BackWeb
Badass
Bandit
-Battleztar Bazinga
BBBike
BDCbot
-BDFetch
-BetaBot
Bigfoot
-Bitacle
-Blackboard
-Black Hole
+Blow
BlowFish
Bolt
-BotALot
+Brandprotect
+Buddy
+BuiltBotTough
+Bullseye
BunnySlippers
+BuzzSumo
+CATExplorador
+CCBot
Cegbfeieh
+CherryPicker
CheTeam
-ChinaClaw
-Cloud mapping
-Cogentbot
-Collector
-Copier
-CopyRightCheck
-Cosmos
+Chlooe
+cognitiveseo
+Craftbot
+crawler.feedback
+crawl.sogou.com
CrazyWebCrawler
-CrunchBot
-Curious
Custo
DataCha0s
-demandbase-bot
Demon
+Deusu
DigitalPebble
-DIIbot
-Disco
+Dirbuster
+Discoverybot
Dispatch
DittoSpyder
-DnyzBot
DomainAppender
-DomainSigmaCrawler
-DomainStatsBot
-Dotbot
+DomainCrawler
+Download Wonder
+Dragonfly
+Drip
EasyDL
-Ebingbong
-ECCP/1.0
-Ecxi
+eCatch
+EirGrabber
+EMail Siphon
EMail Wolf
-Express WebPictures
+EroCrawler
Extractor
+ExtractorPro
EyeNetIE
Ezooms
-FemtosearchBot
+Fimap
Firefox/7.0
-Flunky
+FlashGet
Freeuploader
-Fyrebot
+FrontPage
+FyberSpider
GalaxyBot
Genieo
-GetRight
+Getintent
GetWeb
-Gotit
+Gigabot
+Go-Ahead-Got-It
GoZilla
Go!Zilla
-GridBot
-Haansoft
-Harvest
+GrapeFX
+GrapeshotCrawler
+HaosouSpider
Havij
-HEADMasterSEO
+Hloader
+HMView
HTMLparser
-HTTP::Lite
-HybridBot
-Iblog
-IDBot
+HTTrack
IlseBot
+Image Fetch
Image Sucker
+IndeedBot
+Indy Library
InfoTekies
-Intelliseek
-Internet Ninja
+instabid
+internetVista monitor
IRLbot
-JennyBot
-JetCar
+JamesBOT
+JOC Web Spider
+Joomla
Jyxobot
-Keyword Density
-LeechGet
-Lftp
-Likse
-LinkextractorPro
+Kozmosbot
+Larbin
+LeechFTP
+LibWeb
+Linkdexbot
LinkpadBot
LinkScan
-LinqiaScrapeBot
-Lipperhey Spider
-Ltx71
+Lipperhey
+Litemage_walker
+LNSpiderguy
+lwp-request
LWP::Simple
lwp-trivial
-Magnet
-Mag-Net
-Majestic-SEO
-MarkMonitor
-MarkWatch
+Mail.RU_Bot
+Majestic SEO
Masscan
-Mass Downloader
MauiBot
-Meanpathbot
-MeanPath Bot
-Mediatoolkitbot
-mediawords
MegaIndex.ru
+Metauri
+MFC_Tear_Sample
+Microsoft Data Access
+Microsoft URL Control
MIDown tool
-MIIxpc
-Mister PiX
-MJ12bot
-Mojeek
-MSFrontPage
-Msrabot
-Needle
-NetAnts
+Morfeus Fucking Scanner
+Mr.4x3
+muhstik-scan
+Nameprotect
+Nessus
+netEstate NE Crawler
NetMechanic
-NetSpider
Net Vampire
Netvibes
-Nibbler
+NetZIP
+NextGenSearchBot
Niki-bot
+Nikto
NimbleCrawler
-Ninja
-Nutch
+Nmap
+NPbot
oBot
-Octopus
-Offline Navigator
+Offline Explorer
OnCrawl
-Openfind
-OrangeSpider
-OutclicksBot
+OpenVAS
OutfoxBot
-Page Analyzer
-PageGrabber
-page scorer
PageScorer
-Pandalytics
Pavuk
-PECL::HTTP
PeoplePal
+PHPCrawl
Picsearch
PictureFinder
-Pimonster
Pi-Monster
-Pixray
plumanalytics
-ProPowerBot
+Probethenet
ProWebWalker
-QueryN Metasearch
-RankFlex
-RankingBot
+Pump
+RankActiveLinkBot
RankingBot2
-Rankivabot
+RankurBot
Reaper
-RebelMouse
-Recorder
+RedesScrapy
RepoMonkey
-Ripper
-SalesIntelligent
+Rogerbot
+s1z.ru
SBIder
ScanAlert
-Scrapy
+Scanbot
+ScoutJet
SemrushBot
+SEOkicks
+SEOkicks-Robot
+SEOlyticsCrawler
Seomoz
+SEOprofiler
seoscanners
SeoSiteCheckup
-serpstatbot
-sexsearcher
-Shodan
+SEOstats
Siphon
SISTRIX
Sitebeam
+SiteExplorer
Siteimprove
+SiteLockSpider
+SiteSnagger
SiteSucker
-Sitevigil
-SMTBot
+Site Sucker
+SlySearch
+SmartDownload
Snake
+SocialRankIOBot
Sociscraper
sogouspider
-Sosospider
+Sogou web spider
Sottopop
SpaceBison
-Spammen
-SputnikBot
-Sqworm
+Spanner
Steeler
-Sucuri
+Sucker
+SuperBot
SuperHTTP
Surfbot
-SurveyBot
-Suzuran
-sysscan
+Swiftbot
Szukacz
-T0PHackTeam
-Teleport
TeleportPro
-Telesoft
Telesphorep
The Intraformant
-TightTwatBot
+TheNomad
Titan
-Toata
-Tracemyfile
-trendiction.com
+Trendictionbot
trendiction.de
-True_Robot
-TurnitinBot
-Twice
-UnisterBot
-VB Project
+Upflow
+Vacuum
+VeriCiteCrawler
Virusdie
VoidEYE
Voltron
-WASALive-Bot
+Wallpapers/3.0
+WBSearchBot
+Webalta
+WebAuto
+Web Auto
Web Collage
+WebCopier
WEBDAV
-WebFetch
+WebEnhancer
Web Fetch
-WebFuck
-Web Fuck
+WebGo IS
+WebLeacher
WebmasterWorldForumBot
-webmeup-crawler
-Web Pix
-WebReaper
+WebSauger
Web Sauger
-Web Sucker
-WebZIP
+WebsiteQuester
+Webster
+WebStripper
+WebSucker
Whack
-Whatweb
+Who.is Bot
+Widow
WinHTTrack
+WISENutbot
Wonderbot
-Wotbox
+WWW-Collector-E
+WWW-Mechanize
+WWW::Mechanize
WWWOFFLE
x09Mozilla
-Xaldon WebSpider
-Xenu
+x22Mozilla
YoudaoBot
-Zade
zauba.io
-Zermelo
+Zeus
+ZyBorg
diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list
index 036826623..d7a550db7 100644
--- a/.dev-tools/test_units/random-referrers-for-test-quick.list
+++ b/.dev-tools/test_units/random-referrers-for-test-quick.list
@@ -1,10 +1,10 @@
-bluesalt.co
-herokuapp.com
-jerseyschinabizwholesale.us
-kinosed.net
-moesen-ficken.com
-oconto.ru
-vclicks.net
-vip-parfumeria.ru
-xn--d1abj0abs9d.in.ua
-zaloadi.ru
+91abcw.com
+agreda.pluto.ro
+aproposde.com
+asseenontvonline.ru
+clean-start.net
+distver.ru
+lida-ru.com
+shlyahten.ru
+toyota.7zap.com
+ugguk.online
diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list
index 88d19e586..d0e80cacb 100644
--- a/.dev-tools/test_units/random-referrers-for-test.list
+++ b/.dev-tools/test_units/random-referrers-for-test.list
@@ -1,500 +1,500 @@
-12bet.com
-178evakuator178.ru
-202ch.com
-24-ak.ru
-24x7-server-support.site
-2x2fan.ru
-3rbseyes.com
-40cg.com
-4rent.ru
-8558l.jobs.net
-8xv8.com
-98oi.ru
-991mostfm.co.id
-aasoldes.fr
-abusora.com
-actulite.com
-addbin.men
-adloads.net
-adserver-e7.com
-adsloads.com
-adultmeetonline.info
-advancedsoftwaresupport.com
-advokateg.ru
+03e.info
+2nt.ru
+4ur.click
+4ureyesonly.com
+5000-cotydzien.com
+5forex.ru
+5tey7463.weebly.com
+aa9d046aab36af4ff182f097f840430d51.com
+aab94f698f36684c5a852a2ef272e031bb.com
+aanapa.ru
+addray.pro
+adinterax.com
+ads.gold
+aero2.ru
+affrh2015.com
+afmuseum.com
+agahinameh.com
+agapovdg.ru
ahmedabadwebs.com
-ahrntal.verymes.xyz
-airmaxshoes-2016.com
-akama.com
-allcredits.su
-alvtank.se
-ample-awards-today.us
-anal-acrobats.hol.es
+airlandsea.info
+albumsuper.info
+alert-fjg.xyz
+alfabot.xyz
+alienwheels.de
+aliexpress.com
+all4invest.info
+allboard.xobor.de
+alltheviews.com
+aloofly.com
+altamayoztourism.com
+amateurmatch.com
+anaksma.info
+an-donut.com
android-systems.ru
animal36.com
-animaltoplist.com
-apelsinnik.website
-apiappenableinfo-a.akamaihd.net
-apiitsthirteende-a.akamaihd.net
-apiluckyleapnet-a.akamaihd.net
-appleid-verification.com
-aramaicmedia.org
-arendatora.ru
-arpelsreplica.xyz
-atleticpharm.org
-autoblogger24.info
-auto-moto-elektronika.cz
-auto-zapchasti.org
-avkzarabotok.com
-avtovolop.ru
-ayanaspa.com
-b00kmarks.com
-babosas.com
-babs.com.ua
-babyfactory.fr
-backgroundpictures.net
-baladur.ru
-balois.worldbreak.com
-bankiem.pl
-baoxaydung.com.vn
-battleforupdating.site
-bestofferswalkmydogouteveryday.gq
-bestserials.com
-bestwebsiteawards.com
-beta.hotkeys.com
-bigcareer.info
-biglistofwebsites.com
-bildsuche.ru
-bingo8888.com
-biznesluxe.ru
-bjorkbacken.nu
-bkgr.se
+arcteryxstore.online
+asacopaco.tk
+atmagroup.ru
+audiofree.ru
+autoplate.info
+aviav.co
+ayakino.net
+azbuka-mo.ru
+azlex.uz
+b2b-lounge.com
+baersaratov.ru
+bard-real.com.ua
+bashtime.ru
+battle.net
+bazaronline24.ru
+becuo.com
+belinvestdom.by
+bensbargains.net
+best-mam.ru
+bestpriceninja.com
+betune.onlinewebshop.net
+bingoporno.com
+biznesschat.net
+bjgugu.net.ua
blavia.00author.com
+blogs.rediff.com
+bluejays-jerseys.us
bombla.org
-brianhenry.co
-brillianty.info
-brimstonehillfortress.org
-bristolhotel.com.ua
-buketeg.xyz
-bum.com.ru
-buntube.net
-businesxxl.com
-buyfriend.ru
-buy-meds24.com
-buyscabiescream.com
-cannibalcheerleader.com
-canoncdriverq3.pen.io
-carloans.com
-cartierreplica.win
-cartiertrinity.xyz
+boole.onlinewebshop.net
+buchananshardware.com
+buehne-fuer-menschenrechte.de
+bukleteg.xyz
+burkesales.com
+buyantiviralwp.com
+buypillsorderonline.com
+bvlgariring.xyz
+bzero1jewelry.net
+calvet.altervista.org
+candiceloves.us
+candlewooddentalcentre.com.au
+cars-modification.net
cartierwatch.xyz
-cascadelink.org
-ccbill.com
-cementaresearch.se
-cerev.info
-chatroulette.online
-cherrypointplace.ca
-cialisndbrx.com
-clarithromycin500mg.com
-clodo.ru
-club-lukojl.ru
-coldfilm.ru
-comissionka.net
-compliance-alexa.top
-compliance-checker-7.info
-compliance-checker.info
-compliance-elena.xyz
-compliance-willy.top
-com-secure.download
-com-supportcenter.website
-coolingoods.com
-copyrightclaims.org
-cowlmash.com
-cpajunkies.com
-crackguru.tk
-crackzplanet.com
-cs-passion.pl
-cv.wallhade.co
-cymbaltaandtizanidine.blogspot.com
-daisye.top
-daneshetabiat.com
-darkbooks.org
-daydream-studio.ru
-detmebel.su
-detskie-zabavi.ru
-dichvuvesinhhanoi.com
-dieswaene.com
-digitalfaq.com
-dignitasdata.se
-dimaka.info
-distver.ru
-doctissimo.fr
-documentsite.net
-domainanalyzing.xyz
-downloaddy.net
-doxysexy.com
-drugspowerstore.com
-ebooknovel.club
-econom.co
-edelstahlschornstein-123.de
-eduinfosite.com
-egovaleo.it
-elidelcream.weebly.com
-el-nation.com
-empathica.com
+casinos4dummies.co.uk
+caveavins.fr
+cdncash.net
+cfsrating.sonicwall.com
+cheapestjerseysonwholesale.com
+cheapmoncler.xyz
+chinese-amezon.com
+christianlouboutinsaleonline.us
+cialgenisrx.com
+cicaki.net
+coin-hive.com
+compliance-jane.top
+compliance-walt.top
+computernetworksonline.com
+conboy.us
+consorzioilmosaico.org
+construmac.com.mx
+contentlook.co
+corta.co
+counter.bloke.com
+counter.yadro.ru
+creativehutindia.com
+customboxes4less.com
+customcollegeessays.net
+custom-electric-guitar.com
+cypernhuset.se
+dating-time-now.com
+datsun-do.com
+ddlmega.net
+dengi-pod-zalog-nedvizhimosti.ru
+derevesendeco.com
+diamond-necklace.info
+digilander.libero.it
+dimkino.ru
+dipstar.org
+dktr.ru
+doktoronline.no
+dolohen.com
+domcran.net
+domination.ml
+dominterior.org
+donna7753191.ru
+download-wallpaper.net
+drillsaw.ru
+drupa.com
+easync.io
+easyukraine.com
+editmedios.com
+edudocs.net
+ege-essay.ru
+elektrischezigarette2.devhub.com
+elektrischezigaretten1.blogse.nl
+elektronischezigarette2.mex.tl
+elektrozigarette2.wordpressy.pl
+elitedollars.com
+elitepcgames.com
+elmacho.xyz
+elvenmachine.com
+emergencyneeds.org
+emerson-rus.ru
+encodable.com
+enews.tech
enhand.se
-epicbrogaming.com
-eropho.com
-erotik0049.com
-ertelecom.ru
-escortslet.net
+enskedesquashclub.se
+eropornosex.ru
+escortplius.com
+estelight.ru
europages.com.ru
-exchangeit.gq
-exct.net
-exonline.info
-ezigarettekaufen2.mpbloggar.se
-f07.de
-falllow.gq
-fanpagerobot.com
-favoritemoney.ru
-fealq.com
-fikasound.tk
-findacheaplawyers.com
-finemanteam.com
-finstroy.net
-firstdrugmall.ru
-flavors.me
-flooringinstallation-edmonton.com
-flytourisme.org
-fodelsedagspresenter.nu
-footballfarrago.com
-forensicpsychiatry.ru
-forms-mtm.ru
-formulaantiuban.com
-forpostlock.ru
-forums.toucharcade.com
-free-laptop-reward.com
-freemags.cc
-freesitetest.com
-freewhatsappload.com
+extlabs.io
+ezigarettekaufen2.blox.pl
+f1nder.org
+facebook-mobile.xyz
+factorynightclub.com
+famix.xyz
+fanrto.com
+fantasticpixcool.com
+faz99.com
+feorina.ru
+fertilitetsradgivningen.se
+fickblock18.com
+figuringmoneyout.com
+filerockstar298.weebly.com
+filesdatabase.net
+filesvine.com
+fishingwholesale.us
+floridamhca.org
+footbalive.org
+foto-basa.com
+fotoxxxru.com
+fotzen-ficken.com
+foxtechfpv.com
+freenom.link
+free-traffic.xyz
freshsuperbloop.com
-freza-sverlo.ru
-frighteningremain.cf
-front.to
-ftns.ru
-gagrasector.ru
-galaxys6manual.info
-galeon.com
-gambarkataku.co
-gameplexcity.com
-getlamborghini.ga
-girlporn.ru
-girls-ufa.ru
-goblacked.com
-gonextmedia.com
-goodwinmetals.co
-google-liar.ru
-gotcher.us
-gq-catalog.gq
-greatgrace.ru
-growshop.es
-guardlink.org
-habermetre.com
-haikuware.com
-hawaiisurf.com
+fsakhalin.ru
+funtoonez.com
+gavuer.ru
+gembird.com
+gettpromos.com
+gg-arena.ru
+gg.zzyjxs.com
+ghernnqr.skyrock.com
+girlsfuckdick.com
+gk-atlant.info
+global-ics.co.za
+globatur.ru
+goforexvps.com
+goodprotein.ru
+googglet.com
+gorabagrata.ru
+gosreg.amchs.ru
+grtyi.com
+gwebtools.com.br
+gwhwpxbw.bloger.index.hr
+gymi.name
+hahashka.ru
+handicapbathtubarea.com
+hatedriveapart.com
+hazardky.net
+hd-film.pl
+hdseriale.pl
hermesreplica.pw
-hermesreplica.win
-homeart.space
-host-protection.com
-hybrid.ru
+hledejvshopech.cz
+home-task.com
+hostsshop.ru
+hotchatdate.com
+hoverboard360.nl
+howmuchdoestizanidinecost.blogspot.com
+hscsscotland.com
+htmlcorner.com
+huhn.altervista.org
+hut1.ru
hyhj.info
-iccornacircri.cf
-idegenvezeto.eu
-ido3.com
-ilovevitaly.xyz
-images-free.net
-i-midias.net.br
-increasewwwtraffic.info
-infobabki.ru
-invest-pamm.ru
-iqupdatetmz.win
-isotoner.com
-istanbulit.com
-jerseychinabizwholesale.com
-jose.mulinohouse.co
-justbcause.com
-kabbalah-red-bracelets.com
-katadhin.co
-kazan.zrus.org
-keywesthideaways.co
-kichenaid.ru
-kinky-fetishes.com
-kino-filmi.com
-kino-key.info
-kirov.zrus.org
-koleso24.com.ua
-kopihijau.info
-krassh.ru
-kw21.org
-labafydjxa.su
-labelwater.se
-lalalove.ru
-lazymanyoga.com
-lego4x4.ru
-lifeinsurancekb.com
-link.ac
-lisque.batcave.net
+ideibiznesa2015.ru
+ifirestarter.ru
+igrovyeavtomaty777.ru
+ilikevitaly.com
+incest-ru.com
+inclk.com
+inet-traffic.com
+infoupdate.org
+innodgfdriverhm.aircus.com
+insider.pro
+instasexyblog.com
+intim-uslugi.info
+irkutsk.online-podarki.com
+islamtoday.co.za
+jerseysbizwholesalecheap.com
+jmat.cn
+johnrobertsoninc.com
+journeydownthescale.info
+justkillingti.me
+justprofit.xyz
+kasino-money.pw
+kazka.ru
+kerch.site
+khafre.us
+kidd.reunionwatch.com
+kinogolos.ru
+kinohall.ru
+kinoplen.ru
+kinostorm.net
+kiwi237au.tk
+klin3952.ru
+komp-pomosch.ru
+kostenlos-sexvideos.com
+labplus.ru
+lafourmiliaire.com
+lanadelreyfans.us
+lanasshop.ru
+laparfumotec.com
+laudit.ru
+law-check-nine.xyz
+ldrtrack.com
+lida-ru.com
+likesdesign.com
+lineshops.biz
+lion.conboy.us
+liumimgx.pw
+livefixer.com
livejasmin.com
-liveu.infoteka.hu
-lkbennettstore.online
-love-baby.cz
-lutherstable.org
-magazintiande.ru
+lkbennettoutlet.online
+lob.com.ru
+localmatchbook.com
+long-beach-air-conditioning.com
+longlifelomilomi.info
+lovi-moment.com.ua
+lsex.xyz
+lubetube.com
+mackeeper-center.club
+macrotek.ru
magicalfind-a.akamaihd.net
-magnetic-bracelets.ru
-malwareremovalcenter.com
-mansiondelrio.co
-max-eclat.men
-mebel-ekb.com
-medicines-choice.com
-medictube.ru
-medtherapy.ru
-meendo-free-traffic.ga
-mexicosleevegastrectomy.com
-mobioffertrck.com
-modnie-futbolki.net
-mojpreskumanie.com
-monarhs.info
-monclerjacketsoutlet.win
-moneyteam24.com
-moomi-daeri.com
-moz.com
-mp3ringtone.info
+marketland.ml
+matrixalchemy.com
+maximilitary.ru
+maximpartnerspr.com
+maxxtor.eu
+mcar.in.ua
+mediaoffers.click
+mediawhirl.net
+meetingrainstorm.bid
+megahdporno.net
+mezaruk.info
+microsoftportal.net
+miniads.ca
+mlf.hordo.win
+mobile.ok.ru
+modenamebel.ru
+moncleroutletonline.xyz
+mosrif.ru
mrwhite.biz
-my-cash-bot.co
-mzdish.site
-nalogovyykodeks.ru
-naperehresti.info
-net-profits.xyz
-next-dentists.tk
-nextlnk12.com
-nfljerseyscheapchinabiz.com
-nippon-bearings.ru
+mtmtv.info
+mygameplus.ru
+myhitmp3.club
+mymobilemoneypages.com
+myonlinepayday.co
+mypets.by
+mysexpics.ru
+nakozhe.com
+naturalshair.site
+netpics.org
+nextbackgroundcheck.gq
njpalletremoval.com
-nl.netlog.com
-nottyu.xyz
-obnal.org
-odywpjtw.bloger.index.hr
-okroshki.ru
-oksrv.com
-oktube.ru
-olvanto.ru
-omoikiri-japan.ru
-onblastblog.online
-onclkads.com
-onemactrckr.com
-online-sbank.ru
-onlythegames.com
-origin-my.ru
-orsonet.ru
+nordstar.pro
+noscrapleftbehind.co
+nullrefer.com
+obuv-kupit.ru
+odesproperty.com
+offers.bycontext.com
+offer.wpsecurity.website
+oneclickfiles.com
+oneminutesite.it
+opinionreelle.com
+ordernorxx.com
+osagonline.ru
osb.se11.ru
-ostrovtaxi.ru
outclicks.net
oxford-book.com.ua
-partner-pop.men
-pattersonsweb.com
-picturesboss.com
-picturesfrom.com
-pinsdaddy.com
-piski.top
-pixshark.com
+page2rss.com
+paidonlinesites.com
+pandarastore.top
+parajumpersjakkesalgnorge.info
+park.above.com
+pcads.ru
+pc-test.net
+penisvergrotendepillennl.ovh
+phormchina.com
+pic2fly.com
+picsforkeywordsuggestion.com
+piter.xrus.org
+pizza-imperia.com
pkr1hand.com
pk-services.ru
-poem-paying.gq
-poppen-nw.net
-popugauka.ru
-pornhive.org
-porno-chaman.info
+pony-business.com
+popunder.net
+porndroids.com
+pornhub-forum.ga
+pornobrazzers.biz
pornosee.info
-porn-w.org
-potoideas.us
-preparevideosafesystem4unow.site
-primedice.com
-printingpeach.com
-profitkode.com
+pornotubs.com
+porto.abuilder.net
+positive2b.ru
+powc.r.ca.d.sendibm2.com
+pregnant.guru
+pretty-mart.com
+priscilarodrigues.com.br
+privat-girl.net
+privetsochi.ru
+prodess.ru
+profit-opportunity.com
+prointer.net.ua
projefrio.com.br
-promodj.com
-pro-poly.ru
-proposal-engine.com
-pts163.ru
-pufip.com
-pulse33.ru
-pwwysydh.com
-pxhdwsm.com
-qexyfu.bugs3.com
-quickloanbank.com
-rada.ru
-radiogambling.com
-rangjued.com
-realnye-otzyvy.info
-r-e-f-e-r-e-r.com
-reining.lovasszovetseg.hu
-reklamuss.ru
-relayblog.com
-replica-watch.ru
+prom23.ru
+pronekut.com
+propranolol40mg.blogspot.com
+pro-tec.kz
+psvita.ru
+ragecash.com
+rainbowice.ru
+rank-checker.online
+ranksonic.org
+razorweb-a.akamaihd.net
+realmonte.net
+rebelmouse.com
+refads.pro
+registrysweeper.com
+reklama1.ru
+repeatlogo.co.uk
responsinator.com
-restorator-msk.ru
-rewardit.com
-ritlweb.com
-rocketchange.ru
-romhacking.ru
-rostov.xrus.org
-rusprostitute.com
-rvtv.ru
-s8-nowy-wygraj.comli.com
-sad-torg.com.ua
-salut-camp.ru
-sandhillsonline.com
-savetubevideo.com
-sawin.beth.webd.pl
-sbf441.com
-scanner-marwin.top
+rock-cafe.info
+rollercoin.com
+ros-ctm.ru
+royalads.net
+royalvegascasino.com
+ruspdd.com
+saddiechoua.com
+salmonfishingsacramentoriver.com
+scanmyphones.com
+scanner-jane.top
+scanner-josh.top
+scanner-julia.top
+scat.porn
+scrapy.org
+scrnet.biz.ua
sc-specialhost.com
-sdrescher.net
-searchengineranker.email
-search-error.com
-searchwebknow-a.akamaihd.net
-securityallianceservices.com
-segol.tv
-sel-hoz.com
-seocdvig.ru
-seoholding.com
-servisural.ru
-seryeznie-znakomstva.ru
-sexcamamateurchat.com
-sexobzor.info
-sexsearch.com
+searchtooknow-a.akamaihd.net
+secret.xn--oogle-wmc.com
+security60-e.com
+seorankinglinks.us
+serw.clicksor.com
+sex-tracker.de
+sharebutton.org
shisha-swag.de
-shitmovs.com
-shopvilleroyboch.com.ua
-sibvitr.ru
+shoesonlinebuy.xyz
+sibtest.ru
sildenafil-tadalafil.info
-snip.to
-snow.nvr163.com
-socialmonkee.com
-social-s-ggg.xyz
+similardeals.net
+site.ru
+sitevaluation.org
+slkrm.ru
+slonechka.ru
+snowplanes.com
+snyatie-lomki-v-stacionare.ru
+socialbookmarksubmission.org
social-s-hhh.xyz
+sodexo.com
+sogimlecal.tk
+souvenir.cc
+space4update.pw
+spbchampionat.ru
spidtest.space
-spinnerco.ca
-sribno.net
-stanthonyscatholicchurch.org
-startufa.ru
-staticfs.host
-sterva.cc
-stpicks.com
-stroiminsk.com
-stroiminsk.org
-students-cheapskate.ml
-sugarkun.com
+spitfiremusic.com
+sps-shop.com
+starpages.net
+stjamesschool.info
+stmassage.ru
+stockquotes.wooeb.com
+stockspmb.info
+stop-gepatit.te.ua
+streetfooduncovered.com
+studiokamyk.com.pl
superkanpo.com
-supervesti.ru
-svnuppsalaorebro.se
swingerseiten.com
-szamponrevita.pl
tabakur77.com
-tackletarts.co
-taihouse.ru
-takeflyte.com
-techart24.com
-telegraf.by
-telsis.com
-tgtclick.com
-thecoral.com.br
-theguardlan.com
-the-usa-games.blogspot.com
+tacbelarus.ru
+tattomedia.com
+teenbbw.yopoint.in
+texbaza.by
+thecoolimages.net
+theprofitsmaker.net
+thetoiletpaper.com
+thewebsitetemplate.info
+thruport.com
+ticketsys.inetwd.com
timeallnews.ru
-timetorelax.biz
+timhost.ru
+titan-cloud.life
tizanidine4mgstreetvalue.blogspot.com
-tizanidinefibromyalgia.blogspot.com
+tizanidine4mguses.blogspot.com
+tizanidine6mg.blogspot.com
tizanidineformigraines.blogspot.com
-tizanidinehcl4mgtablet.blogspot.com
-tizanidineshowupondrugtest.blogspot.com
-tizanidinewithdrawal.blogspot.com
-tmtrck.com
-tn811.us
-token-lab.org
-tomck.com
-topcar-krasnodar.ru
-top-karkas.ru
-topsy.com
-totu.info
-toys.erolove.in
-tracksz.co
-trackzapper.com
-tsstcorpcddvdwshbbdriverfb.aircus.com
-twelvevisionspartyofcolorado.com
-typer.one
-tytoona.com
-uginekologa.com
-ukkelberg.no
-uptime-delta.net
-urdoot.win
-uroki.net
-urzedowski.eu
-userequip.com
-ustion.ru
-valsalud.com
-vavilone.com
+tizanidineforopiatewithdrawal.blogspot.com
+tizanidineorflexeril.blogspot.com
+tizanidinewithdrawalsymptoms.blogspot.com
+tootoo.to
+trackmedia101.com
+traffixer.com
+tuberkuleznik.ru
+turvgori.ru
+twinderbella.com
+uhod-za-sobakoj.ru
+ukkala.xyz
+unitexindia.com
+unitygame3d.com
+uptimechecker.com
+url2image.com
+us-america.ru
+vchulkax.com
vertaform.com
-video-camer.com
-videochat.life
-video-chat.love
-videochat.world
-vip2ch.com
-vipps.com.my
-vladimir.xrus.org
-voluumtrk.com
-vpnhowto.info
-vroze.com
-vykup-avto-krasnodar.ru
-wahicbefa31.soup.io
-webads.co.nz
-webalan.ru
-webmasters.stackexchange.com
-webscouter.net
-websiteaccountant.de
-websites-reviews.com
-westen-z.life
-westermarkanjou.se
-whereverdesperate.gq
-wholesalejerseysgaa.com
-wildworld.site
-womensterritory.ru
-worst-sites.online
+videogamesecrets.com
+videotuber.ru
+vip.51.la
+vip-parfumeria.ru
+vipsiterip.org
+visionwell.com.cn
+vkmusics.ru
+voloslove.ru
+wallpapersdesk.info
+wapsite.me
+webcamdevochka.com
+webradiology.ru
+welcomeauto.ru
+while.cheapwebsitehoster.com
+wholesalejerseyscheapjerseys.us.com
+wma-x.com
+womama.ru
+wonderfulflowers.biz
+word-vorlagen.net
+works.if.ua
ww2awards.info
-xbaboon.com
-xclicks.net
-x-diesel.info
-xingzi-vision.com
-x-mix.info
-xmladserver.com
-xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai
-xn----7sbbagbq7bd5aheftfllo4m.xn--p1ai
-xn--80ajjbdhgmudixfjc8c5a9df8b.xn--p1ai
-yadro.ru
-yellowstonesafaritours.com
+xchangetrak.com
+xedserver.com
+xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai
+xn----7sbifcamovvfggw9d.xn--p1ai
+xn--80abgj3a5acid6ghs.top
+xn--80aeahghtf8ac5i.xn--p1ai
+xn--h1aakne2ba.xn--p1ai
+y8games-free.com
+yaaknaa.info
+yeartwit.com
ynymnwbm.bloger.index.hr
-yottos.com
-your-bearings.com
-yourmovies.pl
+yougetsignal.com
+youtube-downloader.savetubevideo.com
+ypmuseum.ru
yubikk.info
-yunque.pluto.ro
-yurgorod.ru
-yuweng.info
-zakonobosago.ru
-zeleznobeton.ru
-zeroredirect1.com
-zetgie.com.pl
-zigarettenonlinekaufen1.blox.pl
-zog.link
-zoompegs.com
-zrelaya.pw
+zastenchivosti.net
+zeikopay.com
+zhacker.net
+znakomstvaonlain.ru
+zona-aqua.ru
+zoominfo.com
+zoomovies.org
+zryydi.com
diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list
index d77ba913e..842114383 100644
--- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list
+++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list
@@ -1,250 +1,250 @@
-03p.info
-0912701309f8ce.com
-0daymusic.org
-123cha.com
-abbanreddy.com
-abcguru.xyz
-abctoppictures.net
-acunetix-referrer.com
-adminshop.com
-adultfullhd.com
-adultmeetonline.info
-ad-words.ru
-akama.com
-allcredits.su
-allforminecraft.ru
-allornamenti.com
-alpharma.net
-amazon-seo-service.com
-ample-awards-today.us
-an-donut.com
-aosexkontakte.net
-apiluckyleapnet-a.akamaihd.net
-api.stathat.com
-apptester.tk
-avon-ukraine.com
-azartmix.com
-balans.shahterworld.org
-barbourjackets.online
-bedcapdealers.com
-besttorrentknifta.weebly.com
-bigcareer.info
-biglistofwebsites.com
-boazpower.com
-bodybuilding-shop.biz
+123locker.com
+66cpwgln.space
+7yue.org
+aa9d046aab36af4ff182f097f840430d51.com
+above.com
+adclickthru.net
+adconscious.com
+adpremium.org
+ads.gold
+adviceforum.info
+agroeconom.kz
+alfa9.com
+alfapro.ru
+allesohnegirls.net
+aludecor.info
+alvtank.se
+amoi.tn
+ap.senai.br
+aramaicmedia.org
+arate.ru
+audiofree.ru
+backgroundpictures.net
+bankmib.ru
+bannerconnect.net
+bausparen.bz.it
+beachdriveblog.com
+beauty-b0x.pl
+benchmarkcommunications.co.uk
+best-games.today
+best-seo-solution.com
+bestwebsiteawards.com
+betune.onlinewebshop.net
+bildsuche.ru
+bizru.info
+blackwitchcraft.ru
+boc.kz
boeuklad.com
-bonzibuddi.com
-brandehk.dk
-brk-rti.ru
-b-styles.xyz
-bugof.gq
-buydissertation.net
-buyessayonline19.blogspot.ru
-buyviagraa.blogspot.com
-bvlgariwallet.xyz
-candlehandmade.com
-capacitacionyaprendizaje.com
-cezartabac.ro
-cfsrating.sonicwall.com
-chatroulette.life
-cheapmarmot.online
-checkpagerank.net
-chinese-amezon.com
-cialis-samples.com
-cjs.com.ru
-clickpapa.com
-compliance-barak.top
-compliance-checker-7.info
-consorzioilmosaico.org
-cranly.net
-customchocolate.business-for-home.com
-dafatiri.com
-dairyindia.in
-delta-line.men
-detective01.ru
-dev78.dioniqlabb.se
-diarioaconcagua.com
-dodlive.mil
-dogbreedspicture.net
-domainanalyzing.xyz
-druzhbany.ru
-dynainbox.com
+bonanza-fortune.men
+buybest1.biz
+callcafe.info
+calvet.altervista.org
+camsex.xxx-cam.webcam
+candelluxsklep.pl
+cdn.walkme.com
+cellfun.mobi
+cheatcode-lita12.rhcloud.com
+clippingphotoindia.com
+coaching-netz.info
+compliance-josh.top
+copro.pw
+crafthubs.com
+customergrowthsystems.com
+dealighted.com
+dengi-pod-zalog-nedvizhimosti.ru
+dentalpearls.com.au
+dfiles.me
+diamond-necklace.info
+dikx.gdn
+dinkolove.ya.ru
+djekxa.ru
+domaincrawler.com
+dosug-lux.ru
+downloadeer.net
e705.net
-e-commerce-seo.com
-ed-shop01.ru
-ekaterinburg.xrus.org
-eldiariodeguadalajara.com
-elektrischezigarette2.onsugar.com
-elektronischezigarette2.mex.tl
-elektrozigarette2.wordpressy.pl
-esopini.com
-etenininrade.ga
-europeanwatches.ru
-exchangeit.gq
-extremal-blog.com
+ecommerce-seo.org
+elektronischezigarettekaufen2.tumblr.com
+empathica.com
+ereko.ru
+errorfixing.space
+essay-edu.biz
+ezigaretteshop2.vefblog.net
+f5mtrack.com
+faracontrol.ir
farmingworm.com
-fearcrow.com
-fickkontakte.org
-filunika.com.ru
-finteks.ru
+felizporno.com
+fickluder69.com
+filesclub.net
+findsexguide.com
+finemanteam.com
fok.nl
-forum.tvmir.org
-freelotto.com
-freemags.cc
-freshdz.com
-futbolkisales.ru
-galaxy-family.ru
-gamewrath.com
-generalporn.org
-gk-atlant.info
-glicol.kz
-gnuetella.com
-gwagka.com
-hamilton.ca
-herokuapp.com
-hikesearch.net
-holidaypics.org
-house.sieraddns.com
-hoverboard360.de
-https-legalrc.biz
-hustoon.over-blog.com
-incomekey.net
-investingclub.ru
-invivo.hu
-iopeninghours.co.uk
-irkutsk.zrus.org
-isistaylorporn.info
-jaxcube.info
-jetsli.de
-jewelryandfiligree.com
-j-times.ru
-klumba55.ru
-laapp.com
-labafydjxa.su
-lazymanyoga.com
-likrot.com
-linkredirect.biz
-lisque.batcave.net
-lolnex.us
-lonerangergames.com
-m292.info
-manimpotence.com
-master-muznachas.ru
-masterseek.com
-matchpal-a.akamaihd.net
-mcnamaratech.com
-meet-flirt-dating.com
-mielec.pl
-millionare.com
-miniature.io
-missclub.info
-moscow-clining.ru
-moyaterapiya.ru
-mrpornogratis.xxx
+frighteningremain.cf
+fuel-gas.com
+funcrushgames.com
+gallictures.com
+gamedayassist.com
+gameplexcity.com
+gearsadspromo.club
+getdot.ru
+gettpromos.com
+gidonline.one
+girlspicsa.com
+goldandcard.ru
+gomusix.com
+gosarhivrt.ru
+gourcy.altervista.org
+gratis-sexkontakte.com
+greatfind-a.akamaihd.net
+guigyverpo.cf
+hdpixent.com
+hollywoodactress.info
+hoporno.com
+hosting-tracker.com
+hotenergy.ru
+hotxnights.info
+hoztorg-opt.ru
+hut1.ru
+hystersister.com
+icaseclub.ru
+infektsii.com
+inmate-locator.us
+interferencer.ru
+investzalog.ru
+iqoption.com
+iron-age.info
+iskalko.ru
+jackpotchances.com
+jolic2.com
+jrcigars.com
+kakablog.net
+kanimage.com
+kino-key.info
+kiprinform.com
+kollesa.ru
+leboard.ru
+letslowbefast.site
+lezbiyanki.net
+lion.bolegapakistan.com
+lombardfinder.ru
+louboutinshoes.xyz
+lovelycraftyhome.com
+lyrster.com
+m4ever.net
+manipulyator-peterburg.ru
+mecash.ru
+mega-polis.biz.ua
+merryhouse.co.uk
+microsearch.ru
+migente.com
+mobifunapp.weebly.com
+moesexy.com
+mole.pluto.ro
+mowser.com
+mrwhite.biz
musflashtv.com
-mybinaryoptionsrobot.com
-mymobilemoneypages.com
-naphukete.ru
-nash-krym.info
+myhitmp3.club
+naturalbreakthroughsresearch.com
+nbsproject.ru
+noclegonline.info
+notfastfood.ru
+novgorod.xrus.org
+nullrefer.com
obnal.org
+orsonet.ru
+osb.se11.ru
+oz-offers.com
pacificair.com
-page2rss.com
-paparazzistudios.com.au
-par-fallen.ga
-pastaleads.com
-pattersonsweb.com
-penisvergrotendepillennl.ovh
-picsforkeywordsuggestion.com
-popup-jdh.xyz
-pornhubforum.tk
-pornoforadult.com
-pospr.waw.pl
-pricheski-video.com
-printie.com
-privatefx.all4invest.info
+partner-high.men
+partner-trustworthy.men
+pay2me.pl
+pc-virus-d0l92j2.pw
+plohaya-kreditnaya-istoriya.ru
+popunder.ru
+porn-w.org
+pronorm.fr
purplesphere.in
-pwwysydh.com
-ranksays.com
-redhotfreebies.co.uk
-refads.pro
+purplestats.com
+puzo2arbuza.ru
+raavidesigns.com
+rankscanner.com
+rapidokbrain.com
r-e-f-e-r-e-r.com
-regionshop.biz
-remorcicomerciale.ro
-resant.ru
-retailwith.com
-revistaindustria.com
-reyel1985.webnode.fr
-ring4rhino.com
-rostov.xrus.org
-rubanners.com
-ruscoininvest.company
-russia-tao.ru
-russia-today-video.ru
-s1z.ru
-samo-soznanie.ru
-scanner-brian.top
-scanner-jane.top
-scanner-julianna.top
-sdsjweb.com
-seeresultshub-a.akamaihd.net
+responsive-test.net
+restorator-msk.ru
+reversing.cc
+rewardpoll.com
+sabizonline.com
+sadaholding.com
+scansafe.net
+seinterface.com
seksvideoonlain.com
-seoprofiler.com
-seotoolsagency.com
-sexcamamateurchat.com
-sexuria.net
-sexyteens.hol.es
+sell-fb-group-here.com
+seminarykansas904.ml
+seokicks.de
+seorankinglinks.xyz
+sexs-foto.com
+sexs-foto.top
+shopcheermakeup.info
+sientalyric.co
+simul.co
+sinel.info
siteripz.net
-smartpet.ru
-social-fun.ru
-sockshare.net
-softxaker.ru
-soserfis.com
-spaceshipad.com
-spasswelt.xyz
-spyfu.com
-stathat.com
-statustroll.com
-supervesti.ru
-suture.co
-takethatad.com
-thaisamkok.com
-thetoiletpaper.com
-thfox.com
-tizanidinehydrochloride2mg.blogspot.com
-tizanidinepain.blogspot.com
-tizanidinesleepaid.blogspot.com
-top10registrycleaners.com
-topcar-krasnodar.ru
-torrent-newgames.com
+sivs.ru
+skanninge.se
+slujbauborki.ru
+sms2x2.ru
+snjack.info
+snowplanes.com
+songplanet.ru
+speechfoodie.com
+spinnerco.ca
+sprttrack.com
+stefanbakosab.se
+stjamesschool.info
+stroicol.net
+strongsignal-a.akamaihd.net
+suggest-keywords.com
+swingerseiten.com
+techart24.com
+tedy.su
+theplacetoupdating.pw
+thetattoohut.com
+the-trader.net
+tiandeural.ru
+tomck.com
+torrents.cd
trackzapper.com
-tramadolandtizanidine.blogspot.com
-trudogolik.net
-tubeoffline.com
-ucsol.ru
-urengoy.pro
-valsalud.com
-vertaform.com
-videochat.bz
-video-chat.in
-villacoloniale.com
-vk-mus.ru
-watchmyfb.pl
-wcb.su
-webadvance.club
-whatzmyip.net
-windowssearch-exp.com
-win-spy.com
-winterclassichockeyjerseys.com
-wjgony.com
-worldlovers.ru
-worldmusicfests.com
-wormix-cheats.ru
-xn--b1adccaf1bzj.xn--p1ai
-xn--h1ahbi.com.ua
-xtubeporno.net
-xxxguitars.com
-xz618.com
-yebocasino.com
-zhuravlev.info
-zixizop.net.ru
-zocaparj.kz
+traffic2money.com
+traffixer.com
+trubywriting.com
+tvoystartup.ru
+ufa.xrus.org
+ultimatesetnewfreeallsoftupgradesystems.pw
+unece.org
+uptime-gamma.net
+usadacha.net
+uytmaster.ru
+varikozdok.ru
+vchulkax.com
+veerotech.com
+video-chat.cn
+vseigru.one
+vvon.co.uk
+webmonetizer.net
+webnode.me
+website-audit.com.ua
+websites-reviews.com
+websitevaluebot.com
+whatsappdownload10.com
+wholesalecheapjerseysfree.com
+wineration.com
+worldhistory.biz
+ws.ampower.me
+wurr.voila.net
+xep.info
+xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai
+xn--80abgj3a5acid6ghs.top
+xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks
+xnxxandxvideos.com
+zapiszto.pl
+zytpirwai.net
diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt
index ed30c44f7..a683b3cc9 100644
--- a/.dev-tools/test_units/ratelimittest-beta.txt
+++ b/.dev-tools/test_units/ratelimittest-beta.txt
@@ -5,6 +5,14 @@
nginx/1.16.1