V4.2021.02.2266 [ci skip]

.dev-tools/_robots_input/robots-input.txt

@@ -163,6 +163,7 @@ Flunky
 Foobot
 Freeuploader
 FrontPage
+Fuzz
 FyberSpider
 Fyrebot
 GalaxyBot
@@ -305,6 +306,7 @@ Mr.4x3
 MSFrontPage
 MSIECrawler
 Msrabot
+MTRobot
 muhstik-scan
 Musobot
 Name Intelligence
@@ -332,7 +334,9 @@ NimbleCrawler
 Nimbostratus
 Ninja
 Nmap
+node-fetch
 NPbot
+Nuclei
 Nutch
 oBot
 Octopus
@@ -353,6 +357,7 @@ Page Analyzer
 PageGrabber
 page scorer
 PageScorer
+PageThing.com
 Pandalytics
 Panscient
 Papa Foto
@@ -404,6 +409,7 @@ s1z.ru
 SalesIntelligent
 satoristudio.net
 SBIder
+scalaj-http
 ScanAlert
 Scanbot
 scan.lol

.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf

@@ -1,2 +1,2 @@
-104.154.182.187	1;
+34.122.208.80	1;
 127.0.0.1	1;

.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf

@@ -1,2 +1,2 @@
-104.154.182.187	0;
+34.122.208.80	0;
 127.0.0.1	0;

.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf

@@ -1,2 +1,2 @@
-104.154.182.187	1;
+34.122.208.80	1;
 127.0.0.1	1;

.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf

@@ -1,2 +1,2 @@
-104.154.182.187	0;
+34.122.208.80	0;
 127.0.0.1	0;

.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf

@@ -1,2 +1,2 @@
-104.154.182.187	1;
+34.122.208.80	1;
 127.0.0.1	1;

.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf

@@ -224,6 +224,7 @@
 	"~*(?:\b)Foobot(?:\b|)"		0;
 	"~*(?:\b)Freeuploader(?:\b|)"		0;
 	"~*(?:\b)FrontPage(?:\b|)"		0;
+	"~*(?:\b)Fuzz(?:\b|)"		0;
 	"~*(?:\b)FyberSpider(?:\b|)"		0;
 	"~*(?:\b)Fyrebot(?:\b|)"		0;
 	"~*(?:\b)GalaxyBot(?:\b|)"		0;
@@ -366,6 +367,7 @@
 	"~*(?:\b)MSFrontPage(?:\b|)"		0;
 	"~*(?:\b)MSIECrawler(?:\b|)"		0;
 	"~*(?:\b)Msrabot(?:\b|)"		0;
+	"~*(?:\b)MTRobot(?:\b|)"		0;
 	"~*(?:\b)muhstik-scan(?:\b|)"		0;
 	"~*(?:\b)Musobot(?:\b|)"		0;
 	"~*(?:\b)Name\ Intelligence(?:\b|)"		0;
@@ -393,7 +395,9 @@
 	"~*(?:\b)Nimbostratus(?:\b|)"		0;
 	"~*(?:\b)Ninja(?:\b|)"		0;
 	"~*(?:\b)Nmap(?:\b|)"		0;
+	"~*(?:\b)node-fetch(?:\b|)"		0;
 	"~*(?:\b)NPbot(?:\b|)"		0;
+	"~*(?:\b)Nuclei(?:\b|)"		0;
 	"~*(?:\b)Nutch(?:\b|)"		0;
 	"~*(?:\b)oBot(?:\b|)"		0;
 	"~*(?:\b)Octopus(?:\b|)"		0;
@@ -414,6 +418,7 @@
 	"~*(?:\b)PageGrabber(?:\b|)"		0;
 	"~*(?:\b)page\ scorer(?:\b|)"		0;
 	"~*(?:\b)PageScorer(?:\b|)"		0;
+	"~*(?:\b)PageThing.com(?:\b|)"		0;
 	"~*(?:\b)Pandalytics(?:\b|)"		0;
 	"~*(?:\b)Panscient(?:\b|)"		0;
 	"~*(?:\b)Papa\ Foto(?:\b|)"		0;
@@ -465,6 +470,7 @@
 	"~*(?:\b)SalesIntelligent(?:\b|)"		0;
 	"~*(?:\b)satoristudio.net(?:\b|)"		0;
 	"~*(?:\b)SBIder(?:\b|)"		0;
+	"~*(?:\b)scalaj-http(?:\b|)"		0;
 	"~*(?:\b)ScanAlert(?:\b|)"		0;
 	"~*(?:\b)Scanbot(?:\b|)"		0;
 	"~*(?:\b)scan.lol(?:\b|)"		0;

.dev-tools/beta_conf_files_whitelist/bots.d/custom-bad-referrers.conf

@@ -463,6 +463,7 @@
 	"~*(?:\b)anaksma\.info(?:\b)" 	0;
 	"~*(?:\b)anal\-acrobats\.com(?:\b)" 	0;
 	"~*(?:\b)anal\-acrobats\.hol\.es(?:\b)" 	0;
+	"~*(?:\b)analnoeporno\.tv(?:\b)" 	0;
 	"~*(?:\b)analytics\-ads\.xyz(?:\b)" 	0;
 	"~*(?:\b)ananumous\.ru(?:\b)" 	0;
 	"~*(?:\b)anapa\-inns\.ru(?:\b)" 	0;

.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf

@@ -1,2 +1,2 @@
-104.154.182.187	0;
+34.122.208.80	0;
 127.0.0.1	0;

.dev-tools/referrers-regex-format-whitelist-test.txt

@@ -416,6 +416,7 @@
 "~*(?:\b)anaksma\.info(?:\b)" 	0;
 "~*(?:\b)anal\-acrobats\.com(?:\b)" 	0;
 "~*(?:\b)anal\-acrobats\.hol\.es(?:\b)" 	0;
+"~*(?:\b)analnoeporno\.tv(?:\b)" 	0;
 "~*(?:\b)analytics\-ads\.xyz(?:\b)" 	0;
 "~*(?:\b)ananumous\.ru(?:\b)" 	0;
 "~*(?:\b)anapa\-inns\.ru(?:\b)" 	0;

.dev-tools/referrers-regex-format.txt

@@ -414,6 +414,7 @@
 "~*(?:\b)anaksma\.info(?:\b)" 	1;
 "~*(?:\b)anal\-acrobats\.com(?:\b)" 	1;
 "~*(?:\b)anal\-acrobats\.hol\.es(?:\b)" 	1;
+"~*(?:\b)analnoeporno\.tv(?:\b)" 	1;
 "~*(?:\b)analytics\-ads\.xyz(?:\b)" 	1;
 "~*(?:\b)ananumous\.ru(?:\b)" 	1;
 "~*(?:\b)anapa\-inns\.ru(?:\b)" 	1;

.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf

@@ -45,4 +45,4 @@
 # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. 
 # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
 # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-34.122.208.80           0;
+34.122.173.211          0;

.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf

@@ -1,2 +1,2 @@
-35.193.184.18	1;
+104.154.182.187	1;
 127.0.0.1	1;

.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf

@@ -1,2 +1,2 @@
-35.193.184.18	0;
+104.154.182.187	0;
 127.0.0.1	0;

.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf

@@ -1,2 +1,2 @@
-35.193.184.18	1;
+104.154.182.187	1;
 127.0.0.1	1;

.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf

@@ -1,2 +1,2 @@
-35.193.184.18	0;
+104.154.182.187	0;
 127.0.0.1	0;

.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf

@@ -1,2 +1,2 @@
-35.193.184.18	1;
+104.154.182.187	1;
 127.0.0.1	1;

.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf

@@ -224,6 +224,7 @@
 	"~*(?:\b)Foobot(?:\b|)"		0;
 	"~*(?:\b)Freeuploader(?:\b|)"		0;
 	"~*(?:\b)FrontPage(?:\b|)"		0;
+	"~*(?:\b)Fuzz(?:\b|)"		0;
 	"~*(?:\b)FyberSpider(?:\b|)"		0;
 	"~*(?:\b)Fyrebot(?:\b|)"		0;
 	"~*(?:\b)GalaxyBot(?:\b|)"		0;
@@ -366,6 +367,7 @@
 	"~*(?:\b)MSFrontPage(?:\b|)"		0;
 	"~*(?:\b)MSIECrawler(?:\b|)"		0;
 	"~*(?:\b)Msrabot(?:\b|)"		0;
+	"~*(?:\b)MTRobot(?:\b|)"		0;
 	"~*(?:\b)muhstik-scan(?:\b|)"		0;
 	"~*(?:\b)Musobot(?:\b|)"		0;
 	"~*(?:\b)Name\ Intelligence(?:\b|)"		0;
@@ -393,7 +395,9 @@
 	"~*(?:\b)Nimbostratus(?:\b|)"		0;
 	"~*(?:\b)Ninja(?:\b|)"		0;
 	"~*(?:\b)Nmap(?:\b|)"		0;
+	"~*(?:\b)node-fetch(?:\b|)"		0;
 	"~*(?:\b)NPbot(?:\b|)"		0;
+	"~*(?:\b)Nuclei(?:\b|)"		0;
 	"~*(?:\b)Nutch(?:\b|)"		0;
 	"~*(?:\b)oBot(?:\b|)"		0;
 	"~*(?:\b)Octopus(?:\b|)"		0;
@@ -414,6 +418,7 @@
 	"~*(?:\b)PageGrabber(?:\b|)"		0;
 	"~*(?:\b)page\ scorer(?:\b|)"		0;
 	"~*(?:\b)PageScorer(?:\b|)"		0;
+	"~*(?:\b)PageThing.com(?:\b|)"		0;
 	"~*(?:\b)Pandalytics(?:\b|)"		0;
 	"~*(?:\b)Panscient(?:\b|)"		0;
 	"~*(?:\b)Papa\ Foto(?:\b|)"		0;
@@ -465,6 +470,7 @@
 	"~*(?:\b)SalesIntelligent(?:\b|)"		0;
 	"~*(?:\b)satoristudio.net(?:\b|)"		0;
 	"~*(?:\b)SBIder(?:\b|)"		0;
+	"~*(?:\b)scalaj-http(?:\b|)"		0;
 	"~*(?:\b)ScanAlert(?:\b|)"		0;
 	"~*(?:\b)Scanbot(?:\b|)"		0;
 	"~*(?:\b)scan.lol(?:\b|)"		0;

.dev-tools/test1_conf_files_whitelist/bots.d/custom-bad-referrers.conf

@@ -463,6 +463,7 @@
 	"~*(?:\b)anaksma\.info(?:\b)" 	0;
 	"~*(?:\b)anal\-acrobats\.com(?:\b)" 	0;
 	"~*(?:\b)anal\-acrobats\.hol\.es(?:\b)" 	0;
+	"~*(?:\b)analnoeporno\.tv(?:\b)" 	0;
 	"~*(?:\b)analytics\-ads\.xyz(?:\b)" 	0;
 	"~*(?:\b)ananumous\.ru(?:\b)" 	0;
 	"~*(?:\b)anapa\-inns\.ru(?:\b)" 	0;

.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf

@@ -1,2 +1,2 @@
-35.193.184.18	0;
+104.154.182.187	0;
 127.0.0.1	0;

.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf

@@ -46,4 +46,3 @@
 # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
 # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
 104.154.182.187         0;
-35.193.184.18           0;

.dev-tools/test3_conf_files/bots.d/whitelist-ips.conf

@@ -45,5 +45,5 @@
 # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. 
 # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
 # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
+35.193.184.18           0;
 104.154.182.187         0;
-34.122.208.80           0;

.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf

@@ -45,6 +45,4 @@
 # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. 
 # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
 # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-34.122.208.80           0;
-104.154.182.187         0;
-35.193.184.18           0;
+34.122.173.211          0;

.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf

@@ -45,5 +45,6 @@
 # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. 
 # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
 # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-34.122.208.80           0;
 104.154.182.187         0;
+34.122.173.211          0;
+34.122.208.80           0;

.dev-tools/test_units/blacklist-ips.conf

@@ -1,2 +1,2 @@
-104.154.182.187	1;
+34.122.208.80	1;
 127.0.0.1	1;

.dev-tools/test_units/blacklist-user-agents.conf

@@ -224,6 +224,7 @@
 	"~*(?:\b)Foobot(?:\b|)"		0;
 	"~*(?:\b)Freeuploader(?:\b|)"		0;
 	"~*(?:\b)FrontPage(?:\b|)"		0;
+	"~*(?:\b)Fuzz(?:\b|)"		0;
 	"~*(?:\b)FyberSpider(?:\b|)"		0;
 	"~*(?:\b)Fyrebot(?:\b|)"		0;
 	"~*(?:\b)GalaxyBot(?:\b|)"		0;
@@ -366,6 +367,7 @@
 	"~*(?:\b)MSFrontPage(?:\b|)"		0;
 	"~*(?:\b)MSIECrawler(?:\b|)"		0;
 	"~*(?:\b)Msrabot(?:\b|)"		0;
+	"~*(?:\b)MTRobot(?:\b|)"		0;
 	"~*(?:\b)muhstik-scan(?:\b|)"		0;
 	"~*(?:\b)Musobot(?:\b|)"		0;
 	"~*(?:\b)Name\ Intelligence(?:\b|)"		0;
@@ -393,7 +395,9 @@
 	"~*(?:\b)Nimbostratus(?:\b|)"		0;
 	"~*(?:\b)Ninja(?:\b|)"		0;
 	"~*(?:\b)Nmap(?:\b|)"		0;
+	"~*(?:\b)node-fetch(?:\b|)"		0;
 	"~*(?:\b)NPbot(?:\b|)"		0;
+	"~*(?:\b)Nuclei(?:\b|)"		0;
 	"~*(?:\b)Nutch(?:\b|)"		0;
 	"~*(?:\b)oBot(?:\b|)"		0;
 	"~*(?:\b)Octopus(?:\b|)"		0;
@@ -414,6 +418,7 @@
 	"~*(?:\b)PageGrabber(?:\b|)"		0;
 	"~*(?:\b)page\ scorer(?:\b|)"		0;
 	"~*(?:\b)PageScorer(?:\b|)"		0;
+	"~*(?:\b)PageThing.com(?:\b|)"		0;
 	"~*(?:\b)Pandalytics(?:\b|)"		0;
 	"~*(?:\b)Panscient(?:\b|)"		0;
 	"~*(?:\b)Papa\ Foto(?:\b|)"		0;
@@ -465,6 +470,7 @@
 	"~*(?:\b)SalesIntelligent(?:\b|)"		0;
 	"~*(?:\b)satoristudio.net(?:\b|)"		0;
 	"~*(?:\b)SBIder(?:\b|)"		0;
+	"~*(?:\b)scalaj-http(?:\b|)"		0;
 	"~*(?:\b)ScanAlert(?:\b|)"		0;
 	"~*(?:\b)Scanbot(?:\b|)"		0;
 	"~*(?:\b)scan.lol(?:\b|)"		0;

.dev-tools/test_units/custom-bad-referrers.conf

@@ -463,6 +463,7 @@
 	"~*(?:\b)anaksma\.info(?:\b)" 	0;
 	"~*(?:\b)anal\-acrobats\.com(?:\b)" 	0;
 	"~*(?:\b)anal\-acrobats\.hol\.es(?:\b)" 	0;
+	"~*(?:\b)analnoeporno\.tv(?:\b)" 	0;
 	"~*(?:\b)analytics\-ads\.xyz(?:\b)" 	0;
 	"~*(?:\b)ananumous\.ru(?:\b)" 	0;
 	"~*(?:\b)anapa\-inns\.ru(?:\b)" 	0;

.dev-tools/test_units/random-bots-for-test-quick.list

@@ -1,10 +1,10 @@
-cognitiveseo
-DBLBot
-demandbase-bot
-Image Sucker
-Mb2345Browser
-OPPO A33
-Page Analyzer
-Probethenet
-WebZIP
-WeSEE
+Digincore
+Extractor
+Jyxobot
+MJ12bot
+Searchestate
+SemrushBot
+TheNomad
+TightTwatBot
+URLy Warning
+WebsiteExtractor

.dev-tools/test_units/random-bots-for-test.list

@@ -1,250 +1,250 @@
-AhrefsBot
-Alligator
-AllSubmitter
+404checker
+404enemy
+Abonti
+Aboundexbot
+AlphaBot
 Anarchie
-Anarchy
 Anarchy99
 Ankit
-archive.org_bot
 arquivo-web-crawler
+Aspiegel
 Asterias
-Attach
-autoemailspider
 AwarioSmartBot
-backlink-check
-BackStreet
-BDCbot
-Blackboard
+Backlink-Ceck
+BacklinkCrawler
+Barkrowler
+BatchFTP
+BBBike
+BDFetch
+Bitacle
+Black Hole
+BlackWidow
 BLEXBot
-BlowFish
-Boardreader
 Bolt
-Buddy
-BuiltBotTough
-Bullseye
-CazoodleBot
-CensysInspect
+Brandwatch
+BunnySlippers
+Calculon
+CCBot
+Cegbfeieh
 CheeseBot
-CheTeam
+CherryPicker
+ChinaClaw
+Chlooe
+Claritybot
+Cliqzbot
 Cloud mapping
 CODE87
+Cogentbot
 cognitiveseo
 com.plumanalytics
-Copier
 CopyRightCheck
-Copyscape
-Cosmos
+crawler4j
+crawler.feedback
+crawl.sogou.com
 Crescent
-CrunchBot
+CSHttp
 Custo
-CyotekWebCopy
+DataCha0s
 DBLBot
 demandbase-bot
-Deusu
-Digincore
-DigitalPebble
+Devil
+Discobot
+Dispatch
+DittoSpyder
+DomainAppender
 DomainCrawler
-DomainSigmaCrawler
-Domains Project
-domainsproject.org
-DomainStatsBot
-Dotbot
-DSearch
-EasyDL
+DTS Agent
 Ebingbong
 eCatch
-Ecxi
-EirGrabber
-EMail Wolf
+EMail Siphon
 Evil
-ExtLinksBot
-ExtractorPro
-Ezooms
-facebookscraper
-FDM
-FemtosearchBot
+Exabot
+Express WebPictures
+Extractor
 FHscan
-Firefox/7.0
+Fimap
+Foobot
 FrontPage
+Fuzz
 FyberSpider
 Fyrebot
 GalaxyBot
-Getintent
+Genieo
 GetRight
+Gigablast
 G-i-g-a-b-o-t
-Gotit
-Grafula
+GridBot
 Haansoft
-HEADMasterSEO
-heritrix
-Heritrix
-Hloader
-HTMLparser
-HTTP::Lite
+HaosouSpider
+Harvest
+HMView
+HTTrack
 Humanlinks
 HybridBot
-Iblog
 IDBot
+IDBTE4M
 IlseBot
+Image Fetch
 IndeedBot
+Indy Library
+InfoNaviRobot
+InfoTekies
+InternetSeer
 internetVista monitor
-Iskanie
+isitwp.com
+IstellaBot
 JamesBOT
 Jbrofuzz
-JikeSpider
+JennyBot
+JetCar
+Joomla
 Jorgee
-Kinza
-Lftp
-Libwhisker
-LinkextractorPro
-LinkpadBot
-LinkScan
-LinqiaMetadataDownloaderBot
-LinqiaRSSBot
+JustView
+Jyxobot
+Keyword Density
+Larbin
+LeechFTP
+LeechGet
+LibWeb
+LieBaoFast
+Likse
+LinksManager
+LinqiaScrapeBot
 Lipperhey
+Lmspider
 Ltx71
 lwp-request
+LWP::Simple
+Mag-Net
 magpie-crawler
-Majestic SEO
-Majestic-SEO
-MarkMonitor
 MarkWatch
-Masscan
 Mass Downloader
 Mata Hari
-Mb2345Browser
-meanpathbot
-MeanPath Bot
+MauiBot
+mediawords
+MFC_Tear_Sample
 MicroMessenger
 Microsoft Data Access
 MIDown tool
+Mister PiX
+Mojeek
 Mojolicious
-Morfeus Fucking Scanner
 MQQBrowser
-MSFrontPage
-Msrabot
-muhstik-scan
+Musobot
 Name Intelligence
-Netcraft
-netEstate NE Crawler
-NetLyzer
+Navroad
+NetAnts
+NetMechanic
 Net Vampire
-Netvibes
-NetZIP
 NextGenSearchBot
+NICErsPRO
 Niki-bot
+Nimbostratus
 Ninja
 Nmap
-Nutch
+Nuclei
 oBot
+Octopus
 Offline Explorer
 OnCrawl
+Openfind
+OpenVAS
 OPPO A33
 OrangeBot
-OrangeSpider
+OutclicksBot
 PageAnalyzer
-Pandalytics
+page scorer
 Panscient
+Pavuk
+PECL::HTTP
+PeoplePal
 Picsearch
-PictureFinder
 Pimonster
-Pixray
-plumanalytics
+Pi-Monster
 Pockey
-POE-Component-Client-HTTP
-polaris version
-Probethenet
-Quick-Crawler
-RankFlex
-Rankivabot
-Reaper
-RebelMouse
+PxBroker
+QueryN Metasearch
+RankActiveLinkBot
+RankingBot2
 Recorder
-Ripper
+RedesScrapy
 RocketCrawler
-Rogerbot
 RSSingBot
+SalesIntelligent
+satoristudio.net
 SBIder
-ScanAlert
-ScoutJet
-Scrapy
-Screaming
-ScreenerBot
+scalaj-http
+Scanbot
 SearchmetricsBot
-SEOkicks
-Seomoz
+Semrush
+SEOlyticsCrawler
+SeoSiteCheckup
+SEOstats
 sexsearcher
-Siphon
+Shodan
 SISTRIX
 Sitebeam
-sitechecker.pro
+SiteExplorer
+SiteSnagger
 SlySearch
-SmartDownload
-Snake
 Snapbot
+Snoopy
 SocialRankIOBot
 Sogou web spider
 Sosospider
-Spammen
+Sottopop
 SpankBot
+Spanner
 sp_auditbot
-Spbot
-Spinn3r
-Sqlmap
-Stripper
+Surfbot
 SurveyBot
+Swiftbot
+T0PHackTeam
 T8Abot
 Teleport
+TeleportPro
 Telesoft
-Telesphorep
-TightTwatBot
+Thumbor
 Titan
-Toata
 Toweyabot
-Tracemyfile
-Trendiction
 trendiction.de
-True_Robot
 Turnitin
-UnisterBot
+TwengaBot
+Twice
+Typhoeus
 URLy.Warning
-Vacuum
-VelenPublicWebCrawler
 VeriCiteCrawler
-VidibleScraper
-Virusdie
 VoidEYE
 Voltron
 Wallpapers/3.0
-WASALive-Bot
+Webalta
+WebAuto
 WebCollage
 WEBDAV
-Web Fetch
-WebFuck
+WebEnhancer
+Web Enhancer
+WebFetch
 Web Fuck
-WebImageCollector
+WebGo IS
+WebmasterWorldForumBot
 webmeup-crawler
-WebReaper
+Web Pix
+WebSauger
+WebsiteExtractor
 WebsiteQuester
-WebSucker
-Web Sucker
+Website Quester
+WebStripper
 Whack
-Whatweb
-Who.is Bot
-Widow
+WiseGuys Robot
 WISENutbot
-Wonderbot
-Woobot
 Wprecon
-WPScan
-WWW-Collector-E
-xpymep1.exe
-YoudaoBot
+x09Mozilla
 Zauba
+Zermelo
 Zeus
 zgrab
-Zitebot
 ZmEu
 ZoomBot
 ZoominfoBot
+ZumBot
 ZyBorg

.dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404enemy
-80legs
-Abonti
+404checker
 Aboundexbot
 Acunetix
 ADmantX
-AfD-Verbotsverfahren
-AhrefsBot
-AIBOT
-Alligator
-AllSubmitter
+AiHitBot
 Anarchie
+Anarchy
 Anarchy99
+Ankit
 Anthill
-Aspiegel
-ASPSeek
-autoemailspider
+AwarioRssBot
 AwarioSmartBot
-BackDoorBot
-Backlink-Ceck
-BacklinkCrawler
-BackStreet
+backlink-check
 BackWeb
-Battleztar Bazinga
+Badass
+Barkrowler
+BatchFTP
+BBBike
+BDFetch
 BetaBot
-Blackboard
 Black Hole
-BLEXBot
-BotALot
-Brandprotect
-BuiltWith
+BlackWidow
+BlowFish
+Boardreader
+Bolt
+Buck
+BuiltBotTough
 BuzzSumo
-Calculon
-CATExplorador
 CazoodleBot
+CensysInspect
 CherryPicker
-CheTeam
 ChinaClaw
-Chlooe
-Claritybot
-Cogentbot
-cognitiveseo
-Collector
+Cloud mapping
+com.plumanalytics
 Copier
+CopyRightCheck
+Cosmos
+CrazyWebCrawler
 CrunchBot
-Custo
-DatabaseDriverMysqli
-DataCha0s
+CSHttp
 Demon
 Deusu
 Devil
-DigitalPebble
-DIIbot
 Disco
 Discoverybot
 Dispatch
 DittoSpyder
-DnyzBot
-DomainStatsBot
-Dotbot
-Download Wonder
-Drip
+DomainAppender
+DomainSigmaCrawler
+domainsproject.org
 EasyDL
+eCatch
 ECCP/1.0
-Ecxi
-EirGrabber
-EMail Siphon
 EMail Wolf
 EroCrawler
-evc-batch
 Evil
+Exabot
+Express WebPictures
+ExtLinksBot
 ExtractorPro
 Extreme Picture Finder
-Ezooms
 facebookscraper
-FemtosearchBot
-Flunky
-Foobot
+FHscan
+Fimap
+Firefox/7.0
 Freeuploader
-FrontPage
 FyberSpider
-Fyrebot
 GalaxyBot
-Genieo
 Getintent
-GetWeb
-Gigabot
-Go-Ahead-Got-It
-Gotit
+GoZilla
+GrabNet
 Grafula
-GrapeshotCrawler
-GridBot
 GT::WWW
-Haansoft
-HaosouSpider
-Harvest
 HEADMasterSEO
-heritrix
-HTTP::Lite
+Heritrix
+Hloader
+HMView
+Humanlinks
 Iblog
 IDBot
+IDBTE4M
+Id-search
+IlseBot
+Image Fetch
 Indy Library
 InfoNaviRobot
-instabid
 InterGET
 Internet Ninja
-Iria
+InternetSeer
+internetVista monitor
+IstellaBot
 JamesBOT
-JetCar
-Jetty
-JOC Web Spider
-Jyxobot
+JennyBot
+Jorgee
+Kozmosbot
 Larbin
 LeechFTP
-LeechGet
-LexiBot
 Libwhisker
-LieBaoFast
+Lightspeedsystems
+Linkbot
 LinkextractorPro
 LinkpadBot
-LinksManager
+LinkWalker
 LinqiaMetadataDownloaderBot
-LinqiaScrapeBot
+LinqiaRSSBot
 Lipperhey
-Ltx71
-lwp-request
-lwp-trivial
-Magnet
+Lipperhey Spider
+Mag-Net
+Mail.RU_Bot
+Majestic12
 Majestic SEO
 MarkMonitor
+MarkWatch
+Masscan
 Mata Hari
-MauiBot
 Mb2345Browser
-MeanPath Bot
-Mediatoolkitbot
-mediawords
+Meanpathbot
+MegaIndex.ru
 Metauri
+MFC_Tear_Sample
 Microsoft URL Control
-MIDown tool
-MIIxpc
 Mister PiX
+MJ12bot
+Mojeek
 Mojolicious
-Morfeus Fucking Scanner
-Mr.4x3
+MQQBrowser
 Msrabot
-Navroad
+MTRobot
+Musobot
+Nameprotect
 NearSite
-Needle
+netEstate NE Crawler
 NetMechanic
+Nettrack
 Net Vampire
-NextGenSearchBot
+NetZIP
+Nibbler
 Nikto
+NimbleCrawler
+Ninja
 Nmap
+NPbot
+Nutch
 oBot
-Octopus
-Offline Explorer
-OnCrawl
+Openfind
+OpenLinkProfiler
+Openvas
 OpenVAS
+OPPO A33
 OrangeSpider
-Pandalytics
+PageAnalyzer
+Page Analyzer
+PageGrabber
+PageScorer
+PageThing.com
 Panscient
-Papa Foto
-PeoplePal
-PHPCrawl
+PECL::HTTP
 Picscout
-Picsearch
-Pixray
-PleaseCrawl
-POE-Component-Client-HTTP
-Probethenet
-ProWebWalker
+PictureFinder
+Pimonster
+Pi-Monster
+ProPowerBot
+Pump
 PxBroker
 QueryN Metasearch
-Quick-Crawler
-RankActive
 RankActiveLinkBot
-RealDownload
-Recorder
+RankingBot
+Reaper
+RebelMouse
+RedesScrapy
+RepoMonkey
 Rogerbot
+RSSingBot
 SBIder
+ScanAlert
 Scanbot
-scan.lol
 ScoutJet
-ScreenerBot
-Searchestate
-SemrushBot
-SEOprofiler
-SeoSiteCheckup
+Scrapy
+SEOkicks-Robot
+SEOlyticsCrawler
+seoscanners
 sexsearcher
-SiteExplorer
-Siteimprove
+Siphon
+SISTRIX
+sitechecker.pro
+SiteSnagger
+SiteSucker
+Sitevigil
+SlySearch
 SmartDownload
-Snake
-Snapbot
-Snoopy
-Sogou web spider
+SMTBot
+SocialRankIOBot
+Sociscraper
 Sosospider
-SpaceBison
-Spammen
-SpankBot
-Stripper
+Sottopop
+Spanner
+sp_auditbot
+Sqworm
 Sucker
+Sucuri
 SuperBot
-SuperHTTP
 Surfbot
 Suzuran
-Swiftbot
-sysscan
 T8Abot
-tAkeOut
-Teleport
 TeleportPro
-TheNomad
+Telesoft
+Telesphoreo
+Telesphorep
 Thumbor
-Trendictionbot
-trendiction.de
-True_Robot
-Twice
-URLy Warning
-URLy.Warning
-VCI
+Tracemyfile
+Turingos
+VB Project
+VelenPublicWebCrawler
 VeriCiteCrawler
-WBSearchBot
+VoidEYE
+Wallpapers/3.0
+WASALive-Bot
 WebAuto
+Web Auto
 WebBandit
-Web Collage
-WEBDAV
-Web Enhancer
+WebEnhancer
+WebFuck
+Web Fuck
+WebImageCollector
+WebLeacher
 WebmasterWorldForumBot
 webmeup-crawler
-WebsiteQuester
+WebPix
+Web Sauger
+WebsiteExtractor
 WebStripper
 WebSucker
+Web Sucker
 WebWhacker
-WeSEE
-Whack
 Whatweb
+WiseGuys Robot
+WISENutbot
+Wonderbot
 Woobot
+Wotbox
 Wprecon
-WWW-Collector-E
-x09Mozilla
-x22Mozilla
-Xenu
-Zade
+WPScan
+WWW::Mechanize
+Xaldon WebSpider
+Xaldon_WebSpider
 Zauba
 Zermelo
-Zitebot
-ZmEu
+Zeus
 ZoominfoBot
-ZumBot
+ZyBorg

.dev-tools/test_units/random-referrers-for-test-quick.list

@@ -1,10 +1,10 @@
-bookhome.info
-com-supportcenter.website
-ege-russian.ru
-gamerextra.com
-monthlywinners.com
-mrvideospornogratis.xxx
-printdirectforless.com
-site.ru
-supermama.top
-veles.shop
+alfa-img.com
+exchangeit.gq
+focalink.com
+iinstalll-fii1leis.jus0wil.pp.ua
+mega-bony-2017.pl
+novosibirsk.xrus.org
+xn----9sbebi2bvzr7h.xn--p1ai
+xtraffic.plus
+xxlargepop.com
+zeroredirect2.com

.dev-tools/test_units/random-referrers-for-test.list

@@ -1,500 +1,500 @@
-1001desert.com
-1adult.com
-1xbet4you.com
-21h2o.com
-3w24.com
-5ws.dating-app.ru
-777-club.ru
+01apple.com
+0912701309f8ce.com
+123kuma.com
+123locker.com
+1bet.com
+1pamm.ru
+214jaluwobapef.cf
+273-fz.ru
+28n2gl3wfyb0.ru
+2x2fan.ru
+4kepics.com
+5u.com
+7search.com
+7wind.ru
+83net.jp
+8xv8.com
+91abcw.com
 a14download.com
-a1security.com.ua
-adman.gr
-adultmeetonline.info
+aa625d84f1587749c1ab011d6f269f7d64.com
+abcdeg.xyz
+adclickthru.net
+add-add.men
+addray.pro
+adimmix.com
+adnotbad.com
+adsfresh.men
+adultfullhd.com
 advokateg.ru
-akuhni.by
+aghanyna.com
+aihelen.net
+aktivator-windows10.blogspot.com
 albamargroup.com
-alevinefesleri.com
-allboard.xobor.de
-allmarketsnewdayli.gdn
-allwidewallpapers.com
-amateurlivechat.org
+alborzan.com
+albuteroli.com
+alert-fdm.xyz
+alert-hgd.xyz
+alfa-img.com
 amehdaily.com
-amung.us
-anal-acrobats.hol.es
-anonymizeme.pro
-aosheng-tech.com
-apiitsthirteende-a.akamaihd.net
-apilinkswiftco-a.akamaihd.net
-appfastplay.com
-applicationg29.com
-aquarium-pro.ru
-arendovalka.xyz
-arewater.com
-artdestshop.eu
+andpolice.com
+anlimebel.kiev.ua
+apibrowsesmartne-a.akamaihd.net
+app5.letmacworkfaster.world
+arabsexxxtube.com
+arate.ru
+arcadepages.com
 artel116.ru
-aruplighting.com
-asiavirtualsolutions.com
-asscenihotosidea.blogspot.co.za
-atley.eu.pn
-australia-opening-times.com
-aviav.ru.com
-azartclub.org
-azartmix.com
-azinoofficial777.ru
-b2b-lounge.com
-backlinkwatch.com
-balook.com
-bannerbank.ru
-bannerspace.com
-battle.net
-bbtec.net
-bear.gotcher.us
-best-games.today
-besthoro.ru
-bestofferswalkmydogouteveryday.gq
-best-seo-solution.com
-betterscooter.com
-betwinservice.com
-bget.ru
-bikini-image.com
-biographiya.com
-bio-market.kz
-bitcoinpile.com
-blog.axant.it
-blog.koreadaily.com
+asia-forum.ru
+asscenihotosidea.blogspot.com
+autonew.biz
+avirasecureserver.com
+avtoarenda.by
+azazu.ru
+azbuka-mo.ru
+azlex.uz
+baladur.ru
+banki76.ru
+bard-real.com.ua
+batanga.net
+bavsac.com
+bazaronline24.ru
+beauty-b0x.pl
+beauty-clinic.ru
+berdasovivan.ru
+best-coupon-offer.com
+bet365.com
+betonka.pro
+bigpictures.club
+bimatoprost-careprost.com.ua
+bizfly.info
+blavia.00author.com
+blockworld.ru
 blogos.kz
-boeuklad.com
-bookhome.info
-boom-celebs.com
-bosman.pluto.ro
-brandehk.dk
-bristolhotel.com.ua
-buehne-fuer-menschenrechte.de
-burkesales.com
-buron.pw
-businescoop.men
-candcstuccoandstone.com
-candelluxsklep.pl
-candytiner.com
-carina-sy.de
-carstrends2015.com
-cartierreplica.xyz
-casinos4dummies.co.uk
-caulderwoodkennels.com
-ceotrk.com
-cheapestjerseysonwholesale.com
-cheapestjerseys-wholesale.com
-cheapjerseysfootballshop.com
-cheapwebsitehoster.com
+blogsfunky672.weebly.com
+bodybuilding-shop.biz
+bonux.nextview.ru
+bosefux.esy.es
+breget74.ru
+builtwith.com
+buketeg.xyz
+burn-fat.ga
+butstrap.space
+buy-loft.ru
+buytizanidineonline.blogspot.com
+bycontext.com
+callawaygolfoutlet.online
+callejondelpozo.es
+camsex.xxx-cam.webcam
+candypeople.se
+capture-room.com
+cardiosport.com.ua
+cartierwatch.xyz
+casablancamanor.co.za
+cashonads.com
+ccbill.com
+cezartabac.ro
+cfacarrosserie74.com
+charmstroy.info
+cheapbarbour.online
+checkm8.com
+chidporn.com
 christianlouboutinreplica.win
-cialis-samples.com
-cicaki.net
-cienum.fr
-cinemaenergy-hd.ru
-cinemaflix.website
-cjs.com.ru
-clayvasedesigns.tk
-clickintext.net
-clickzzs.nl
-clips.ua.ac.be
-clknsee.com
-clksite.com
-clmforexeu.com
-codysbbq.com
-compliance-andrew.top
-compliance-barak.xyz
-compliance-don.top
-compliance-elena.xyz
-compliance-mary.top
-coolingoods.com
+cialisndbrx.com
+citetick.com
+cityadspix.com
+cleanpcnow.com
+clubfashionista.com
+coderstate.com
+compliance-brian.xyz
+compliance-jack.top
+compliance-olga.top
+compliance-viktor.top
+com-supportcenter.website
+coolwallpapers-hd.com
 cosmetic.donna7753191.ru
-counterbot.com
-creams.makeforum.eu
-cymbaltaandtizanidine.blogspot.com
-dawlenie.com
-daydream-studio.ru
-day-news.info
-deanmoore.ie
-dekoration.us
-demian.kz
-destinationrealestate.com
-dev-seo.blog
-diesel-parts28.ru
-dieswaene.com
-dimaka.info
-directivepub.com
-discovertreasurenow.com
-djekxa.ru
-dms-sw.ru
-dostavka-v-ukrainu.ru
-driving.kiev.ua
-duplicashapp.com
-dvd-famille.com
-dvervmoskvu.ru
-easycommerce.cf
-ebonyporn.site
-e-commerce-seo1.com
-ecommerce-seo.com
-ecommerce-seo.org
-ecookna.com.ua
-educatemetv.com
-election.interferencer.ru
-elektronischezigarette2.mex.tl
-elektrozigaretten2.yn.lt
-elstal.com.pl
-empis.magix.net
-enginebay.ru
-envaseslotusama.com
-ereko.ru
-eropho.com
-eshop4u.jp
-etotupo.ru
-expertnaya-ocenka.ru
-expresstoplivo.ru
-ezigarettekaufen2.mpbloggar.se
+countercrazy.com
+craftburg.ru
+crd.clan.su
+creativehutindia.com
+customcatchcan.com
+dafatiri.com
+danschawbel.com
+daretodonate.co
+decorationspcs.com
+digilander.libero.it
+dipstar.org
+dirtpics.pw
+domaincheck.io
+domain-submit.info
+dorratex.tn
+dostavka-v-krym.com
+doyouknowtheword-flummox.ml
+dprtb.com
+draniki.org
+drev.biz
+drpornogratisx.xxx
+dscaas.website
+dvr.biz.ua
+dvrlists.com
+ebooknovel.club
+efkt.jp
+efnor-ac.com
+electro-prom.com
+elektrischezigarette2.onsugar.com
+elektronischezigarettekaufen2.tumblr.com
+enge-fotzen.info
+englate.com
+epngo.bz
+eredijovon.com
+erotik-kostenlos.net
+ertelecom.ru
+espaceinventoristes.com
+essay-writing.work
+europages.com.ru
+euroskat.ru
+extrabot.com
+extremal-blog.com
+extstat.com
 ezigaretteshop2.vefblog.net
-fable.in.ua
-factorynightclub.com
-fbdownloader.com
-fellowshipoftheminds.com
-fenoyl.batcave.net
-ferretsoft.com
-figuringmoneyout.com
-filmi-onlain.info
-find1friend.com
-finemanteam.com
-finstroy.net
-flavors.me
+ezigaretteshop.webs.com
+f5mtrack.com
+fashionindeed.ml
+fashion-mk.net
+favornews.com
+fikasound.tk
+financepoints.eu
+findacheaplawyers.com
+findsexguide.com
+fireads.men
+flagcounter.me
+flashahead.info
+flex4launch.ru
+f-loaded.de
+fodelsedagspresenter.nu
 food.dtu.dk
-footballfarrago.com
-forex.osobye.ru
-forsex.info
-fortevidyoze.net
-fotosfotos.eu
-frameimage.org
-franch.info
-freecamdollars.com
-free-share-buttons.top
-freesitetest.com
+forexunion.net
+forum69.info
+foto-basa.com
+foxinsocks.ru
+foxweber.com
+franklinfire.co
+free-fbook-traffic.com
+free-traffic.xyz
 freshberry.com.ua
-freshdz.com
-fullfileaccess.com
-funnel.co.za
-gateway.zscalerone.net
-generalporn.org
-getlamborghini.ga
-getmiro.com
-getprismatic.com
-getridofstretchmarks.org
-girlsatgames.ru
-gkvector.ru
-glall.ru
-globetrotting-culture.ru
-goodhousekeeping.com
-groupmoney.ru
+frivgame250.com
+fr.netlog.com
+fullgirl.ru
+fun2cell.net
+funcrushgames.com
+fun-mobi.pl
+game300.ru
+gamedayhouse.com
+gay.adultgalls.com
+getdot.ru
+getpopunder.com
+getyourimage.club
+get-your-social-buttons.info
+ghazel.ru
+gne8.com
+goforexvps.com
+golden-praga.ru
+golmau.host.sk
+googst2.ru
+goosefishpost.bid
+gourcy.altervista.org
+graphics8.info
 growboxbl.ru
-gsasearchenginerankerhelp.com
-gsasearchengineranker.top
-guod.me
-gwagka.com
-hamptonoaks.ca
-haveinc.xyz
-heygidday.biz
-histock.info
-hit-kino.ru
-hitmuzik.ru
-hobild.net
-homegardenlova.com
-hop.clickbank.net
-hornymatches.com
-hoverboard360.de
-hoverboard360.es
-hoztorg-opt.ru
-humanorightswatch.org
-hundejo.com
-idealtits.net
-ido3.com
-i-hobot.ru
-impisr.edunsk.ru
-impresagaia.it
-inmate-locator.us
-integritylandscapeservices.com
-int.search.tb.ask.com
-io9.com
-iqoption-bin.com
-iqoption.pro
-irkutsk.online-podarki.com
-itsdp3.com
-javspace.net
-jbl-charge.info
-jerseychinabizwholesale.com
-jerseyswholesalechinalimited.com
-job.icivil.ir
-justkillingti.me
-kabbalah-red-bracelets.com
-kazka.ru
-kemerovo.zrus.org
-keywesthideaways.co
-kidskunst.info
-kiskinhouse.com
+grupografico-pilar.com.ar
+guardlink.org
+gwhwpxbw.bloger.index.hr
+gz2.bbsoldes.fr
+haikuware.com
+handicapvansarea.com
+hanwei.us
+hazardky.net
+hdhc.ru
+hem.passagen.se
+hidemyass.com
+high-speed1.net
+hikesearch.net
+hildinghr.se
+hit-men.men
+hobbyhuren-datenbank.com
+hotchatdate.com
+hothot.ru
+houdom.net
+housediz.com
+housekuba.org
+hreade.com
+huhn.altervista.org
+humanelydrew.com
+imicrovision.com
+iminent.com
+instabid.tech
+intervsem.ru
+invitefashion.com
+inzn.ru
+i-service.kz
+iskalko.ru
+itrevolution.cf
+iusstf.org
+jabimgo.pw
+jackwolfskinoutlet.online
+janerikholst.se
+javatex.co.id
+jerseywholesalechinabiz.com
+jolic2.com
+jrcigars.com
+kalb.ru
+kaliningrad.zrus.org
+kanimage.com
+kashubadesign.ru
+keywordsdoctor.com
+kichenaid.ru
+kihi.gdn
+kiprinform.com
+klejonka.info
 kmd-pto.ru
-komputernaya-pomosh-moscow.ru
-komputers-best.ru
-konkursov.net
-labelwater.se
-lacave.ntic.fr
-lakomka.com.ua
-lankarns.com
-lapitec.eu
-law-check-eight.xyz
-law-check-seven.xyz
-liffa.ru
-lignofix.ua
-lion.misba.us
-lion.snapmap.us
-littleberry.ru
-ljusihus.se
-loadingpages.me
-localflirtbuddies.com
-locatellicorretor.com.br
+kongruan.com
+kostenloser-sex.com
+kung-fu-ru.com
+kupiproday.com.ua
+kvartir-remont.biz
+lacapilla.info
+ladsblue.com
+laxob.com
+legalrc.biz
+letolove.ru
+levaquin750mg.blogspot.com
+lifebyleese.com
+liky.co.ua
+lincolntheatre.com
+liupis.com
+llastbuy.ru
+localmatchbook.com
 locimge.pw
-longgreen.info
-lottospring.com
-lunamedia.co
-magazintiande.ru
-magicdiet.gq
-mail.allnews24.in
-mantramusic.ru
-maximpartnerspr.com
-maxxtor.eu
-mbiologi.ru
+lollypopgaming.com
+lol-smurfs.com
+love-baby.cz
+maslenka.kz
+max-eclat.men
+medispainstitute.com.au
 megakino.net
-members.ghanaweb.com
-menhealed.net
-mexicosleevegastrectomy.com
-minecraft-neo.ru
-minegam.com
-mir-betting.ru
-mir-business-24.ru
-mmostrike.ru
-mnogolok.info
-mobile-appster.ru
-moesonce.com
-monetizationking.net
-moyaterapiya.ru
-mrbojikobi4.biz
-mtmtv.info
-mug-na-chas-moscow.ru
-muschisexbilder.com
-musicdaddy.net
-myblogregistercm.tk
-myfreemp3.eu
-mygameplus.com
-mygameplus.ru
-myonigroup.com
-nalogovyykodeks.ru
-naphukete.ru
-narosty.com
-national-today-winning-winner.club
-netcheckcdn.xyz
-networkcheck.xyz
-new-apps.ru
-newhairstylesformen2014.com
-news-readers.ru
-nippon-bearings.ru
-novodigs.com
-oakridgemo.com
+metabar.ru
+microsoftportal.net
+miniads.ca
+mobile.ok.ru
+mobplayer.ru
+mojaocena.com
+mokrayakiska.com
+moneyteam24.com
+monsterdivx.tv
+moonci.ru
+morefastermac.trade
+mowser.com
+muabancantho.info
+mydoctorok.ru
+mydownlodablefiles.com
+mysex21.com
+mystats.xyz
+myyour.eu
+nagdak.ru
+najaden.no
+naj-filmy24.pl
+nakozhe.com
+napalm51.nut.cc
+nardulan.com
+narkologiya-belgorod.ru
+naturalbreakthroughsresearch.com
+naturalshair.site
+needtosellmyhousefast.com
+new7ob.com
+nlfjjunb5.ru
+notaria-desalas.com
+notfastfood.ru
+nottyu.xyz
+nullrefer.com
+oballergiya.ru
+obiavo.by
+obiavo.com
+obiavo.ru
+oddamzadarmo.eu
 office2web.com
-oneminutesite.it
-onlineku.com
-onlineporno.site
-online-sbank.ru
-optikremont.ru
+ok-ua.info
+olvanto.ru
+onclkads.com
+onlyporno.ru
+o-o-11-o-o.com
+o-o-8-o-o.ru
+openstat.com
+optitrade24.com
 ororodnik.goodbb.ru
 ostrovtaxi.ru
-outshop.ru
-pandroid.co
-paphoselectricianandplumber.com
-paramountmarble.co.uk
-partnerworkroom.men
-percin.biz.ly
-photo-clip.ru
-phuketscreen.com
-picsearch.com
-pihl.se
-piuminiita.com
-plastjulgranar.se
-podshipniki-ntn.ru
-pogodnyyeavarii.gq
-pokemongooo.ml
-polimga.pw
-pomoc-drogowa.cba.pl
-popunder.net
-popup-jdh.xyz
-pornobrazzers.biz
+oynat.info
+oz-offers.com
+parajumpersjakkesalgnorge.info
+par-fallen.ga
+parfusale.se
+pcgroup.com.uy
+pc-test.net
+pelfind.me
+picquery.com
+pinsdaddy.com
+pix-hd.com
+plastweb.ru
+play.leadzupc.com
+pobeiranie.pl
+polcin.de
+porno-chaman.info
 porno-play.net
-pornoplen.com
-pornotubs.com
-portal-eu.ru
-pozdravleniya-c.ru
-priceg.com
-prod2016.com
-proekt-gaz.ru
-profitwithalex.info
-progress-upakovka.ru
-prohoster.info
-promalp-universal.ru
-prombudpostach.com.ua
-prostitutki-belgoroda.org
-pro-tec.kz
-pr-ten.de
-pussysaga.com
-radiodigital.co
-rankings-analytics.com
-realting-moscow.ru
-remontbiz.ru
-remont-comp-pomosh.ru
-renecaovillasale.online
+pornoreino.com
+porno-transsexuals.ru
+portside.cc
+poshiv-chehol.ru
+prefersurvey.net
+primedice.com
+privatefx-in.ru
+prosmibank.ru
+prpops.com
+psvita.ru
+qitt.ru
+raavidesigns.com
+rapevideosmovies.com
+razvratnoe.org
+rcb101.ru
+realnye-otzyvy.info
+recipedays.ru
+redirect.trafficreceiver.club
+referencemoi.com
+reining.lovasszovetseg.hu
+renecaovilla.online
+rep-am.com
+replicalouboutin.xyz
+responsive-test.net
 ring4rhino.com
-ritlweb.com
-rixpix.ru
 rockingclicks.com
-roleforum.ru
-royalads.net
-royalcar-ufa.ru
-rus-teh.narod.ru
-sakhboard.ru
-saneitconsulting.com
-score-ads.men
+rootandroid.org
+rostov.xrus.org
+rusoft-zone.ru
+russia-today-video.ru
+sabizonline.com
+samo-soznanie.ru
+sanatorrii.ru
+sandhillsonline.com
+sarm.tk
+sbprabooks.com
+scanner-andrew.top
+scanner-jack.top
+scanner-julianna.top
+scanner-nelson.top
+scanner-olga.top
+schoolfiles.net
+scopich.com
 scrnet.biz.ua
-sc-specialhost.com
-sell-fb-group-here.com
-seoboxes.com
-seorankinglinks.xyz
+sdelai-prosto.ru
+search-goo.com
+searchkut.com
+seo-2-0.com
+seomarketings.online
+serpstat.com
+servingnotice.com
+sexreliz.com
 sextracker.be
-sexvporno.ru
-sex-znakomstva.online
-shama-rc.net
-shymkent.xkaz.org
-sideeffectsoftizanidine.blogspot.com
-sierraapps.com
-simple-image.com.ua
-sindragosa.comxa.com
-site3.free-share-buttons.com
-sitopreferito.it
-s-iwantyou.com
-skatestick.bid
-slutloadlive.com
-small-games.biz
-smartbalanceworld.com
-social-button.xyz
-soc-proof.su
-soft-terminal.ru
-softwaretrend.net
-solnplast.ru
-sosdepotdebilan.com
-sovetogorod.ru
-spin2016.cf
-spy-app.info
-spymac.net
-spywarebegone.com
-stairliftsarea.com
-start.myplaycity.com
-stathat.com
-steame.ru
-stoki.ru
-stpolice.com
-stretchmate.net
-students-cheapskate.ml
+sexyebonyteen.com
+shakhtar-doneck.ru
+shell-pmr.ru
+shop.xz618.com
+sientalyric.co
+simplepooltips.com
+sirpornogratis.xxx
+sitevalued.com
+sklad-24.ru
+small-game.com
+sneakyboy.com
+social-buttons.xyz
+soft1.ru
+softlinesolutions.me
+songoo.wz.cz
+spaceshipad.com
+spb.ru
+spy-sts.com
+stanthonyscatholicchurch.org
+startwp.org
+stats-collector.org
+streetfire.net
+stroydetali.ru
+stroymonolit.su
+studiofmp.com
 studiokamyk.com.pl
-studworks.org
-supermodni.com.ua
-surffoundation.nl
-taaaak.com
+superinterstitial.com
+svnuppsalaorebro.se
 tarad.com
-taraz.xkaz.org
+taxi-v-eisk.ru
 tds-advert005.info
+tedy.su
 teenporn18.net
+template-kid.com
+terraclicks.com
 tfxiq.com
-thaismartloan.com
-thegamerznetwork.com
-thejournal.ru
-ticketsys.inetwd.com
-titangel-vietnam.com
-tizanidineandhydrocodone.blogspot.com
-tizanidinecapsules.blogspot.com
-tizanidinedruginteractions.blogspot.com
-tizanidinehcl4mgisitanarcotic.blogspot.com
-tizanidinezanaflex.blogspot.com
-tnaionline.org
-tootoo.to
-topcar-krasnodar.ru
-topclickguru.com
-torrentgamer.net
-track2.shop
-trafficmonetize.org
-tramadolandtizanidine.blogspot.com
-trudogolik.net
-uamtrk.com
-ukkala.xyz
+thaisamkok.com
+thenews-today.info
+thepantonpractice.co.uk
+thetattoohut.com
+thetoiletpaper.com
+time-japan.ru
+titan-ads.life
+tizanidineonline.blogspot.com
+toloka.hurtom.com
+toolsky.com
+tooplay.com
+topvidos.ru
+touchmods.fr
+trafficreceiver.club
+trichizobswiv.agddns.net
+tubeline.biz
+tupper-posuda.ru
+tvorozhnaja-zapekanka-recept.ru
+ukr-lit.com
+ultimateclassicrock.com
 umaseh.com
-umekana.ru
-um-razum.ru
-unitygame3d.com
-upstore.me
-urccvfmc.bloger.index.hr
-urzedowski.eu
-usadacha.net
-utrolive.ru
-videogamesecrets.com
+undergroundcityphoto.com
+uni.me
+unrealcommander.org
+usacasino.com
+valuado.com
+valueclick.com
+veerotech.com
+viandpet.com
+video-hollywood.ru
+vinylvault.co.uk
 vip-dom.in
-vip-file.com
-vipromoffers.com
-vitoriacabos.com
-vodkoved.ru
-voloslove.ru
-vonradio.com
-vriel.batcave.net
-watchinf.com
-watchmyfb.pl
-watracker.net
-weboptimizes.com
-website-analyzer.info
-website-datenbank.de
-westum.se
-wfb.hatedriveapart.com
-whatistizanidine2mg.blogspot.com
+vremya.eu
+vseigry.fun
+wait3sec.org
+wholesalejerseysgaa.com
 williamrobsonproperty.com
-wjgony.com
-womensplay.net
-wosik-dach.service-for-web.de
-wpthemedetector.co.uk
-www.wohnkabinen-shop.de
+www.pinnacle-bets.com
 wzgyyq.com
-x5market.ru
-x-diesel.info
-x-diesel.org
-xitjw.info
-xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai
-xn----7sbbagbq7bd5aheftfllo4m.xn--p1ai
-xn----7sbbahaq9bb5afgiqfliv4m.xn--p1ai
+xjrul.com
+x-lime.net
+xn--24-glceagatoq7c2a6ioc.xn--p1ai
+xn--80abgj3a5acid6ghs.top
 xn--80ajjbdhgmudixfjc8c5a9df8b.xn--p1ai
-xn--c1acygb.xn--p1ai
-xn--d1abj0abs9d.in.ua
-xxxkaz.org
-xxx-treker.ru
-yellocloud.be
-yougotanewdomain.com
-youporn-forum.ga
-youporn-ru.com
-yourmovies.pl
-yubikk.info
-yuweng.info
-zeroredirect12.com
-zkjovpdgxivg.ga
-zlatnajesen.com
-zolotoy-lis.ru
-zooggames.com
+xn----8sbhefaln6acifdaon5c6f4axh.xn--p1ai
+xnxx699.com
+xolodremont.ru
+xxxhdvideo.site
+xz618.com
+yellowads.men
+yougame.biz
+your-bearings.com
+yuarra.pluto.ro
+zaloadi.ru
+zambini.ru
+zapiszto.pl
+zastroyka.org
+zebradudka.com
+zeroredirect.com
+zoompegs.com
 zscalertwo.net
-zvooq.eu
-zx6.ru
-zzbroya.com.ua
+zvetki.ru

.dev-tools/test_units/random-referrers-for-whitelist-test.list

@@ -1,250 +1,250 @@
-0lovespells0.blogspot.com
-1xbet4you.com
-2000k.ru
-24-ak.ru
-3-letter-domains.net
-51unlim.ru
-5elementov.ru
-abcdeg.xyz
-activepr.ru
-adanih.com
-adcash.com
-adminshop.com
-adpremium.org
-adsfresh.men
-ads-seo.men
-adtech.fr
-adverster.com
-alborzan.com
-all4invest.info
-all-number.com
-animali.deagostinipassion.it
-anjalika.co.in
-aosexkontakte.net
-applyneedy.xyz
-arabseyes.com
-arenda-avtoprokat-krasnodar.ru
-artdeko.info
-asmxsatadriverin.aircus.com
-autoplate.info
-autotop.com.ua
+2daytrendingnews.com
+300richmond.co.nz
+45en.ru
+5ws.dating-app.ru
+abovetherivernc.com
+acheterviagrafr24.com
+ads-cool.pro
+adsref.men
+advancedcleaner.com
+alert-hgd.xyz
+animenime.ru
+appfixing.space
+apple.com-webbrowsing-security.science
+applicationg29.com
+arabsexxxtube.com
+artparquet.ru
+asophoto.com
 autotrends.today
-aviav.co
-azarttoto.com
-babyguns.ru
-bamo.xsl.pt
-bazaronline24.ru
-bensbargains.net
-blavia.00author.com
-bloggers.nl
-bnt-team.com
-bookmaker-bet.com
-brainsandeggs.blogspot.com
-brainzpod.win
-bytimedance.ru
-cacheimages.com
-callstevens.com
-camdolls.com
-casablancamanor.co.za
-casite-513049.cloudaccess.net
-cbcseward.com
-chcu.net
+aviav.ru
+b2b-lounge.com
+babs.com.ua
+bad-stars.net
+baersaratov.ru
+banksrf.ru
+bannerads.de
+bdsmgalls.net
+betwinservice.com
+bildsuche.ru
+billiard-classic.com.ua
+binomo.com
+blockworld.ru
+bloggerads.net
+blogs.rediff.com
+bonky.biz
+brateg.xyz
+bravegirlsclub.com
+californianews.cf
+calvet.altervista.org
+canacopegdl.com
+candypeople.se
+cbozhe.com
+check-host.net
+cialgenisrx.com
+ciceros.co
 ciproandtizanidine.blogspot.com
-cleanmypc.com
+citetick.com
+clean-start.net
 clipartnew.com
-com-cleaner.systems
-compliance-alexa.top
-compliance-barak.xyz
-compliance-viktor.top
-coop-gamers.ru
-copenergo.ru
-copypaste.traderzplanet.in
-creativehutindia.com
-csgo4.win
-dairyindia.in
-dating-app.ru
-detmebel.su
-dfwu1019.info
-divatraffic.com
+cognitiveseo.com
+coinsspb.com
+compliance-ivan.xyz
+compliance-jack.top
+cornomase.win
+cottageofgrace.com
+cvety24.by
+damedingel.ya.ru
+daptravel.com
+dealighted.com
+den-noch24.ru
+derevesendeco.com
+discovertreasure-a.akamaihd.net
+disruptingdinnerparties.com
+dkvorota.ru
 domainsatcost.com
-domain.webkeyit.com
-dominterior.org
-domznaniy.ru
-downtuptv.gq
-dscaas.website
-dverimegapolis.ru
-ecxtracking.com
-elexies.info
-emerson-rus.ru
-escortplius.com
-essaypro.com
-eu-cookie-law.blogspot.com
-f00kclan.de
-figuringmoneyout.com
-filkhbr.com
-forexunion.net
-frameimage.org
-frbizlist.com
-free-deals.faith
-friendflnder.com
-funnymama.com
-gavuer.ru
-glasof.es
-gok-kasten.net
-googglet.com
-gov.yanao.ru
-graphicwe.org
-gratis-sexkontakte.com
-groupmoney.ru
-gsbs.com.ua
-guardlink.com
-guitar-master.org
-gwebtools.com.br
-highland-homes.com
-highstairs-a.akamaihd.net
-host-tracker.com
-idealtits.net
-imagui.eu
-irkutsk.online-podarki.com
-irunfar.com
-itis4you.com
-itsdp3.com
-iwantmyfreecash.com
-jabimgo.pw
-jetsli.de
-juliadiets.com
-justucalling32211123456789.tk
-kakadu-interior.com.ua
-keywordblocks.com
-kirov.zrus.org
-kollesa.ru
-lashstudia.ru
-linkpulse.com
-livejasmin.com
-lyngdalhudterapi.no
-mactechinfo.info
-magicplayer-s.acestream.net
-maglid.ru
-mainlinehobby.net
-marcowebonyodziezowe.pl
-matchpal-a.akamaihd.net
-mediaoffers.click
+dustyorate.com
+ecommerce-seo.org
+econom.co
+ekaterinburg.xrus.org
+elektrischezigaretten1.blogse.nl
+eorogo.top
+erectile.bid
+espaceinventoristes.com
+evaashop.ru
+evening-dating-club.info
+eywords-monitoring-your-success.com
+ezigarettenshop1.yolasite.com
+factorynightclub.com
+favoritemoney.ru
+fetroshok.ru
+filefilter.weebly.com
+filmgo.ru
+flashbannernow.com
+fm-upgrade.ru
+fordsonmajbor.cf
+fotopop.club
+fotoxxxru.com
+free-share-buttons.top
+fres-news.com
+fuck-paid-share-buttons.xyz
+garciniaxt.us
+generousdeal-a.akamaihd.net
+getprismatic.com
+get-seo-domain.com
+good-mummy.ru
+gotorussia.com
+gsou.cf
+hccoder.info
+helvetia.com.ua
+hop.clickbank.net
+hulfingtonpost.com
+iboss.com
+ifmo.ru
+images-free.net
+infobanks.ru
+inspiring-desperate.tk
+int.search.mywebsearch.com
+io9.com
+iplusbit.blogspot.co.za
+iqoption.com
+i-service.kz
+ivanovo.zrus.org
+ivearchenceinflu.cf
+jackwolfskinoutlet.online
+jerseysbizwholesalecheap.com
+jumptap.com
+justprofit.xyz
+kashubadesign.ru
+katadhin.co
+keywordhouse.com
+keywordsdoctor.com
+keyword-suggestions.com
+kicknights.gq
+klumba55.ru
+komputers-best.ru
+koopilka.com
+kotaku.com
+koversite.info
+krasivoe-hd.com
+kw21.org
+lacave.ntic.fr
+lafriore.ru
+lakomka.com.ua
+leftofcentrist.com
+lignofix.ua
+linkdebrideur.xyz
+linkrr.com
+ltvperf.com
+m4ever.net
+magento-crew.net
+massage-info.nl
+mebel-alait.ru
+mebeldekor.com.ua
+medi-fitt.hu
+mere.host.sk
+mexicosleevegastrectomy.com
 misswell.net
+mixx.com
 mobilierland.com
-moivestiy.biz
-monarhs.info
-morf.snn.gr
-msfsaar.de
-muz-tracker.net
-naphukete.ru
-narutonaruto.ru
-nastydollars.com
-notfastfood.ru
-nuup.info
-nw-servis.ru
-obiavo.in
-offf.info
-office2web.com
-office-windows.ru
-of-ireland.info
-onlinefilmz.net
-o-o-8-o-o.com
-ops.picscout.com
-oranga.host.sk
-orgasmatrix.com
-osb.se11.ru
-picmoonco.pw
-pivka.xyz
-play-mp3.com
-pornogad.com
-pornoplen.com
-porn-w.org
-pozdrawleniya.ru
-privat-girl.net
-pronekut.com
-purplestats.com
-pyramidlitho.webs.com
-rabotaetvse.ru
-radiogambling.com
-rainbowice.ru
-ranksonic.com
-ranksonic.info
-rent2spb.ru
-rixpix.ru
-runofilms.ru
-saddiechoua.com
-sarafangel.ru
-satellite.maps.ilovevitaly.com
-sbwealthsolutions.ca
-securesmrt-dt.com
-seeingmeerkat.com
-seeresultshub-a.akamaihd.net
-seksotur.ru
-seoheap.com
-seo-tools-optimizing.com
-sexstream.pl
-sexyteens.hol.es
-sharebutton.net
-shell-pmr.ru
-sherlock.se
-shijian.ac.cn
-s-iwantyou.com
-socialmadesimple.com
-soft1.ru
-spb.ru
-specialfinanceoffers.com
-speedup-my.site
-spywarebegone.com
-stroimajor.ru
-sugarlyflex.pw
-surflinksmedical.com
-taqywu51.soup.io
-teguh.info
-telesvoboda.ru
-timdreby.com
-tizanidinehcl2mgsideeffects.blogspot.com
-tizanidinehcl2mgtablet.blogspot.com
-tizanidinehcl4mgtablet.blogspot.com
-top-deal.com.pl
-tracksurf.daooda.com
-traffic-club.info
-tubeline.biz
-tvand.ru
-umityangin.net
-uytmaster.ru
-vancleefreplica.pw
-v-doc.co
+moesexy.com
+mountainstream.ms
+multgo.ru
+myhealthcare.com
+myhitmp3.club
+nailsimg.com
+negral.pluto.ro
+nerudlogistik.ru
+netanalytics.xyz
+onlineku.com
+online-sbank.ru
+onlineslotmaschine.com
+openfrost.net
+oz-offers.com
+panchro.co.uk
+pc-virus-d0l92j2.pw
+pekori.to
+pinwallpaper.xyz
+pistonclasico.com
+piter.xrus.org
+pixell.club
+pladform.ru
+pogodnyyeavarii.gq
+pornogig.com
+prchecker.info
+printdirectforless.com
+proprostatit.com
+quickchange.cc
+ragecash.com
+realmonte.net
+r-e-f-e-r-e-r.com
+relax.ru
+remont-mobile-phones.ru
+resant.ru
+responsive-test.net
+risparmiocasa.bz.it
+rosbalt.com.ua
+rusenvironmental.net
+rusexy.xyz
+samolet.fr
+savefrom.com
+scansafe.net
+searchaddis.com
+search.pch.com
+secret.xn--oogle-wmc.com
+seoexperimenty.ru
+seofied.com
+seoprofiler.com
+shoplvlv.us
+shopwme.ru
+sierraapps.com
+skylta.com
+slomm.ru
+smichovbike.cz
+snow.nvr163.com
+soc-econom-problems.ru
+spidtest.space
+sugvant.ru
+supermama.top
+swinginwithme.ru
+tanieaukcje.com.pl
+taranerymagesswa.blogspot.com
+texbaza.by
+thedownloadfreeonlinegames.blogspot.com
+thedownloadfromwarez.blogspot.com
+thefotosgratis.eu
+timecrimea.ru
+titan-cloud.life
+topclickguru.com
+trafficstars.com
+turizmus.us
+twinderbella.com
+u-cheats.ru
+uchetunet.su
+ukrtextbook.com
+ultramart.biz
+underthesite.com
+veopornogratis.xxx
+vereo.eu
 vesnatehno.ru
-viagroid.ru
-viel.su
-vltai.com
-vonradio.com
-vpnhowto.info
-vremya.eu
-vykupavto-krasnodar.ru
+villacoloniale.com
+visa-china.ru
+waterpurifier.club
+watracker.net
+we-are-gamers.com
 web-betting.ru
-weboptimizes.com
-websitevaluebot.com
-whatsappdownload10.com
-wholesalejerseychinaoutlet.com
-whos.amung.us
-wineitudes.wordpress.com
-wowcasinoonline.ooo
-woweb.com.ua
-wrona.it
-ws.ampower.me
-wttavern.com
-xn--24-glceagatoq7c2a6ioc.xn--p1ai
-xn--80aanaardaperhcem4a6i.com
-xn--80aikhbrhr.net
-xn-----8kcatubaocd1bneepefojs1h2e.xn--p1ai
-xn--h1ahbi.com.ua
-xn----itbeirbjbi7bc6bh2d.xn--p1ai
-x-porno.video
-y8games-free.com
-youporn-forum.ga
-youtube-downloader.savetubevideo.com
-zaim-pod-zalog-krasnodar.ru
-zarabiaj-dzis.pl
-zigarettenonl.canalblog.com
-zooggames.com
+website-audit.com.ua
+website-speed-check.site
+wildworld.site
+workle.website
+xkaz.org
+x-lime.com
+xn-----7kcabaipgeakzcss7bjdqdwpfnhv.xn--p1ai
+x-stars.ru
+yousense.info
+ytmnd.com
+zaimite.ru
+zakonobosago.ru
+zazagames.org
+zeroredirect8.com
+zrelaya.pw
+zzbroya.com.ua

.dev-tools/test_units/ratelimittest-beta.txt

@@ -581,4 +581,14 @@
 <h1>Welcome to the NginX Ultimate Bad Bot Blocker</h1>
 <p>This is merely used for testing !!!</p>
 </body>
+</html><!DOCTYPE html>
+<html>
+<head>
+<title>Welcome to NginX</title>
+</head>
+<body>
+<div align=center>
+<h1>Welcome to the NginX Ultimate Bad Bot Blocker</h1>
+<p>This is merely used for testing !!!</p>
+</body>
 </html>

.dev-tools/test_units/whitelist-ips.conf

@@ -1,2 +1,2 @@
-104.154.182.187	0;
+34.122.208.80	0;
 127.0.0.1	0;

AUTO-CONFIGURATION.md

@@ -4,9 +4,9 @@
 ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation:
 
 _______________
-#### Version: V4.2021.02.2265
-#### Bad Referrers Blocked: [6928](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [594](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Version: V4.2021.02.2266
+#### Bad Referrers Blocked: [6929](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
+#### Bad User-Agents (Bots) Blocked: [600](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
 #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
 ____________________
 

MANUAL-CONFIGURATION.md

@@ -4,9 +4,9 @@
 ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation:
 
 _______________
-#### Version: V4.2021.02.2265
-#### Bad Referrers Blocked: [6928](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [594](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Version: V4.2021.02.2266
+#### Bad Referrers Blocked: [6929](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
+#### Bad User-Agents (Bots) Blocked: [600](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
 #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
 ____________________
 

README.md

@@ -11,9 +11,9 @@
 ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots)
 
 _______________
-#### Version: V4.2021.02.2265
-#### Bad Referrers Blocked: [6928](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [594](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Version: V4.2021.02.2266
+#### Bad Referrers Blocked: [6929](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
+#### Bad User-Agents (Bots) Blocked: [600](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
 #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
 ____________________
 

_generator_lists/bad-referrers.list

@@ -1,8 +1,3 @@
-vids18.site
-analnoeporno.tv
-1008.su
-dojki365.online
-sex-znakomstva.online
 000free.us
 007angels.com
 00author.com
@@ -421,6 +416,7 @@ anabolics.shop
 anaksma.info
 anal-acrobats.com
 anal-acrobats.hol.es
+analnoeporno.tv
 analytics-ads.xyz
 ananumous.ru
 anapa-inns.ru

_generator_lists/bad-user-agents.list

@@ -1,10 +1,3 @@
-MTRobot
-node-fetch
-scalaj-http
-serpstatbot
-PageThing.com
-Nuclei
-Fuzz
 360Spider
 404checker
 404enemy
@@ -170,6 +163,7 @@ Flunky
 Foobot
 Freeuploader
 FrontPage
+Fuzz
 FyberSpider
 Fyrebot
 GalaxyBot
@@ -312,6 +306,7 @@ Mr.4x3
 MSFrontPage
 MSIECrawler
 Msrabot
+MTRobot
 muhstik-scan
 Musobot
 Name\ Intelligence
@@ -339,7 +334,9 @@ NimbleCrawler
 Nimbostratus
 Ninja
 Nmap
+node-fetch
 NPbot
+Nuclei
 Nutch
 oBot
 Octopus
@@ -360,6 +357,7 @@ Page\ Analyzer
 PageGrabber
 page\ scorer
 PageScorer
+PageThing.com
 Pandalytics
 Panscient
 Papa\ Foto
@@ -411,6 +409,7 @@ s1z.ru
 SalesIntelligent
 satoristudio.net
 SBIder
+scalaj-http
 ScanAlert
 Scanbot
 scan.lol

_google_analytics_ghost_spam/README.md

@@ -3,9 +3,9 @@
 # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM
 
 _______________
-#### Version: V4.2021.02.2265
-#### Bad Referrers Blocked: [6928](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [594](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Version: V4.2021.02.2266
+#### Bad Referrers Blocked: [6929](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
+#### Bad User-Agents (Bots) Blocked: [600](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
 #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
 ____________________
 

_google_analytics_ghost_spam/google-exclude-05.txt

@@ -1 +1 @@
-xn\-\-80aagddcgkbcqbad7amllnejg6dya\.xn\-\-p1ai|xn\-\-80aanaardaperhcem4a6i\.com|xn\-\-80ab4aa2g\.xn\-\-p1ai|xn\-\-80abgj3a5acid6ghs\.top|xn\-\-80adaggc5bdhlfamsfdij4p7b\.xn\-\-p1ai|xn\-\-80aeahghtf8ac5i\.xn\-\-p1ai|xn\-\-80aebbcbcdemfkhba4byaehoejh8dza3v\.xn\-\-p1ai|xn\-\-80ahdheogk5l\.xn\-\-p1ai|xn\-\-80ahvj9e\.xn\-\-p1ai|xn\-\-80aikhbrhr\.net|xn\-\-80ajbshivpvn2i\.xn\-\-p1ai|xn\-\-80ajjbdhgmudixfjc8c5a9df8b\.xn\-\-p1ai|xn\-\-80ak6aa92e\.com|xn\-\-80aodinpgi\.xn\-\-p1ai|xn\-\-90acenikpebbdd4f6d\.xn\-\-p1ai|xn\-\-b1adccaf1bzj\.xn\-\-p1ai|xn\-\-b1addnj3cah\.xn\-\-p1ai|xn\-\-b1ag5cfn\.xn\-\-p1ai|xn\-\-b1agm2d\.net|xn\-\-c1acygb\.xn\-\-p1ai|xn\-\-d1abj0abs9d\.in\.ua|xn\-\-d1aifoe0a9a\.top|xn\-\-e1afanlbnfckd7c3d\.xn\-\-p1ai|xn\-\-e1aggki3c\.xn\-\-80adxhks|xn\-\-h1aakne2ba\.xn\-\-p1ai|xn\-\-h1ahbi\.com\.ua|xn\-\-hxazdsfy\.blogspot\.com|xn\-\-l1aengat\.xn\-\-p1ai|xn\-\-lifehacer\-1rb\.com|xn\-\-oogle\-wmc\.com|xn\-\-q1a\.xn\-\-b1aube0e\.xn\-\-c1acygb\.xn\-\-p1ai|xnxx\-n\.com|xnxx699\.com|xnxxandxvideos\.com|xolodremont\.ru|xportvusbdriver8i\.snack\.ws|xpresscare\.ru|xrus\.org|xsfetish\.org|xsion\.net|xtraffic\.plus|xtrafficplus\.com|xtremeeagles\.net|xtube\.com|xtubeporno\.net|xuki\.us|xvideosbay\.com|xvideosporn\.biz|xvideospornoru\.com|xwatt\.ru|xxart\.ru|xxlargepop\.com|xxx\-cam\.webcam|xxx\-treker\.ru|xxxasianporn\.net|xxxdatinglocal\.us|xxxguitars\.com|xxxhdvideo\.site|xxxkaz\.org|xxxnatelefon\.ru|xxxrus\.org|xxxsiterips\.xyz|xxxtube69\.com|xxxtubesafari\.com|xz618\.com|xzlive\.com|y8games\-free\.com|yaaknaa\.info|yachts\-cruise\.info|yaderenergy\.ru|yadro\.ru|yaminecraft\.ru|yaoguangdj\.com|yatrk\.xyz|yeartwit\.com|yebocasino\.co\.za|yebocasino\.com|yellocloud\.be|yellowads\.men|yellowfootprints\.com|yellowproxy\.net|yellowstonesafaritours\.com|yellowstonevisitortours\.com|yes\-com\.com|yginekologa\.com|yhit\.press|ynymnwbm\.bloger\.index\.hr|yogamatsexpert\.com|yoluxuryevents\.com|yoopsie\.com|yopoint\.in|yoshkarola\.zrus\.org|yottos\.com|you\-shall\-not\-pass\.is74\.ru|youandcredit\.ru|youbloodyripper\.com|youbrainboost\.asia|youdao\.com|youdesigner\.kz|yougame\.biz|yougetsignal\.com|youghbould\.wordpress\.com|yougotanewdomain\.com|youjizz\.com|youjizz\.vc|youporn\-forum\.ga|youporn\-ru\.com|your\-bearings\.com|youradexchange\.com|yourads\.website|youradulthosting\.com|youraticles\.pl|yourdesires\.ru|youresponsive\.com|yourmovies\.pl|yourothersite\.com|yourporn\.com|yourporngay\.com|yoursearch\.me|yourserverisdown\.com|yoursite\.com|yourtemplatefinder\.com|yousense\.info|youthreaders\.com|youtoner\.it|youtube\-downloader\.savetubevideo\.com|youtubedownload\.org|youtubologia\.it|youtuhe\.com|ypmuseum\.ru|ytmnd\.com|yuarra\.pluto\.ro|yubikk\.info|yugk\.net|yugo\-star\.ru|yun56\.co|yunque\.pluto\.ro|yur\-p\.ru|yurgorod\.ru|yuweng\.info|z\-master\.ru|za\-fun\-offer\.com|za\-music\.mymobiplanet\.com|zaapplesales\.blogspot\.com|zacreditom\.ru|zagadki\.in\.ua|zahvat\.ru|zaidia\.xhost\.ro|zaim\-pod\-zalog\-krasnodar\.ru|zaimhelp\.ru|zaimite\.ru|zajm\-pod\-zalog\-nedvizhimosti\.ru|zajm\-zalog\-krasnodar\.ru|zakazfutbolki\.com|zakazvzloma\.com|zakon\-ob\-obrazovanii\.ru|zakonobosago\.ru|zaloadi\.ru|zaloro\.com|zambini\.ru|zaobao\.com\.sg|zapatosenventa\.info|zapiszto\.pl|zarabiaj\-dzis\.pl|zarabotat\-na\-sajte\.ru|zarabotok\-\-doma\.ru|zarajbuilders\.com|zarenica\.net|zarepta\.com|zastenchivosti\.net|zastroyka\.org|zatjmuzu\.info|zawyna\.ua|zazagames\.org|zdesformula\.ru|zdesoboi\.com|zebradudka\.com|zebramart\.ru|zed21\.net|zeg\-distribution\.com|zeikopay\.com|zeleznobeton\.ru|zero1\.it|zerocash\.msk\.ru|zeroredirect\.com|zeroredirect1\.com|zeroredirect10\.com|zeroredirect11\.com|zeroredirect12\.com|zeroredirect2\.com|zeroredirect5\.com|zeroredirect6\.com|zeroredirect7\.com|zeroredirect8\.com|zeroredirect9\.com|zetgie\.com\.pl|zetmaster\.ru|zhacker\.net|zhongwenlink\.com|zhorapankratov7\.blogspot\.com|zhuravlev\.info|zigarettenonl\.canalblog\.com|zigarettenonlinekaufen\.tumblr\.com|zigarettenonlinekaufen1\.bloog\.pl|zigarettenonlinekaufen1\.blox\.pl|zigarettenonlinekaufen2\.bloog\.pl|zigarettenonlinekaufen2\.drupalgardens\.com|zigzog\.ru|zionstar\.net|zirondelli\.it|zixizop\.net\.ru|zkjovpdgxivg\.ga|zlatnajesen\.com|zmoda\.hostreo\.com|znakom\.sibtest\.ru|znakomstva\-moskva77\.ru|znakomstva\-piter78\.ru|znakomstvaonlain\.ru|znaniyapolza\.ru|znaturaloriginal\.com|zocaparj\.kz|zog\.link|zojirushi\-products\.ru|zolotoy\-lis\.ru|zona\-aqua\.ru|zone\-kev717\.info|zoodrawings\.com|zoogdiesney\.com|zoogdinsney\.com|zoogdisany\.com|zooggames\.com|zoolubimets\.ru|zoominfo\.com|zoomovies\.org|zoompegs\.com|zoosexart\.com|zootoplist\.com|zootravel\.com|zophim\.me|zrelaya\.pw|zreloeporno\.tv|zrizvtrnpale\.tk|zrus\.org|zryydi\.com|zs2vm\.top|zscaler\.net|zscalerone\.net|zscalertwo\.net|zskdla\.site|zverokruh\-shop\.cz|zvetki\.ru|zvezdagedon\.ru|zvooq\.eu|zvuker\.net|zx6\.ru|zygophyceous\.womanstars\.site|zynax\.ua|zytpirwai\.net|zzbroya\.com\.ua|zzlgxh\.com
+xn\-\-80aafb2a\.xn\-\-p1ai|xn\-\-80aagddcgkbcqbad7amllnejg6dya\.xn\-\-p1ai|xn\-\-80aanaardaperhcem4a6i\.com|xn\-\-80ab4aa2g\.xn\-\-p1ai|xn\-\-80abgj3a5acid6ghs\.top|xn\-\-80adaggc5bdhlfamsfdij4p7b\.xn\-\-p1ai|xn\-\-80aeahghtf8ac5i\.xn\-\-p1ai|xn\-\-80aebbcbcdemfkhba4byaehoejh8dza3v\.xn\-\-p1ai|xn\-\-80ahdheogk5l\.xn\-\-p1ai|xn\-\-80ahvj9e\.xn\-\-p1ai|xn\-\-80aikhbrhr\.net|xn\-\-80ajbshivpvn2i\.xn\-\-p1ai|xn\-\-80ajjbdhgmudixfjc8c5a9df8b\.xn\-\-p1ai|xn\-\-80ak6aa92e\.com|xn\-\-80aodinpgi\.xn\-\-p1ai|xn\-\-90acenikpebbdd4f6d\.xn\-\-p1ai|xn\-\-b1adccaf1bzj\.xn\-\-p1ai|xn\-\-b1addnj3cah\.xn\-\-p1ai|xn\-\-b1ag5cfn\.xn\-\-p1ai|xn\-\-b1agm2d\.net|xn\-\-c1acygb\.xn\-\-p1ai|xn\-\-d1abj0abs9d\.in\.ua|xn\-\-d1aifoe0a9a\.top|xn\-\-e1afanlbnfckd7c3d\.xn\-\-p1ai|xn\-\-e1aggki3c\.xn\-\-80adxhks|xn\-\-h1aakne2ba\.xn\-\-p1ai|xn\-\-h1ahbi\.com\.ua|xn\-\-hxazdsfy\.blogspot\.com|xn\-\-l1aengat\.xn\-\-p1ai|xn\-\-lifehacer\-1rb\.com|xn\-\-oogle\-wmc\.com|xn\-\-q1a\.xn\-\-b1aube0e\.xn\-\-c1acygb\.xn\-\-p1ai|xnxx\-n\.com|xnxx699\.com|xnxxandxvideos\.com|xolodremont\.ru|xportvusbdriver8i\.snack\.ws|xpresscare\.ru|xrus\.org|xsfetish\.org|xsion\.net|xtraffic\.plus|xtrafficplus\.com|xtremeeagles\.net|xtube\.com|xtubeporno\.net|xuki\.us|xvideosbay\.com|xvideosporn\.biz|xvideospornoru\.com|xwatt\.ru|xxart\.ru|xxlargepop\.com|xxx\-cam\.webcam|xxx\-treker\.ru|xxxasianporn\.net|xxxdatinglocal\.us|xxxguitars\.com|xxxhdvideo\.site|xxxkaz\.org|xxxnatelefon\.ru|xxxrus\.org|xxxsiterips\.xyz|xxxtube69\.com|xxxtubesafari\.com|xz618\.com|xzlive\.com|y8games\-free\.com|yaaknaa\.info|yachts\-cruise\.info|yaderenergy\.ru|yadro\.ru|yaminecraft\.ru|yaoguangdj\.com|yatrk\.xyz|yeartwit\.com|yebocasino\.co\.za|yebocasino\.com|yellocloud\.be|yellowads\.men|yellowfootprints\.com|yellowproxy\.net|yellowstonesafaritours\.com|yellowstonevisitortours\.com|yes\-com\.com|yginekologa\.com|yhit\.press|ynymnwbm\.bloger\.index\.hr|yogamatsexpert\.com|yoluxuryevents\.com|yoopsie\.com|yopoint\.in|yoshkarola\.zrus\.org|yottos\.com|you\-shall\-not\-pass\.is74\.ru|youandcredit\.ru|youbloodyripper\.com|youbrainboost\.asia|youdao\.com|youdesigner\.kz|yougame\.biz|yougetsignal\.com|youghbould\.wordpress\.com|yougotanewdomain\.com|youjizz\.com|youjizz\.vc|youporn\-forum\.ga|youporn\-ru\.com|your\-bearings\.com|youradexchange\.com|yourads\.website|youradulthosting\.com|youraticles\.pl|yourdesires\.ru|youresponsive\.com|yourmovies\.pl|yourothersite\.com|yourporn\.com|yourporngay\.com|yoursearch\.me|yourserverisdown\.com|yoursite\.com|yourtemplatefinder\.com|yousense\.info|youthreaders\.com|youtoner\.it|youtube\-downloader\.savetubevideo\.com|youtubedownload\.org|youtubologia\.it|youtuhe\.com|ypmuseum\.ru|ytmnd\.com|yuarra\.pluto\.ro|yubikk\.info|yugk\.net|yugo\-star\.ru|yun56\.co|yunque\.pluto\.ro|yur\-p\.ru|yurgorod\.ru|yuweng\.info|z\-master\.ru|za\-fun\-offer\.com|za\-music\.mymobiplanet\.com|zaapplesales\.blogspot\.com|zacreditom\.ru|zagadki\.in\.ua|zahvat\.ru|zaidia\.xhost\.ro|zaim\-pod\-zalog\-krasnodar\.ru|zaimhelp\.ru|zaimite\.ru|zajm\-pod\-zalog\-nedvizhimosti\.ru|zajm\-zalog\-krasnodar\.ru|zakazfutbolki\.com|zakazvzloma\.com|zakon\-ob\-obrazovanii\.ru|zakonobosago\.ru|zaloadi\.ru|zaloro\.com|zambini\.ru|zaobao\.com\.sg|zapatosenventa\.info|zapiszto\.pl|zarabiaj\-dzis\.pl|zarabotat\-na\-sajte\.ru|zarabotok\-\-doma\.ru|zarajbuilders\.com|zarenica\.net|zarepta\.com|zastenchivosti\.net|zastroyka\.org|zatjmuzu\.info|zawyna\.ua|zazagames\.org|zdesformula\.ru|zdesoboi\.com|zebradudka\.com|zebramart\.ru|zed21\.net|zeg\-distribution\.com|zeikopay\.com|zeleznobeton\.ru|zero1\.it|zerocash\.msk\.ru|zeroredirect\.com|zeroredirect1\.com|zeroredirect10\.com|zeroredirect11\.com|zeroredirect12\.com|zeroredirect2\.com|zeroredirect5\.com|zeroredirect6\.com|zeroredirect7\.com|zeroredirect8\.com|zeroredirect9\.com|zetgie\.com\.pl|zetmaster\.ru|zhacker\.net|zhongwenlink\.com|zhorapankratov7\.blogspot\.com|zhuravlev\.info|zigarettenonl\.canalblog\.com|zigarettenonlinekaufen\.tumblr\.com|zigarettenonlinekaufen1\.bloog\.pl|zigarettenonlinekaufen1\.blox\.pl|zigarettenonlinekaufen2\.bloog\.pl|zigarettenonlinekaufen2\.drupalgardens\.com|zigzog\.ru|zionstar\.net|zirondelli\.it|zixizop\.net\.ru|zkjovpdgxivg\.ga|zlatnajesen\.com|zmoda\.hostreo\.com|znakom\.sibtest\.ru|znakomstva\-moskva77\.ru|znakomstva\-piter78\.ru|znakomstvaonlain\.ru|znaniyapolza\.ru|znaturaloriginal\.com|zocaparj\.kz|zog\.link|zojirushi\-products\.ru|zolotoy\-lis\.ru|zona\-aqua\.ru|zone\-kev717\.info|zoodrawings\.com|zoogdiesney\.com|zoogdinsney\.com|zoogdisany\.com|zooggames\.com|zoolubimets\.ru|zoominfo\.com|zoomovies\.org|zoompegs\.com|zoosexart\.com|zootoplist\.com|zootravel\.com|zophim\.me|zrelaya\.pw|zreloeporno\.tv|zrizvtrnpale\.tk|zrus\.org|zryydi\.com|zs2vm\.top|zscaler\.net|zscalerone\.net|zscalertwo\.net|zskdla\.site|zverokruh\-shop\.cz|zvetki\.ru|zvezdagedon\.ru|zvooq\.eu|zvuker\.net|zx6\.ru|zygophyceous\.womanstars\.site|zynax\.ua|zytpirwai\.net|zzbroya\.com\.ua|zzlgxh\.com

_google_webmaster_disavow_links/google-disavow.txt

@@ -416,6 +416,7 @@ domain:anabolics.shop
 domain:anaksma.info
 domain:anal-acrobats.com
 domain:anal-acrobats.hol.es
+domain:analnoeporno.tv
 domain:analytics-ads.xyz
 domain:ananumous.ru
 domain:anapa-inns.ru

robots.txt/robots.txt

@@ -6,9 +6,9 @@
 
 ### Version Information #
 ###################################################
-### Version: V4.2021.02.2265
-### Updated: Wed Feb  3 09:17:55 SAST 2021
-### Bad Bot Count: 594
+### Version: V4.2021.02.2266
+### Updated: Thu Feb  4 10:15:53 SAST 2021
+### Bad Bot Count: 600
 ###################################################
 ### Version Information ##
 
@@ -345,6 +345,8 @@ User-agent: Freeuploader
 Disallow:/
 User-agent: FrontPage
 Disallow:/
+User-agent: Fuzz
+Disallow:/
 User-agent: FyberSpider
 Disallow:/
 User-agent: Fyrebot
@@ -629,6 +631,8 @@ User-agent: MSIECrawler
 Disallow:/
 User-agent: Msrabot
 Disallow:/
+User-agent: MTRobot
+Disallow:/
 User-agent: muhstik-scan
 Disallow:/
 User-agent: Musobot
@@ -683,8 +687,12 @@ User-agent: Ninja
 Disallow:/
 User-agent: Nmap
 Disallow:/
+User-agent: node-fetch
+Disallow:/
 User-agent: NPbot
 Disallow:/
+User-agent: Nuclei
+Disallow:/
 User-agent: Nutch
 Disallow:/
 User-agent: oBot
@@ -725,6 +733,8 @@ User-agent: page scorer
 Disallow:/
 User-agent: PageScorer
 Disallow:/
+User-agent: PageThing.com
+Disallow:/
 User-agent: Pandalytics
 Disallow:/
 User-agent: Panscient
@@ -827,6 +837,8 @@ User-agent: satoristudio.net
 Disallow:/
 User-agent: SBIder
 Disallow:/
+User-agent: scalaj-http
+Disallow:/
 User-agent: ScanAlert
 Disallow:/
 User-agent: Scanbot