V4.2022.01.2867 [ci skip]

AUTO-CONFIGURATION.md

@@ -4,9 +4,9 @@
 ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation:
 
 _______________
-#### Version: V4.2022.01.2866
+#### Version: V4.2022.01.2867
 #### Bad Referrers Blocked: [7087](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [633](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Bad User-Agents (Bots) Blocked: [636](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
 #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
 ____________________
 

MANUAL-CONFIGURATION.md

@@ -4,9 +4,9 @@
 ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation:
 
 _______________
-#### Version: V4.2022.01.2866
+#### Version: V4.2022.01.2867
 #### Bad Referrers Blocked: [7087](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [633](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Bad User-Agents (Bots) Blocked: [636](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
 #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
 ____________________
 

README.md

@@ -13,9 +13,9 @@
 ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots)
 
 _______________
-#### Version: V4.2022.01.2866
+#### Version: V4.2022.01.2867
 #### Bad Referrers Blocked: [7087](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [633](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Bad User-Agents (Bots) Blocked: [636](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
 #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
 ____________________
 

_generator_lists/bad-user-agents.list

@@ -27,6 +27,7 @@ Anthill
 Apexoo
 Aspiegel
 Asterias
+Atomseobot
 Attach
 AwarioRssBot
 AwarioSmartBot
@@ -185,6 +186,7 @@ Harvest
 Havij
 Heritrix
 Hloader
+HonoluluBot
 Humanlinks
 HybridBot
 IDBTE4M
@@ -284,6 +286,7 @@ Mister\ PiX
 Moblie Safari
 Mojeek
 Mojolicious
+MolokaiBot
 Morfeus\ Fucking\ Scanner
 Mozlila
 Mr.4x3
@@ -631,6 +634,3 @@ x22Mozilla
 xpymep1.exe
 zauba.io
 zgrab
-Atomseobot
-HonoluluBot
-MolokaiBot

_google_analytics_ghost_spam/README.md

@@ -3,9 +3,9 @@
 # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM
 
 _______________
-#### Version: V4.2022.01.2866
+#### Version: V4.2022.01.2867
 #### Bad Referrers Blocked: [7087](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)
-#### Bad User-Agents (Bots) Blocked: [633](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
+#### Bad User-Agents (Bots) Blocked: [636](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
 #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list)
 ____________________
 

dev-tools/_robots_input/robots-input.txt

@@ -27,6 +27,7 @@ Anthill
 Apexoo
 Aspiegel
 Asterias
+Atomseobot
 Attach
 AwarioRssBot
 AwarioSmartBot
@@ -185,6 +186,7 @@ Harvest
 Havij
 Heritrix
 Hloader
+HonoluluBot
 Humanlinks
 HybridBot
 IDBTE4M
@@ -284,6 +286,7 @@ Mister PiX
 Moblie Safari
 Mojeek
 Mojolicious
+MolokaiBot
 Morfeus Fucking Scanner
 Mozlila
 Mr.4x3

dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf

@@ -1,2 +1,2 @@
-10.1.0.150	1;
+10.1.0.235	1;
 127.0.0.1	1;

dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf

@@ -1,2 +1,2 @@
-10.1.0.150	0;
+10.1.0.235	0;
 127.0.0.1	0;

dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf

@@ -1,2 +1,2 @@
-10.1.0.150	1;
+10.1.0.235	1;
 127.0.0.1	1;

dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf

@@ -1,2 +1,2 @@
-10.1.0.150	0;
+10.1.0.235	0;
 127.0.0.1	0;

dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf

@@ -1,2 +1,2 @@
-10.1.0.150	1;
+10.1.0.235	1;
 127.0.0.1	1;

dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf

@@ -88,6 +88,7 @@
 	"~*(?:\b)Apexoo(?:\b|)"		0;
 	"~*(?:\b)Aspiegel(?:\b|)"		0;
 	"~*(?:\b)Asterias(?:\b|)"		0;
+	"~*(?:\b)Atomseobot(?:\b|)"		0;
 	"~*(?:\b)Attach(?:\b|)"		0;
 	"~*(?:\b)AwarioRssBot(?:\b|)"		0;
 	"~*(?:\b)AwarioSmartBot(?:\b|)"		0;
@@ -246,6 +247,7 @@
 	"~*(?:\b)Havij(?:\b|)"		0;
 	"~*(?:\b)Heritrix(?:\b|)"		0;
 	"~*(?:\b)Hloader(?:\b|)"		0;
+	"~*(?:\b)HonoluluBot(?:\b|)"		0;
 	"~*(?:\b)Humanlinks(?:\b|)"		0;
 	"~*(?:\b)HybridBot(?:\b|)"		0;
 	"~*(?:\b)IDBTE4M(?:\b|)"		0;
@@ -345,6 +347,7 @@
 	"~*(?:\b)Moblie Safari(?:\b|)"		0;
 	"~*(?:\b)Mojeek(?:\b|)"		0;
 	"~*(?:\b)Mojolicious(?:\b|)"		0;
+	"~*(?:\b)MolokaiBot(?:\b|)"		0;
 	"~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)"		0;
 	"~*(?:\b)Mozlila(?:\b|)"		0;
 	"~*(?:\b)Mr.4x3(?:\b|)"		0;

dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf

@@ -1,2 +1,2 @@
-10.1.0.150	0;
+10.1.0.235	0;
 127.0.0.1	0;

dev-tools/buildnumber

@@ -1 +1 @@
-2866
+2867

dev-tools/test1_conf_files/bots.d/whitelist-ips.conf

@@ -45,4 +45,4 @@
 # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. 
 # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
 # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-52.173.16.87            0;
+20.85.236.210           0;

dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf

@@ -1,2 +1,2 @@
-10.1.0.150	1;
+10.1.0.235	1;
 127.0.0.1	1;

dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf

@@ -1,2 +1,2 @@
-10.1.0.150	0;
+10.1.0.235	0;
 127.0.0.1	0;

dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf

@@ -1,2 +1,2 @@
-10.1.0.150	1;
+10.1.0.235	1;
 127.0.0.1	1;

dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf

@@ -1,2 +1,2 @@
-10.1.0.150	0;
+10.1.0.235	0;
 127.0.0.1	0;

dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf

@@ -1,2 +1,2 @@
-10.1.0.150	1;
+10.1.0.235	1;
 127.0.0.1	1;

dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf

@@ -88,6 +88,7 @@
 	"~*(?:\b)Apexoo(?:\b|)"		0;
 	"~*(?:\b)Aspiegel(?:\b|)"		0;
 	"~*(?:\b)Asterias(?:\b|)"		0;
+	"~*(?:\b)Atomseobot(?:\b|)"		0;
 	"~*(?:\b)Attach(?:\b|)"		0;
 	"~*(?:\b)AwarioRssBot(?:\b|)"		0;
 	"~*(?:\b)AwarioSmartBot(?:\b|)"		0;
@@ -246,6 +247,7 @@
 	"~*(?:\b)Havij(?:\b|)"		0;
 	"~*(?:\b)Heritrix(?:\b|)"		0;
 	"~*(?:\b)Hloader(?:\b|)"		0;
+	"~*(?:\b)HonoluluBot(?:\b|)"		0;
 	"~*(?:\b)Humanlinks(?:\b|)"		0;
 	"~*(?:\b)HybridBot(?:\b|)"		0;
 	"~*(?:\b)IDBTE4M(?:\b|)"		0;
@@ -345,6 +347,7 @@
 	"~*(?:\b)Moblie Safari(?:\b|)"		0;
 	"~*(?:\b)Mojeek(?:\b|)"		0;
 	"~*(?:\b)Mojolicious(?:\b|)"		0;
+	"~*(?:\b)MolokaiBot(?:\b|)"		0;
 	"~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)"		0;
 	"~*(?:\b)Mozlila(?:\b|)"		0;
 	"~*(?:\b)Mr.4x3(?:\b|)"		0;

dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf

@@ -1,2 +1,2 @@
-10.1.0.150	0;
+10.1.0.235	0;
 127.0.0.1	0;

dev-tools/test2_conf_files/bots.d/whitelist-ips.conf

@@ -45,4 +45,4 @@
 # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. 
 # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
 # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-52.173.16.87            0;
+20.85.236.210           0;

dev-tools/test3_conf_files/bots.d/whitelist-ips.conf

@@ -45,4 +45,4 @@
 # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. 
 # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks.
 # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST
-52.173.16.87            0;
+20.85.236.210           0;

dev-tools/test_units/blacklist-ips.conf

@@ -1,2 +1,2 @@
-10.1.0.150	1;
+10.1.0.235	1;
 127.0.0.1	1;

dev-tools/test_units/blacklist-user-agents.conf

@@ -88,6 +88,7 @@
 	"~*(?:\b)Apexoo(?:\b|)"		0;
 	"~*(?:\b)Aspiegel(?:\b|)"		0;
 	"~*(?:\b)Asterias(?:\b|)"		0;
+	"~*(?:\b)Atomseobot(?:\b|)"		0;
 	"~*(?:\b)Attach(?:\b|)"		0;
 	"~*(?:\b)AwarioRssBot(?:\b|)"		0;
 	"~*(?:\b)AwarioSmartBot(?:\b|)"		0;
@@ -246,6 +247,7 @@
 	"~*(?:\b)Havij(?:\b|)"		0;
 	"~*(?:\b)Heritrix(?:\b|)"		0;
 	"~*(?:\b)Hloader(?:\b|)"		0;
+	"~*(?:\b)HonoluluBot(?:\b|)"		0;
 	"~*(?:\b)Humanlinks(?:\b|)"		0;
 	"~*(?:\b)HybridBot(?:\b|)"		0;
 	"~*(?:\b)IDBTE4M(?:\b|)"		0;
@@ -345,6 +347,7 @@
 	"~*(?:\b)Moblie Safari(?:\b|)"		0;
 	"~*(?:\b)Mojeek(?:\b|)"		0;
 	"~*(?:\b)Mojolicious(?:\b|)"		0;
+	"~*(?:\b)MolokaiBot(?:\b|)"		0;
 	"~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)"		0;
 	"~*(?:\b)Mozlila(?:\b|)"		0;
 	"~*(?:\b)Mr.4x3(?:\b|)"		0;

dev-tools/test_units/random-bots-for-test-quick.list

@@ -1,10 +1,10 @@
-ALittle Client
+Demon
-Barkrowler
+Discoverybot
-CheeseBot
+GridBot
-InterGET
+Not
-Page Analyzer
+PageScorer
-Seomoz
+ProPowerBot
-Web Auto
+Swiftbot
-Zitebot
+VCI
-sysscan
+WebLeacher
-trendiction.de
+WiseGuys Robot

dev-tools/test_units/random-bots-for-test.list

@@ -1,250 +1,250 @@
 80legs
 ADmantX
+AIBOT
 ALittle Client
-ASPSeek
+AfD-Verbotsverfahren
-Abonti
-Aboundexbot
-Acunetix
 AiHitBot
 Aipbot
-Alexibot
+AllSubmitter
 Anarchy
+Anarchy99
 Anthill
+Aspiegel
 AwarioRssBot
-BLEXBot
+Backlink-Ceck
-BackWeb
-Badass
 BatchFTP
 Battleztar Bazinga
-Boardreader
+BetaBot
+Bitacle
+Blackboard
+Blow
+Brandprotect
+Buck
 BuiltWith
-CCBot
+Bullseye
+CATExplorador
+CODE87
 CSHttp
+CazoodleBot
 Cegbfeieh
+CensysInspect
+CheTeam
 ChinaClaw
 Chlooe
-Claritybot
 Cliqzbot
-Cocolyzebot
+Cogentbot
+Collector
 CopyRightCheck
 Crawling at Home Project
 CrazyWebCrawler
+CrunchBot
 Custo
-DBLBot
+CyotekWebCopy
 DTS Agent
+DataCha0s
 Deusu
-Devil
+Dirbuster
-Digincore
 Disco
 Dispatch
-DomCopBot
+DnyzBot
+DomainCrawler
 DomainSigmaCrawler
-Domains Project
 Download Wonder
-ECCP/1.0
+Dragonfly
 EasyDL
 EirGrabber
+Evil
 Exabot
-Express WebPictures
+ExtLinksBot
-Extractor
-ExtractorPro
-FemtosearchBot
 Fimap
-FlashGet
-Foobot
 Freeuploader
 FrontPage
-Fuzz
 Fyrebot
-G-i-g-a-b-o-t
-GalaxyBot
-Genieo
-Getintent
 Gigabot
-Go!Zilla
 Go-Ahead-Got-It
-Grafula
+GoZilla
+GrabNet
 GrapeFX
+GrapeshotCrawler
 GridBot
-HEADMasterSEO
+HTMLparser
-HMView
+HTTrack
-Harvest
+Haansoft
-Hloader
+HonoluluBot
-Iblog
+Humanlinks
-Id-search
+HybridBot
+IRLbot
+Image Fetch
 IndeedBot
-Indy Library
+InfoNaviRobot
 InfoTekies
-InterGET
+Internet Ninja
 InternetSeer
 Iria
-Iskanie
+Joomla
-JamesBOT
+Keyword Density
-JennyBot
+Kozmosbot
-Jetty
-JikeSpider
-Jorgee
-JustView
-Jyxobot
 LNSpiderguy
-Lanshanbot
+LWP::Simple
-Lightspeedsystems
+Larbin
-Likse
+LeechFTP
+LeechGet
+LexiBot
+Lftp
+Libwhisker
+LieBaoFast
 LinkScan
 LinkWalker
+Linkbot
+LinkextractorPro
+LinkpadBot
+LinksManager
+LinqiaMetadataDownloaderBot
 LinqiaRSSBot
-Lipperhey
+LinqiaScrapeBot
-Lipperhey Spider
 Lmspider
-MIDown tool
+MFC_Tear_Sample
+MIIxpc
+MJ12bot
+MQQBrowser
 MSFrontPage
 MTRobot
-Magnet
+Mag-Net
+Mail.RU_Bot
+Majestic-SEO
 Majestic12
-MarkWatch
+MarkMonitor
-Mass Downloader
+Masscan
-MauiBot
+Meanpathbot
-MeanPath Bot
-MicroMessenger
-Moblie Safari
 Mojolicious
-Morfeus Fucking Scanner
+MolokaiBot
-Mozlila
 Msrabot
-Musobot
+NPbot
-Name Intelligence
-Nameprotect
 Navroad
-NearSite
+Nessus
-NetSpider
+NetMechanic
-NextGenSearchBot
+Nettrack
-Nikto
+Netvibes
-Nimbostratus
+Niki-bot
-Ninja
 Nmap
-OnCrawl
+Nuclei
-OpenLinkProfiler
+Octopus
-Openfind
+Pandalytics
-OrangeSpider
-POE-Component-Client-HTTP
-Page Analyzer
-PageAnalyzer
-PageGrabber
-PageScorer
-PageThing.com
 Panscient
-Papa Foto
-Pavuk
 PeoplePal
-Picsearch
+Petalbot
+Picscout
+Piepmatz
 Pimonster
-ProPowerBot
+Pixray
-Pu_iN
+PleaseCrawl
-Pump
+ProWebWalker
-PyCurl
+Probethenet
+Psbot
 QueryN Metasearch
-RSSingBot
+Quick-Crawler
-RankurBot
+RankActive
+RankingBot
+Reaper
 RebelMouse
-Ripper
+Recorder
-RocketCrawler
+RedesScrapy
+RepoMonkey
 SEOkicks
-SEOkicks-Robot
-SEOprofiler
 SISTRIX
-Scanbot
+SMTBot
+ScanAlert
 Scrapy
-Seekport
+ScrepyBot
-Semrush
+Searchestate
+SemanticJuice
+SemrushBot
 SentiBot
 Seomoz
+Siphon
 Site Sucker
-SiteCheckerBotCrawler
 SiteExplorer
-SiteSnagger
+SiteLockSpider
-Sitebeam
+SiteSucker
-SmartDownload
+Sitevigil
+Snoopy
 SocialRankIOBot
-Sottopop
+Sogou web spider
-Spammen
+SpaceBison
+Spanner
+Spbot
 Spinn3r
-Sqworm
+Sucuri
-Steeler
+Surfbot
-Sucker
+Szukacz
-SuperHTTP
-Suzuran
 Teleport
-Telesphoreo
 Telesphorep
-TheNomad
+The Intraformant
-Turingos
+Trendiction
-TwengaBot
+Trendictionbot
+TurnitinBot
 Twice
+Typhoeus
 URLy.Warning
-UnisterBot
-Upflow
-VB Project
-VCI
 Vacuum
-Vagabondo
+VelenPublicWebCrawler
 VidibleScraper
-VoidEYE
+Voil
 Voltron
-WASALive-Bot
-WBSearchBot
 WEBDAV
-WPScan
+WISENutbot
-WWW-Collector-E
-WWW-Mechanize
 WWWOFFLE
 Wallpapers/3.0
-WallpapersHD
+Web Auto
-WeSEE
-Web Collage
 Web Enhancer
 Web Fuck
-Web Pix
 Web Sucker
 WebCollage
-WebEnhancer
+WebCopier
-WebImageCollector
+WebFetch
-WebReaper
+WebFuck
-WebWhacker
+WebLeacher
-Website Quester
+WebSauger
+Webalta
+Webshag
 WebsiteQuester
+Webster
 Whack
-Whatweb
+Wotbox
+Wprecon
+Xaldon WebSpider
+YoudaoBot
 Zade
-Zauba
+Zermelo
 Zeus
 Zitebot
-ZoomBot
-ZumBot
 adscanner
-archive.org_bot
+autoemailspider
-arquivo.pt
-backlink-check
 cah.io.community
-coccocbot-web
+clark-crawler
-crawler.feedback
+com.plumanalytics
+crawler4j
+demandbase-bot
 domainsproject.org
-evc-batch
+eCatch
-facebookscraper
+gopher
-instabid
+heritrix
-ips-agent
 isitwp.com
+lwp-request
 magpie-crawler
+meanpathbot
 muhstik-scan
-netEstate NE Crawler
+oBot
 plumanalytics
-polaris version
+ripz
-scalaj-http
+seocompany.store
-scan.lol
 seoscanners
-seostar
+serpstatbot
 sitechecker.pro
-sogouspider
+siteripz
+sysscan
 tAkeOut
 trendiction.com
+webmeup-crawler
 webpros.com
-webprosbot
+xpymep1.exe

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
+ASPSeek
-404checker
+Abonti
-404enemy
-AIBOT
-ALittle Client
-Acunetix
 AhrefsBot
-AiHitBot
+Aipbot
-Alexibot
+AllSubmitter
-AwarioRssBot
+Anarchie
-AwarioSmartBot
+Anthill
-BDFetch
+Aspiegel
+Atomseobot
+Attach
+BBBike
 BLEXBot
-BatchFTP
+BackWeb
+Badass
+Bandit
+Battleztar Bazinga
 BetaBot
-BlackWidow
+Bitacle
-Blow
+Black Hole
-Boardreader
+Bolt
+BotALot
 Buddy
 BuiltBotTough
+BuiltWith
 BunnySlippers
-CODE87
+CazoodleBot
-CSHttp
+CensysInspect
-Cegbfeieh
+ChinaClaw
-CheTeam
+Cliqzbot
-Claritybot
+Cocolyzebot
 Cogentbot
 Collector
-Copier
-CopyRightCheck
 Cosmos
-Crawling at Home Project
+Craftbot
-CrunchBot
+Curious
+Custo
 CyotekWebCopy
-DIIbot
+DatabaseDriverMysqli
+Demon
+Deusu
 Dirbuster
-Disco
+DittoSpyder
-Dispatch
-DomainAppender
-DomainSigmaCrawler
-Dotbot
-Download Wonder
-Dragonfly
 Drip
 EMail Wolf
-Ebingbong
-Ecxi
 EirGrabber
-EroCrawler
 Evil
+Exabot
 ExtLinksBot
-ExtractorPro
 Extreme Picture Finder
-FHscan
+EyeNetIE
+Ezooms
+FDM
 FemtosearchBot
-Fimap
 FlashGet
+Flunky
 Foobot
 FrontPage
-Go!Zilla
+G-i-g-a-b-o-t
-GoZilla
+GT::WWW
-Grafula
+GalaxyBot
-HEADMasterSEO
+GetWeb
-HMView
+Gigabot
-HTTrack
+Gotit
-Harvest
+HTMLparser
-Havij
+Hloader
-IDBTE4M
+HybridBot
-IRLbot
+IDBot
-Iblog
+Id-search
 IlseBot
-Indy Library
 InfoNaviRobot
-Intelliseek
+Internet Ninja
-InterGET
+Iskanie
-InternetSeer
+JOC Web Spider
-IstellaBot
+Jbrofuzz
 Jetty
 JikeSpider
-Jyxobot
+Joomla
+JustView
 Kenjin Spider
 Keyword Density
 Kinza
-Lanshanbot
+Kozmosbot
+LNSpiderguy
+LWP::Simple
 LexiBot
 Lftp
 LibWeb
 Libwhisker
 Lightspeedsystems
+LinkScan
 Linkbot
-Linkdexbot
 LinkextractorPro
-LinkpadBot
 Lipperhey
 Litemage_walker
-MFC_Tear_Sample
+Lmspider
-MTRobot
+MIDown tool
-Magnet
+MJ12bot
+MQQBrowser
+MSIECrawler
 Majestic SEO
 Majestic-SEO
+MarkMonitor
 Masscan
 Mb2345Browser
-MeanPath Bot
 Meanpathbot
+Metauri
 MicroMessenger
-Microsoft URL Control
+Minefield
 Mister PiX
+Moblie Safari
+Mojeek
+Mojolicious
+Morfeus Fucking Scanner
+Msrabot
 NICErsPRO
-NPbot
-Name Intelligence
 Nameprotect
-Nessus
+Navroad
 Net Vampire
-NetLyzer
+NetSpider
-Netcraft
 Netvibes
+Nibbler
 Nikto
-Nmap
+Ninja
-Octopus
-Offline Explorer
-Offline Navigator
 Openfind
-Openvas
 OrangeBot
 OutclicksBot
 OutfoxBot
+PECL::HTTP
 PHPCrawl
 POE-Component-Client-HTTP
-Page Analyzer
+PageAnalyzer
-PageGrabber
-PageScorer
 PageThing.com
+Panscient
 Pavuk
-PeoplePal
 Petalbot
 Picscout
-Picsearch
+PictureFinder
-Pimonster
+Piepmatz
-ProPowerBot
+PleaseCrawl
 ProWebWalker
 Probethenet
 Pu_iN
+PxBroker
 QueryN Metasearch
-RSSingBot
+Quick-Crawler
+RankActive
 RankActiveLinkBot
 RankFlex
-RankingBot
+Reaper
-RankurBot
+RebelMouse
-ReGet
+RocketCrawler
-RealDownload
+Rogerbot
-Recorder
-RedesScrapy
-Ripper
-SBIder
 SEOkicks-Robot
-SEOlyticsCrawler
+SMTBot
-SEOprofiler
-SEOstats
 Scanbot
-Scrapy
 Screaming
-Searchestate
+ScrepyBot
+SearchmetricsBot
 Seekport
-SentiBot
+SemanticJuice
-SeoSiteCheckup
+Siphon
-SeobilityBot
-Shodan
-Site Sucker
-SiteExplorer
 SiteLockSpider
-SiteSucker
 Siteimprove
-SlySearch
 SmartDownload
 Snake
-Snoopy
+Sogou web spider
+Sottopop
+SpaceBison
+SpankBot
 Spanner
-Spinn3r
+Spbot
-Sqlmap
+Sqworm
-Sqlworm
+Stripper
-Steeler
-Sucuri
-SuperHTTP
-Suzuran
 Swiftbot
+T0PHackTeam
 T8Abot
+Teleport
 Telesoft
-TightTwatBot
+Telesphoreo
-Toweyabot
+Thumbor
-Tracemyfile
-Trendictionbot
-Turingos
-TurnitinBot
 TwengaBot
-Typhoeus
+URLy Warning
+URLy.Warning
 Upflow
 VB Project
 VelenPublicWebCrawler
 VeriCiteCrawler
 VidibleScraper
-Voil
+Virusdie
-Voltron
+VoidEYE
-WISENutbot
+WASALive-Bot
-WWWOFFLE
-Wallpapers/3.0
 WeSEE
 Web Fetch
-Web Fuck
+Web Pix
-Web Sucker
+Web Sauger
 WebAuto
-WebCollage
-WebCopier
-WebEnhancer
 WebFetch
+WebGo IS
 WebImageCollector
 WebLeacher
 WebPix
-Webster
+WebWhacker
-Widow
+Webalta
-Wonderbot
+Webshag
+Website Quester
+WebsiteQuester
+Whacker
+Whatweb
 Woobot
 Wotbox
 Xenu
-Zermelo
+Zauba
+Zeus
 Zitebot
-ZoomBot
 ZyBorg
 archive.org_bot
-arquivo.pt
+arquivo-web-crawler
-autoemailspider
+backlink-check
+cah.io.community
+clark-crawler
 coccocbot-web
+crawler.feedback
 crawler4j
-demandbase-bot
 domainsproject.org
+eCatch
+evc-batch
 instabid
+ips-agent
 iubenda-radar
+lwp-request
+magpie-crawler
+mediawords
 muhstik-scan
 netEstate NE Crawler
-polaris version
-probe-image-size
 ripz
 s1z.ru
-sp_auditbot
+satoristudio.net
-trendiction.com
+scalaj-http
-trendiction.de
+scan.lol
-webgains-bot
+seobility
-webprosbot
+seocompany.store
+seoscanners
+seostar
+sysscan
+tAkeOut
+voyagerx.com
+webmeup-crawler
+x09Mozilla
 x22Mozilla
 xpymep1.exe
-zgrab

dev-tools/test_units/random-referrers-for-test-quick.list

@@ -1,10 +1,10 @@
-e705.net
+adzpower.com
-findpik.com
+customcollegeessays.net
-freza-sverlo.ru
+espaceinventoristes.com
-gemgrab-a.akamaihd.net
+meget.co.za
-sitiz.club
+onlineslotmaschine.com
-unitexindia.com
+saltspray.ru
-validccseller.com
+seinterface.com
-voloslove.ru
+surintech.ac.th
-vseigry.fun
+topcar-krasnodar.ru
-zreloeporno.tv
+zoogdisany.com

dev-tools/test_units/random-referrers-for-test.list

@@ -1,500 +1,500 @@
-0daymusic.org
+1adult.com
-1000n1.ru
+202ch.com
-108shot.com
+34.gs
-214jaluwobapef.cf
+4replicawatch.net
-2itech.ru
+4webmasters.org
-2pxg8bcf.top
+7minuteworkout.com
-3w24.com
+a.pr-cy.ru
-4kplayer.pl
+absolutelyanalog.com
-4ureyesonly.com
+adcash.com
-7makemoneyonline.com
+add-add.men
-7xc4n.com
+admitad.com
-8lufu.com
+adtech.us
-aa8780bb28a1de4eb5bff33c28a218a930.com
+afora.ru
-aa8b68101d388c446389283820863176e7.com
+agahinameh.com
-abcdeg.xyz
+agysya.ru
-absolute-s.ru
+alif-ba-ta.com
-addbin.men
+alpharma.net
-adssafeprotected.com
+android-systems.ru
-agardomains.com
+angel1777.kz
-aihelen.net
+angry-fermi-7633.arukascloud.io
-air-edem.ru
+anime.dougasouko.com
-airlandsea.info
+anonymizeme.pro
-albamargroup.com
+apiluckyleapnet-a.akamaihd.net
-alevinefesleri.com
+appleid-verification.com
-alfa-img.com
+applicationg29.com
-alfabot.xyz
+applyneedy.xyz
-allsilver925.co.il
+arkartex.ru
-anal-acrobats.hol.es
+artparquet.ru
-anapa-inns.ru
-angkortours.vn
-animenime.ru
-anjalika.co.in
-app5.letmacworkfaster.world
-apper.de
-appfastplay.com
-aproposde.com
-arabgirls.adultgalls.com
-artdeko.info
-arvut.org
-asacopaco.tk
-asiavirtualsolutions.com
 asrvrep-a.akamaihd.net
-atleticpharm.org
+asscenihotosidea.blogspot.com
-auto4style.ru
+atlant-auto.info
-autochoixspinelli.com
+avon-novosib.ru
-autogrep.ru
+avtovolop.ru
-autotop.com.ua
+awaybird.ru
-aviapanda.ru
+azartniy-bonus.com
-avitocars.ru
+azazaporn.com
-avtolombard-krasnodar.com
+bambi.ck.ua
-ayeartoforget.com
+batanga.net
-azbaseimages.net
+beachdriveblog.com
-bankofthewext.com
+beachpics.com
-bavsac.com
+beauty-b0x.pl
-bbtec.net
+beauty-bracelet.com
-bdsmgalls.net
+benchmarkcommunications.co.uk
-bedandbreakfast.com
+berdasovivan.ru
-best-way.men
+bestserials.com
-bet365.com
+betshuckclean.com
-betonka.pro
+bidr.trellian.com
-beyan.host.sk
+biographiya.com
-bgviagrachrx.com
+bird1.ru
-biglistofwebsites.com
+bitcoin-ua.top
-bin-brokers.com
+biteg.xyz
-blog-hits.com
+biznesluxe.ru
-blogig.org
+bizzliving.com
-blogsfunky672.weebly.com
+bloggers.nl
-bradleylive.xyz
+bodybuilding-shop.biz
-browserprotecter.com
+bookmark4you.com
-buehne-fuer-menschenrechte.de
+bpro1.top
-burger-imperia.com
+brendbutik.ru
-buy-cheap-online.info
+brianhenry.co
-buypanicdisorderpill.com
+bristolhostel.com
-buytizanidineonline.blogspot.com
+bristolhotel.com.ua
-buytizanidineonlinenoprescription.blogspot.com
+buckspar.gq
-buzz-porno.info
+budilneg.xyz
-buzzsumo.com
+business-made-fun.com
-bvlgariring.xyz
+buyfriend.ru
-byme.se
+bytimedance.ru
-cam-kontakte.org
+callawaygolfoutlet.online
+camsex.xxx-cam.webcam
+candlehandmade.com
+candlewooddentalcentre.com.au
 canrioloadm.gq
-capture-room.com
+cartierwatch.xyz
-carfax.com.ua
-cashkitten-a.akamaihd.net
-catalogs-parts.com
-caveavins.fr
 cayado.snn.gr
-celebsopics.com
+cbox.ws
-cgi2.nintendo.co.jp
+centrumcoachingu.com
-cheapjerseysfootballshop.com
+chastnoeporno.com
-check-this-out-now.online
+cheap-pills-norx.com
-checkhit.com
+chinacheapelitejerseys.com
-choosecuisine.com
+christianlouboutinshoes.xyz
-clickso.com
+cialgenisrx.com
-clippingphotoindia.com
+cialischstgerts.com
-compliance-alexa.xyz
+cicaki.net
-compliance-ivan.xyz
+citizenclsdriveri7.pen.io
+clickpapa.com
+club-musics.ru
+coin-hive.com
+coindirect.io
+coinsspb.com
+compliance-barak.top
+compliance-jack.top
 compliance-john.top
+compliance-walter.top
 conboy.us
-cpabegins.ru
+cookie-law-enforcement-ee.xyz
-cracksplanet.com
+coolgramgoods.com
-crd.clan.su
+coolyarddecorations.com
-credit-respect.ru
+cornerstone-countertops.com
-cv.wallhade.co
+counter.bloke.com
-d0t.ru
+crafthubs.com
-d2jsp.org
+customcatchcan.com
 daretodonate.co
-degerlund.net
+dealighted.com
-delitime.info
+deanmoore.ie
+deniven.1bb.ru
 design-lands.ru
-dev-seo.blog
+dfiles.me
-dgfitness.co
+dirtpics.pw
-dipstar.org
+diy-handmade-ideas.com
-divci-hry.info
+docstoc.com
-dms-sw.ru
+dogoimage.com
-domainsphoto.com
+dojki365.online
-domik-derevne.ru
+dollartree.info
-dood.live
+domain-tracker.com
-doreenblog.online
+domainanalyzing.xyz
-downloads-whatsapp.com
+domaincheck.io
-drhomes.biz
+domoysshop.ru
-drivotracker.com
+dpihatinh.gov.vn
-dtnlyss.com
+driving.kiev.ua
-e-kwiaciarz.pl
+druzhininevgeniy63.blogspot.com
-el-nation.com
+dscaas.website
-eldiariodeguadalajara.com
+dtm-spain.com
-electro-prom.com
+ecookna.com.ua
-elektrozigarette1.dreamwidth.org
+ecxtracking.com
-elvel.com.ua
+edudocs.net
-englate.com
+eloconcream.blogspot.com
-englishgamer.com
+enge-fotzen.info
-enthuse.computernetworksonline.com
+eorogo.top
+ero-advertising.com
+esopini.com
 espaceinventoristes.com
-essayassist.com
+european-torches.ru
-eugenevaultstorage.com
+everflora.ru
-evidencecleanergold.com
+expertblog.info
-exdocsfiles.com
+eyessurgery.ru
-exonline.info
+ezigarettekaufen2.yolasite.com
-extener.org
-extlinks.com
-eyes-on-you.ga
 ezigarettenshop2.myblog.de
-f07.de
+f5mtrack.com
-faptitans.com
+fashion-mk.net
-faracontrol.ir
+fba-mexico.com
-fastfixing.tech
 femmesdenudees.com
+fenoyl.batcave.net
 fermersovet.ru
-filmgo.ru
+fertilitetsradgivningen.se
-flauntyoursite.com
+ff30236ddef1465f88547e760973d70a.com
-flavors.me
+fickkontakte.org
-footbalive.org
+fidelityfunding.com
+fighrofacciufreesig.ga
+financeloan.us
+financepoints.eu
+fingerprintjs.com
+fiuxy.com
+flagcounter.me
+flashahead.info
+flipper.top
+floridahuntingfishingadventures.com
 fordsonmajbor.cf
-forodvd.com
-forum.doctissimo.fr
 forums.toucharcade.com
 foto-telok.net
-fotzen-ficken.com
+foxinsocks.ru
-free-share-buttons.com
+freemaintenancesysforpcandmac.top
-free-stock-illustration.com
-free-traffic.xyz
 front.ru
-frvo.alptandem.ru
+galeria-zdjec.com
-fsakhalin.ru
+gallerily.com
-fusoradio.info
+gamevalue7.weebly.com
-gallerylisting.com
+gaming-journal.com
-gambarxkata.co
 gateway.zscalerone.net
 getpopunder.com
-given2.com
+glavtral.ru
-glavprofit.ru
+gne8.com
-global-ics.co.za
+goforexvps.com
-globatur.ru
+gogps.me
+gojiberriess.apishops.ru
 gok-kasten.net
-gomusix.com
+golaya.pw
-goodwriterssales.com
+gratis-sexkontakte.com
-gpms.org.my
+greenidesign.co
-gsasearchenginerankerhelp.com
+grupografico-pilar.com.ar
+gsasearchengineranker.site
 gsbs.com.ua
-guiadeserraazul.com
+guge.io
-guildebzh.info
+gunsvicceadadebt.tk
-hacktougroup.ru
+guod.me
-hebr.myddns-flir.com
+handsandlegs.ru
-heygidday.biz
+hard-porn.mobi
-historichometeam.com
+havepussy.com
-hitcpm.com
+homeinns.com
-hosting-tracker.com
+houseofrose.com
-hothot.ru
+hoverboard360.se
-houseofgaga.ru
+hspline.com
-hoverboard360.at
+https-legalrc.biz
-humbmosquina.tk
+hully.altervista.org
 husky-shop.cz
-hyhj.info
+hystersister.com
-ideashome.id
+iinstalll-fii1leis.jus0wil.pp.ua
-iklysha.ml
+ilmexico.com
-ilovevitaly.xyz
+imfamous.info
-imicrovision.com
+infogame.name
-informatiecentro.be
+infospot.pt
-interior-stickers.ru
+insurple.com
-iphantom.com
+intimshop-fantasy.ru
-iqbazar.ru
+investyb.com
-itch.io
+islamtoday.co.za
-itronics.ca
+ispaniya-costa-blanca.ru
-itsdp3.com
+istizanidinelikexanax.blogspot.com
-iwanttodeliver.com
+iweblist.info
-jaxcube.info
+jabimgo.pw
-jimmychoosale.online
+jacago.com
-jo24news.com
+jav-fetish.site
-journalhome.com
+javlibrary.cc
+jerseychinabizwholesale.com
+jerseyssportsshop.com
+jerseywholesaleelitestore.com
+jillepille.com
+johnnyhaley.top
 jrcigars.com
 justbcause.com
-jwcialislrt.com
+justprofit.xyz
-kartiniresto.com
+kaac.ru
-kazka.ru
+keywordsuggests.com
-kerei.ru
+kiinomaniak.pl
-ketrzyn.pl
+kinoflux.net
-kinomaniatv.pl
+kiprinform.com
-kiwe-analytics.com
+klitimg.pw
-klikbonus.com
+kndxbkdx.bloger.index.hr
-klin3952.ru
+kotaku.com
-kostenloser-sex.com
+kskjco.club
-kountrylife.com
-kretpicf.pw
-ku6.com
-kumuk.info
-kvartir-remont.biz
-kw21.org
 ladsblue.com
-landscapebackgrounds.blogspot.com
+lanasshop.ru
-languagecode.com
+lankarns.com
-laparfumotec.com
+larger.io
-larutti.ru
+leadwayau.com
-lescinq.com
+lernur.net
-letmacwork.world
-lexixxx.com
-lezbiyanki.net
 lida-ru.com
-light.ifmo.ru
+link.ac
-livejasmin.com
+liumimgx.pw
-lomza.info
+livesex-amateure.info
-lordzfilmz.me
+livetsomudvekslingstudent.bloggersdelight.dk
-lotto6888.com
+locationdesaison.com
-lovelycraftyhome.com
+loginduepunti.it
-lubetube.com
+lol-smurfs.com
-luckyshop.net.ua
+luckyday.world
-luxup.ru
+m3gadownload.pl
-m0r0zk0-krava.ru
+mac-shield.com
-macdamaged.tech
+mall.uk
-machicon-akihabara.info
+mamasuper.prom.ua
-mackeeper-center.club
+manifestyourmillion.com
-madot.onlinewebshop.net
+manve.info
-magazin-pics.ru
+marcowebonyodziezowe.pl
-magento-crew.net
+marketland.ml
-magiadinamibia.blogspot.com
+matchpal-a.akamaihd.net
-maladot.com
+maxxtor.eu
-manifestation.betteroffers.review
+mazda-roadsters.com
-manimpotence.com
+mb140.ru
-mansiondelrio.co
+mediaoffers.click
-manuscript.su
+medicineseasybuy.com
-markjaybeefractal.com
+medmajor.ru
-marmotstore.online
+megaapteka.ru
-martlinker.com
+midst.eu
-marwer.info
+militarysale.pro
-mearns-tractors.co.uk
+mir-business-24.ru
-mega-bony2017.pl
+mir-limuzinov.ru
-meget.co.za
-mhi-systems.ru
-micasainvest.com
-michaelkorsoutletstore.net
-mikrobiologies.ru
-minegam.com
-mitrasound.ru
 mjchamonix.org
-modabutik.ru
+mmoguider.ru
-moetomnenie.com
+mobot.site
-moi-glazki.ru
+mobplayer.net
-moivestiy.biz
+money-every-day.com
-moncleroutletonline.pw
+morlat.altervista.org
-montredemarque.nl
+morocco-nomad-excursions.com
 mossmesi.com
-mountainstream.ms
+mouselink.co
-mp3downloadhq.com
+moviezbonkerssk.cf
+mozello.ru
+mrcsa.com.au
+mrinsidesales.com
 msk.afora.ru
-muizre.ru
+mtmtv.info
-mural.co
+muztops.ru
-myfreetutorials.com
+my-big-family.com
-nakozhe.com
+myanyone.net
-nakrutka.cc
+myghillie.info
-nalogovyy-kodeks.ru
+myra.top
-nalogovyykodeks.ru
+narosty.com
-nash-krym.info
+net-profits.xyz
-net-radar.com
+nevansk.ru
 newstaffadsshop.club
-ngps1.ru
+newstudio.tv
+next-dentists.tk
+nikkiewart.ru
+nl.netlog.com
 no-fuel.org
-no-rx.info
+noclegonline.info
+nootrino.com
 northfacestore.online
-novosti-hi-tech.ru
+nvformula.ru
-nucia.biz.ly
+oconto.ru
-o-o-6-o-o.ru
+oivcvx.website
-odywpjtw.bloger.index.hr
+okout.ru
-offers.bycontext.com
-officedocuments.net
-ok-ua.info
 omsk.xrus.org
-onefilms.net
+online-hit.info
-ons-add.men
+ops.picscout.com
-osoznanie-narkotikam.net
+ordernorxx.com
-oz-offers.com
+oren-cats.ru
-partner-pop.men
+orgasmus-virtual.com
-partybunny.ru
+orhonit.com
-pastaleads.com
+oxford-book.com.ua
-pay2me.pl
+pandarastore.top
-phobia.us
+partner-high.men
-photochki.com
+partnerline.men
-pic2fly.com
+partnersafe.men
+pattersonsweb.com
+pavlodar.xkaz.org
+percin.biz.ly
+perl.dp.ua
+pestomou.info
+pewit.pw
 picscout.com
-pictures-and-images.com
+picturesboss.com
-pila.pl
+pihl.se
-pillscheap24h.com
+pinwallpaper.xyz
-pinwallpaper.top
+piski.top
-pl.aasoldes.fr
+pixshark.com
-play.leadzupc.com
+placid-rounded-coast.glitch.me
-popander.mobi
+playtap.us
-porndroids.com
+pochemychka.net
-pornhive.org
+porn555.com
+porno-dojki.net
+porno-home365.com
 porno-raskazy.ru
-pornogad.com
+pornokajf.com
-pozdravleniya-c.ru
+pornpost.in
-predmety.in.ua
+postclass.com
-privacyassistant.net
+pr-ten.de
-privacylocationforloc.com
+pricheski-video.com
-projefrio.com.br
+profit-opportunity.com
-propranolol40mg.blogspot.com
+progress-upakovka.ru
-prospekt-st.ru
+promalp-universal.ru
-prostitutki-rostova.org
+prosmibank.ru
+pskcijdc.bloger.index.hr
+pts163.ru
+pxhdwsm.com
 q-moto.ru
-quickbuck.com
+quality-traffic.com
-rankingchart.de
+quickloanbank.com
-rankscanner.com
+rabot.host.sk
-ranksignals.com
+ragecash.com
-realnye-otzyvy.info
+ratemodels.net
-relax.ru
+rating-bestcasino.com
-remybutler.fr
+realizmobi.com
-repeatlogo.co.uk
+rebelmouse.com
-respublica-otel.ru
+rebuildermedical.com
-revistaindustria.com
+redhotfreebies.co.uk
-rfd-split.hr
+remote-dba.de
-rique.host.sk
+rent2spb.ru
-robot-forex.biz
+rewardit.com
+rialp.getenjoyment.net
+rp9.ru
 rrutw.com
-rusoft-zone.ru
+ruscopybook.com
-rutor.group
+russia-today-video.ru
-sady-urala.ru
+russintv.fr
-salutmontreal.com
+rvzr-a.akamaihd.net
+s1z.ru
 sammlungfotos.online
-sanjosestartups.com
+sape.top
-savetubevideo.com
+sashagreyblog.ga
-sbdl.no
+scanner-marwin.top
-sbricur.com
+scanner-walt.top
-scanner-jack.top
+search-error.com
-scanner-margo.top
-school-diplomat.ru
-scoopquest.com
-scopich.com
-scrapinghub.com
-scrnet.biz.ua
 searchwebknow-a.akamaihd.net
-seasonvar.ru
+securesmrt-dt.com
-seo-prof1.xyz
+semrush.com
-seocheki.net
+seo-traffic-ranking.info
-seojokes.net
+seokicks.de
-sex-dating.co
+serpstat.com
-sex-sex-sex5.com
+serviporno.com
-sex-tracker.de
+sex-tracker.com
-sexfreepornoxxx.com
+sexkrasivo.net
-sexkontakte-seite.com
+sexpornotales.net
-sexreliz.com
+sexuria.net
-shakhtar-doneck.ru
+sexxdate.net
-shikiso.info
+sexystrippe.info
-shillyourcoins.com
+shiksabd.com
-shivafurnishings.com
+shisha-swag.de
-shtaketniki.ru
+shitting.pro
-silverage.ru
+shopwme.ru
-simpoed.ufop.br
+sildenafil-tadalafil.info
-sisiynas.ru
+simon3.ru
-skylta.com
+sisi-go.ru
-smartshoppymac.com
+sispe.com.br
-snsdeainavi.info
+sitopreferito.it
-soblaznu.net
+sivs.ru
-soc-econom-problems.ru
+sloopyjoes.com
+smokewithrabbits.com
+snyatie-lomki-v-stacionare.ru
+socas.pluto.ro
+social-buttons.xyz
+social-s-ggg.xyz
+social-search.me
+socialbuttons.xyz
 socialmadesimple.com
-sohoindia.net
+socialmonkee.com
-sotechco.co
+softomix.net
-spabali.org
+songplanet.ru
-spacash.com
+spammen.de
-speeddream.xyz
+srgwebmail.nl
-sports-supplements.us
+stard.shop
-spy-app.info
+stats-collector.org
-stairliftstrue.com
+stromerrealty.com
-startwp.org
-staticfs.host
-streamin.to
-streha-metalko.si
 stroy-portal22.ru
-supermodni.com.ua
+stroyhelp-dv.ru
-takeprofitsystem.com
+suhanpacktech.com
-tastyfoodideas.com
+svensk-poesi.com
-tds-advert005.info
+sygraem.com
-teenbbw.yopoint.in
+symbaloo.com
-teenfuck.tv
+tako3.com
-teesdaleflyballclub.co.uk
+tam-gde-more.ru
-tehngr.ru
+tech4master.com
-telegramdownload10.com
+tecspb.ru
-templates.radiodigital.co
+templates.franklinfire.co
-tilido.com
+terraclicks.com
-time-japan.ru
+terrafootwear.us
-tizanidine6mg.blogspot.com
+the-torrent-tracker.blogspot.com
-tizanidinecost.blogspot.com
+thebestphotos.eu
-tizanidinehydrochloride4mgstreetvalue.blogspot.com
+thecounter.com
-tizanidineoral.blogspot.com
+theporndude.com
-tizanidinestreetprice.blogspot.com
+tizanidinedrugtest.blogspot.com
-tomck.com
+tizanidineformigraines.blogspot.com
-top-study.work
+tizanidinehydrochloride2mg.blogspot.com
-top10-way.com
+tizanidinestreetvalue.blogspot.com
-tourcroatia.co.uk
+top-karkas.ru
+top-l2.com
+top1-seo-service.com
+topads.men
+topshef.ru
+touchmods.fr
+toyota.7zap.com
+tracfone.com
 trafficfactory.biz
-trafficnetzwerk.de
+trafficreceiver.club
 trafficshaper.com
-trailer.cinemaflix.website
+trubywriting.com
-tri-slona.org
+truebeauty.cc
-truck-land.life
+trustedmaccleaner.com
-tubeoffline.com
+turbabitload.weebly.com
-tupper-posuda.ru
+uchebavchehii.ru
-tvnewsclips.info
+unacittaconte.org
-twelvevisionspartyofcolorado.com
+uroffer.link
-uamtrk.com
+v720hd.ru
-uggbootsoutletsale.us
+valuado.com
-uggsale.online
+veerotech.com
-ukrtvory.in.ua
+verymes.xyz
-umekana.ru
+video-camer.com
-univerfiles.com
+video-hollywood.ru
-upproar.com
+videos-for-your-business.com
-uprour.com
+videositename.com
-userequip.com
+viel.su
-usswrite.com
+vigrx-original.ru
-v-doc.co
+visitcambridge.org
-validdomain.xyz
+vitalads.net
-valkiria-tk.ru
+vkontarkte.com
-vbikse.com
+vozbujdenie.com
-vekzdorov.ru
+vrotike.ru
-via-gra.webstarts.com
+vtcdns.com
-viberdownload10.com
+vzubah.com
-video-production.com
+waterefficiency.co
-videochat.life
+weblo.com
-videojam.tv
+webnode.me
-visa-china.ru
+westermarkanjou.se
-vita.com.hr
+wfb.hatedriveapart.com
-voloomoney.com
+whatsappdownload10.com
-voloslove.ru
+whiteelephantwellington.com
-voltrknc1.com
+wholesalejerseychinashop.com
-vriel.batcave.net
+whos.amung.us
-vse-pesni.com
+winner7777.net
-vvpg.ru
+wordkeyhelper.com
-wallpapers-best.com
+worldtraveler.world
-wallpapersdesk.info
+wosik-dach.service-for-web.de
-watchmygf.net
+x-porno.video
-webcamdevochka.com
+x69ty.ru
-webcamtalk.net
+xblognetwork.com
-welck.octopis.com
+xchangetrak.com
-wellcome2slovenia.ru
+xitjw.info
-whatzmyip.net
+xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai
-white-truck.life
+xn--80abgj3a5acid6ghs.top
-wholesalenfljerseys.us.com
+xn--80ak6aa92e.com
-worldis.me
+xn--b1ag5cfn.xn--p1ai
-www.souvenirua.com
+xn--hxazdsfy.blogspot.com
-x-lime.com
+xxx-cam.webcam
-xaijo.com
+xxxdatinglocal.us
-xn--80aebbcbcdemfkhba4byaehoejh8dza3v.xn--p1ai
-xn--e1afanlbnfckd7c3d.xn--p1ai
-xn--e1aggki3c.xn--80adxhks
-xwatt.ru
-y8games-free.com
 yaderenergy.ru
-yoluxuryevents.com
+yellowstonevisitortours.com
-yoopsie.com
+youbrainboost.asia
-youdesigner.kz
+yougotanewdomain.com
-yougame.biz
+zastroyka.org
-yougetsignal.com
+zeroredirect12.com
-youjizz.vc
+zhuravlev.info
-youraticles.pl
+zlatnajesen.com
-yourdesires.ru
+zojirushi-products.ru
-youresponsive.com
+zscalerone.net
-yourporn.com
+zscalertwo.net
-zaidia.xhost.ro
-zajm-pod-zalog-nedvizhimosti.ru
-zakazfutbolki.com
-zarabotat-na-sajte.ru
-zeroredirect11.com
-zhacker.net

dev-tools/test_units/random-referrers-for-whitelist-test.list

@@ -1,250 +1,250 @@
-1008.su
+2nt.ru
-4replicawatch.net
+7zap.com
-51.la
+aa08daf7e13b6345e09e92f771507fa5f4.com
-aa9bd78f328a6a41279d0fad0a88df1901.com
+abclauncher.com
-aasoldes.fr
+admanaerofoil.com
-adservme.com
+adultactioncam.com
-advokateg.ru
+agroeconom.kz
-akuhni.by
+alfa-img.com
-alarmobninsk.ru
+alldownload.pw
-alive-ua.com
+altamayoztourism.com
-all4wap.ru
+anaseracresar.tk
-anapa-inns.ru
+animali.deagostinipassion.it
-android-style.com
+anjalika.co.in
-anniemation.com
+apartamentwroclaw.eu
-aucoinhomes.com
+apxeo.info
-avtochehli.by
+arraty.altervista.org
-azbaseimages.net
+asseenontv.ru
-b2b-lounge.com
+autorn.ru
-beclean-nn.ru
+autovideobroadcast.com
-becuo.com
+avto-oligarh.ru
-belinvestdom.by
+axbocz.net
+balitouroffice.com
 bestcurs.org
-bibys.com
+bestpriceninja.com
-bidbuy.co.kr
+bet365.com
-bingoporno.com
+beztuberkuleza.ru
-bosefux.esy.es
+billiard-classic.com.ua
-brandbucket.com
+blackhatworth.com
-brandimensions.com
+blackwitchcraft.ru
-brewdom.ru
+blog.koreadaily.com
-browsepulse-a.akamaihd.net
+bloggers.nl
-buytizanidineonlinenoprescription.blogspot.com
+bolezniorganov.ru
-buzz-porno.info
+brainboosting.club
-callstevens.com
+braip.com.br
-candypeople.se
+brendbutik.ru
-casinopinup-wins.com
+bristolhotel.com.ua
-caveavins.fr
+bukleteg.xyz
+bus-offer.com
+buzzsumo.com
+cardiosport.com.ua
+cdncash.net
 cercacamion.it
-checktext.org
+cheapjerseysfootballshop.com
-cindymatches.com
+check-host.net
-comissionka.net
+chinaelitecheapjerseys.com
-compliance-andrew.xyz
+choice-credit.ru
-coprofam.org
+circlesl.com
-cougfan.info
+clickcash.com
-craftinsta.ru
+cmhomestayagency.com
-dekoration.us
+cobaltpro.ru
-dieswaene.com
+compliance-andrew.top
-dimfour.com
+conboy.us
-domaincrawler.com
+consorzioilmosaico.org
-downtuptv.gq
+constantaservice.net
-drunkenstepfather.com
+costablue.xyz
+counter.yadro.ru
+cv.wallhade.co
+dailyrank.net
+danashop.ru
+ddospanel.com
+defenderxtactical.com
+demian.kz
+deplim.com
+digital-video-processing.com
+dinosaurus.site
+divci-hry.info
+dosug-lux.ru
 drunkmoms.net
-elektronischezigarettekaufen1.myblog.de
+dyshagi.ru
-elmacho.xyz
+ekn-art.se
-empis.magix.net
+elektrischezigarette1.onsugar.com
-enews.tech
+eng-lyrics.com
-espaceinventoristes.com
+erot.co
-eugenevaultstorage.com
+escortslet.net
-extads.net
+essayservicewriting.org
-ezigarettekaufen2.mpbloggar.se
+ewebarticle.info
-ezigarettenshop2.postbit.com
+exoneration-project.us
-facebook-mobile.xyz
+ezigarettekaufen2.blox.pl
-falopicm.pw
+f0815.de
-fettefrauen.net
+fandlr.com
-filmbokep69.com
+fanrto.com
-findacheaplawyers.com
+farmprofi.net
-floridamobilebillboards.com
+feel-planet.com
-free-share-buttons.com
+finemanteam.com
-fres-news.com
+floating-share-buttons.com
-fuckbuddybestgilf.info
+forum.tvmir.org
-futbolkisales.ru
+fun-mobi.pl
-gardene.ru
+gameprimary.com
-gateway.zscalertwo.net
+gamesprite.me
-gembird.com
+gaming-journal.com
-get-free-social-traffic.com
+gdzkurokam.ru
-getaclueamerica.com
+geileweiber.tk
-getresponse.com
+genericlowlatencyasiodriverhq.aircus.com
-go2mike.ru
+gerhardhealer.com
-google-liar.ru
+go2jump.org
-gopixdatabase.com
+good-mummy.ru
-gov.yanao.ru
+gsasearchenginerankerhelp.com
-greatzip.com
+high-speed1.net
-groupmoney.ru
+hkladys.com
-gsbs.com.ua
+houseofrose.com
-hebr.myddns-flir.com
+imguramx.pw
-herokuapp.com
+incomekey.net
-hitsbox.info
+investpamm.ru
-hotxnights.info
+ip-guide.com
-housemilan.ru
+isistaylorporn.info
-hoverboard360.de
+jav-way.site
-hoztorg-opt.ru
+kidskunst.info
-hyhj.info
+kiskinhouse.com
-idc.com.ua
+kochanelli.com
-ideoworld.org
+komputers-best.ru
-ilbe.club
+koopilka.com
-incanto.in.ua
+krasnodar.zrus.org
-inet-traffic.com
+kriokomora.info
-intim-uslugi.info
+lancheck.net
-iskalko.ru
+ledpolice.ru
-istripper.com
+lescinq.com
-itrevolution.cf
+letsrepair.in
-iwantmyfreecash.com
+letto.by
-jagg.info
+liveinternet.ro
-jerseychinabizwholesale.com
+lockerz.com
-jetsli.de
+lostaruban.com
-jovencitas.gratis
+marketingvici.com
-justucalling32211123456789.tk
+markjaybeefractal.com
-kaac.ru
+matchpal-a.akamaihd.net
-keywesthideaways.co
+maxxtor.eu
-knowyournextmove.com
+medispainstitute.com.au
-krasnodar.xrus.org
+meteostate.com
-languagecode.com
+mir-business-24.ru
-lashstudia.ru
+mlf.hordo.win
-laxob.com
+mmofreegames.online
-le-clos-des-alouettes.com
+mmoguider.ru
-leftofcentrist.com
+mobplayer.ru
-life-instyle.com
+monclerboots.xyz
-linkredirect.biz
+monclercheap.xyz
-lotto6888.com
+moncleronline.xyz
-luciddiagnostics.in
+mountainstream.ms
-lyngdalhudterapi.no
+mozello.ru
-macfix.life
+myanyone.net
-magda-gadalka.ru
+mydoctorok.ru
-magnetic-bracelets.ru
+myftpupload.com
-marwer.info
+myonigroup.com
-meble-bogart.info
+myonlinepayday.co
-medizinreisen.de
+myshopmatemac.com
-meget.co.za
+nac-bearings.ru
-mex-annushka.ru
+neks.info
-micasainvest.com
+nfljerseys.online
-microstatic.pl
+nhl17coins.exblog.jp
-mlvc4zzw.space
+ooo-gotovie.ru
-moncleroutletonline.xyz
+ooo-olni.ru
-moonci.ru
+oxford-book.com.ua
-mp3ritm.top
+ozas.net
-mriyadh.com
+panchro.xyz
-msfsaar.de
+partybunny.ru
-mygameplus.com
+pearlisland.ru
-newsperuse.com
+pechikamini.ru
-nonews.co
+perfectplanned.com
-nudo.ca
+petedrummond.com
-okonich.com.ua
+picmoonco.pw
-old-rock.com
+picurams.pw
-online-podarki.com
+piski.top
-onlyforemont.ru
+piulatte.cz
-pc-virus-d0l92j2.pw
+pixgood.com
-pdns.cz
+pobeiranie.pl
-pipki.r.acdnpro.com
+polytopesexempt.com
-pivka.xyz
+popander.mobi
-pladform.ru
+popserve.adscpm.net
-pliks.pl
+pro-poly.ru
-poligon.com
+prodess.ru
-porno.simple-image.com.ua
+promalp-universal.ru
-pornogig.com
+proposal-engine.com
-predominant-invent.tk
+prostitutki-odessa.org
-preparevideosafesystem4unow.site
+pulse33.ru
-printingpeach.com
+pulseonclick.com
-prizesbook.online
+purplesphere.in
-productarium.com
+rangjued.com
-professionalwritingservices15.blogspot.ru
+redhotfreebies.co.uk
-prostitutki-almata.org
+remmling.de
-psa48.ru
+restaurantlescampi.com
-quangcaons.com
-rankings-analytics.com
-ranksays.com
-registrationdomainsite.com
-registrysweeper.com
-remybutler.fr
-rp9.ru
 rucrypt.com
-runofilms.ru
+sack.net
-rusenvironmental.net
+samo-soznanie.ru
-sankt-peterburg.nodup.ru
+savefrom.com
-sc-specialhost.com
+scanner-fred.top
-scanmarine.info
+scottbywater.com
-scanner-brian.top
+searchlock.com
-scanner-elena.top
+seksvideoonlain.com
-scanspyware.net
+seocheki.net
-school-diplomat.ru
+seorank.info
-sdsjweb.com
+sexvideo-sex.com
-search-goo.com
-search.xtconnect.com
-searchimage.co
-seorankinglinks.xyz
-servisural.ru
-sexad.net
 sexy.babes.frontend-stack.top
-shtora66.ru
+sexyteens.hol.es
-smailik.org
+sierraapps.com
-small-games.biz
+site-auditor.online
-smartbalanceworld.com
+sitebeam.net
-sms2x2.ru
+social-vestnik.ru
-soblaznu.net
+sofit-dmd.ru
-starwars.wikia.com
+soserfis.com
-stats-public.grammarly.io
+spammen.de
-stefanbakosab.se
+spasswelt.net
-stockquotes.wooeb.com
+srdrvp.com
-stopnarco.ru
+statustroll.com
-stromerrealty.com
+stjamesschool.info
-studiofmp.com
+stretchmate.net
-sukirgenk.dvrlists.com
+sucsesofinspiration.com
-superiends.org
+sukarame.net
-supernew.org
+supers.com.ua
-support.nopeas.sk
+surffoundation.nl
-talant-factory.ru
+susanholtphotography.com
-templates.franklinfire.co
+swiped.su
-thexart.club
+sygraem.com
-tizanidinedosageforsleep.blogspot.com
+tdsing.ru
-tizanidinezanaflex.blogspot.com
+tecnoteakviareggio.it
-tourcroatia.co.uk
+texbaza.by
-traderzplanet.in
+theautoprofit.ml
-trafficgenius.xyz
+thin.me.pn
-tula.howotorg.ru
+tizanidinehcl4mgtablet.blogspot.com
-tzritel.tk
+topanasex.com
-udsgame.online
+traffique.net
-uhodzalijami.ru
+ukrup.com
-ukkelberg.no
+umnovocaminho.com
-unimodemhalfduplefw.pen.io
+unmaroll.ya.ru
-univerfiles.com
+ussearche.cf
-vacances-voyages.info
+uytmaster.ru
-velen.io
+validccseller.com
-viagraneggrx.com
+valmetrundan.se
-viel.su
+varbergsvind.se
-visa-pasport.ru
+via-energy-cumpara.com
-vitalads.net
+videochat.mx
-vpnmouse.com
+vonradio.com
-vysigy.su
+vriel.batcave.net
-vzlomtw.com
+vtcdns.com
-wallpaperaccess.com
+vykup-avto-krasnodar.ru
-wapsite.me
+webkeyit.com
-watch-movies.ru
+website-speed-checker.site
-web-analytics.date
+wechatdownload10.com
-webscouter.net
+welck.octopis.com
-weddingdresses.xyz
+westum.se
-wingsoffury2.com
+wufak.com
-winx-play.ru
+xblognetwork.com
-wpsecurity.website
+xn--80atua3d.xn--p1ai
-xaylapdiendanang.com
+xn--b1adccaf1bzj.xn--p1ai
-xn----ctbbcjd3dbsehgi.xn--p1ai
+yellowproxy.net
-xn--80aeahghtf8ac5i.xn--p1ai
-xxxmania.top
 yellowstonesafaritours.com
-yellowstonevisitortours.com
+za-fun-offer.com
-ynymnwbm.bloger.index.hr
+zakon-ob-obrazovanii.ru
-yoopsie.com
+zambini.ru
-youbloodyripper.com
+zarenica.net
-youtube-downloader.savetubevideo.com
+zerocash.msk.ru
-zaidia.xhost.ro
+zhacker.net
-zdesformula.ru
+zkjovpdgxivg.ga
-zeroredirect12.com
-zoompegs.com

dev-tools/test_units/ratelimittest-master.txt

@@ -581,4 +581,14 @@
 <h1>Welcome to the NginX Ultimate Bad Bot Blocker</h1>
 <p>This is merely used for testing !!!</p>
 </body>
+</html><!DOCTYPE html>
+<html>
+<head>
+<title>Welcome to NginX</title>
+</head>
+<body>
+<div align=center>
+<h1>Welcome to the NginX Ultimate Bad Bot Blocker</h1>
+<p>This is merely used for testing !!!</p>
+</body>
 </html>

dev-tools/test_units/whitelist-ips.conf

@@ -1,2 +1,2 @@
-10.1.0.150	0;
+10.1.0.235	0;
 127.0.0.1	0;

dev-tools/tmp/robots.txt

@@ -6,9 +6,9 @@
 
 ### Version Information #
 ###################################################
-### Version: V4.2022.01.2866
+### Version: V4.2022.01.2867
-### Updated: Thu Jan 20 22:01:41 UTC 2022
+### Updated: Fri Jan 21 08:34:09 UTC 2022
-### Bad Bot Count: 633
+### Bad Bot Count: 636
 ###################################################
 ### Version Information ##
 
@@ -73,6 +73,8 @@ User-agent: Aspiegel
 Disallow:/
 User-agent: Asterias
 Disallow:/
+User-agent: Atomseobot
+Disallow:/
 User-agent: Attach
 Disallow:/
 User-agent: AwarioRssBot
@@ -389,6 +391,8 @@ User-agent: Heritrix
 Disallow:/
 User-agent: Hloader
 Disallow:/
+User-agent: HonoluluBot
+Disallow:/
 User-agent: Humanlinks
 Disallow:/
 User-agent: HybridBot
@@ -587,6 +591,8 @@ User-agent: Mojeek
 Disallow:/
 User-agent: Mojolicious
 Disallow:/
+User-agent: MolokaiBot
+Disallow:/
 User-agent: Morfeus Fucking Scanner
 Disallow:/
 User-agent: Mozlila

robots.txt/robots.txt

@@ -6,9 +6,9 @@
 
 ### Version Information #
 ###################################################
-### Version: V4.2022.01.2866
+### Version: V4.2022.01.2867
-### Updated: Thu Jan 20 22:01:41 UTC 2022
+### Updated: Fri Jan 21 08:34:09 UTC 2022
-### Bad Bot Count: 633
+### Bad Bot Count: 636
 ###################################################
 ### Version Information ##
 
@@ -73,6 +73,8 @@ User-agent: Aspiegel
 Disallow:/
 User-agent: Asterias
 Disallow:/
+User-agent: Atomseobot
+Disallow:/
 User-agent: Attach
 Disallow:/
 User-agent: AwarioRssBot
@@ -389,6 +391,8 @@ User-agent: Heritrix
 Disallow:/
 User-agent: Hloader
 Disallow:/
+User-agent: HonoluluBot
+Disallow:/
 User-agent: Humanlinks
 Disallow:/
 User-agent: HybridBot
@@ -587,6 +591,8 @@ User-agent: Mojeek
 Disallow:/
 User-agent: Mojolicious
 Disallow:/
+User-agent: MolokaiBot
+Disallow:/
 User-agent: Morfeus Fucking Scanner
 Disallow:/
 User-agent: Mozlila