vikarti.anatra/nginx-ultimate-bad-bot-blocker
1
0
Fork 0
mirror of https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git synced 2025-09-02 02:29:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

V4.2019.06.1598 [ci skip]

Browse source
This commit is contained in:
Travis 2019-06-25 15:46:11 +02:00
parent cee1e20e59
commit 1518960313
22 changed files with 1758 additions and 1758 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf
View file

@ -33,4 +33,4 @@
# 111.111.111.111 0; # 111.111.111.111 0;
35.193.7.13 0; 35.184.96.71 0;

114
.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf
View file

@ -4,8 +4,8 @@
### VERSION INFORMATION # ### VERSION INFORMATION #
################################################### ###################################################
### Version: V4.2019.06.1597 ### Version: V4.2019.06.1598
### Updated: Tue Jun 25 15:25:42 SAST 2019 ### Updated: Tue Jun 25 15:42:43 SAST 2019
### Bad Referrer Count: 6711 ### Bad Referrer Count: 6711
### Bad Bot Count: 556 ### Bad Bot Count: 556
################################################### ###################################################
@ -693,38 +693,38 @@ map $http_user_agent $bad_bot {
# *********************************************** # ***********************************************
# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###
"~*\badidxbot\b" 0; "~*(?:\b)adidxbot(?:\b|)" 0;
"~*\bAdsBot-Google\b" 0; "~*(?:\b)AdsBot-Google(?:\b|)" 0;
"~*\baolbuild\b" 0; "~*(?:\b)aolbuild(?:\b|)" 0;
"~*\bbingbot\b" 0; "~*(?:\b)bingbot(?:\b|)" 0;
"~*\bbingpreview\b" 0; "~*(?:\b)bingpreview(?:\b|)" 0;
"~*\bDoCoMo\b" 0; "~*(?:\b)DoCoMo(?:\b|)" 0;
"~*\bduckduckgo\b" 0; "~*(?:\b)duckduckgo(?:\b|)" 0;
"~*\bfacebookexternalhit\b" 0; "~*(?:\b)facebookexternalhit(?:\b|)" 0;
"~*\bFeedfetcher-Google\b" 0; "~*(?:\b)Feedfetcher-Google(?:\b|)" 0;
"~*\bGooglebot\b" 0; "~*(?:\b)Googlebot(?:\b|)" 0;
"~*\bGooglebot-Image\b" 0; "~*(?:\b)Googlebot-Image(?:\b|)" 0;
"~*\bGooglebot-Mobile\b" 0; "~*(?:\b)Googlebot-Mobile(?:\b|)" 0;
"~*\bGooglebot-News\b" 0; "~*(?:\b)Googlebot-News(?:\b|)" 0;
"~*\bGooglebot/Test\b" 0; "~*(?:\b)Googlebot/Test(?:\b|)" 0;
"~*\bGooglebot-Video\b" 0; "~*(?:\b)Googlebot-Video(?:\b|)" 0;
"~*\bGoogle-HTTP-Java-Client\b" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0;
"~*\bGravityscan\b" 0; "~*(?:\b)Gravityscan(?:\b|)" 0;
"~*\bgsa-crawler\b" 0; "~*(?:\b)gsa-crawler(?:\b|)" 0;
"~*\bJakarta\ Commons\b" 0; "~*(?:\b)Jakarta\ Commons(?:\b|)" 0;
"~*\bKraken/0.1\b" 0; "~*(?:\b)Kraken/0.1(?:\b|)" 0;
"~*\bLinkedInBot\b" 0; "~*(?:\b)LinkedInBot(?:\b|)" 0;
"~*\bMediapartners-Google\b" 0; "~*(?:\b)Mediapartners-Google(?:\b|)" 0;
"~*\bmsnbot\b" 0; "~*(?:\b)msnbot(?:\b|)" 0;
"~*\bmsnbot-media\b" 0; "~*(?:\b)msnbot-media(?:\b|)" 0;
"~*\bSAMSUNG\b" 0; "~*(?:\b)SAMSUNG(?:\b|)" 0;
"~*\bSlackbot\b" 0; "~*(?:\b)Slackbot(?:\b|)" 0;
"~*\bSlackbot-LinkExpanding\b" 0; "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0;
"~*\bslurp\b" 0; "~*(?:\b)slurp(?:\b|)" 0;
"~*\bteoma\b" 0; "~*(?:\b)teoma(?:\b|)" 0;
"~*\bTwitterBot\b" 0; "~*(?:\b)TwitterBot(?:\b|)" 0;
"~*\bWordpress\b" 0; "~*(?:\b)Wordpress(?:\b|)" 0;
"~*\byahoo\b" 0; "~*(?:\b)yahoo(?:\b|)" 0;
# END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###
# *************************************************** # ***************************************************
@ -735,12 +735,12 @@ map $http_user_agent $bad_bot {
# I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited
# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
"~*\bjetmon\b" 1; "~*(?:\b)jetmon(?:\b|)" 1;
"~*\blibwww-perl\b" 1; "~*(?:\b)libwww-perl(?:\b|)" 1;
"~*\bLynx\b" 1; "~*(?:\b)Lynx(?:\b|)" 1;
"~*\bmunin\b" 1; "~*(?:\b)munin(?:\b|)" 1;
"~*\bPresto\b" 1; "~*(?:\b)Presto(?:\b|)" 1;
"~*\bWget/1.15\b" 1; "~*(?:\b)Wget/1.15(?:\b|)" 1;
# END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
# ************************************************************** # **************************************************************
@ -748,23 +748,23 @@ map $http_user_agent $bad_bot {
# ************************************************************** # **************************************************************
# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
"~*\bAlexa\b" 2; "~*(?:\b)Alexa(?:\b|)" 2;
"~*\barchive.org\b" 2; "~*(?:\b)archive.org(?:\b|)" 2;
"~*\bBaidu\b" 2; "~*(?:\b)Baidu(?:\b|)" 2;
"~*\bBUbiNG\b" 2; "~*(?:\b)BUbiNG(?:\b|)" 2;
"~*\bFlipboardProxy\b" 2; "~*(?:\b)FlipboardProxy(?:\b|)" 2;
"~*\bia_archiver\b" 2; "~*(?:\b)ia_archiver(?:\b|)" 2;
"~*\bMSIE\ 7.0\b" 2; "~*(?:\b)MSIE\ 7.0(?:\b|)" 2;
"~*\bProximic\b" 2; "~*(?:\b)Proximic(?:\b|)" 2;
"~*\bR6_CommentReader\b" 2; "~*(?:\b)R6_CommentReader(?:\b|)" 2;
"~*\bR6_FeedFetcher\b" 2; "~*(?:\b)R6_FeedFetcher(?:\b|)" 2;
"~*\bRED/1\b" 2; "~*(?:\b)RED/1(?:\b|)" 2;
"~*\bRPT-HTTPClient\b" 2; "~*(?:\b)RPT-HTTPClient(?:\b|)" 2;
"~*\bsfFeedReader/0.9\b" 2; "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2;
"~*\bSpaidu\b" 2; "~*(?:\b)Spaidu(?:\b|)" 2;
"~*\bUptimeRobot/2.0\b" 2; "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2;
"~*\bYandexBot\b" 2; "~*(?:\b)YandexBot(?:\b|)" 2;
"~*\bYandexImages\b" 2; "~*(?:\b)YandexImages(?:\b|)" 2;
# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
} }

2
.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf
View file

@ -33,4 +33,4 @@
# 111.111.111.111 0; # 111.111.111.111 0;
35.193.7.13 0; 35.184.96.71 0;

4
.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf
View file

@ -10,8 +10,8 @@
### VERSION INFORMATION # ### VERSION INFORMATION #
################################################### ###################################################
### Version: V3.2019.06.1597 ### Version: V3.2019.06.1598
### Updated: Tue Jun 25 15:25:43 SAST 2019 ### Updated: Tue Jun 25 15:42:44 SAST 2019
### Bad Referrer Count: 6711 ### Bad Referrer Count: 6711
### Bad Bot Count: 556 ### Bad Bot Count: 556
################################################### ###################################################

4
.dev-tools/globalblacklist-testing.template
View file

@ -10,8 +10,8 @@
### VERSION INFORMATION # ### VERSION INFORMATION #
################################################### ###################################################
### Version: V3.2019.06.1597 ### Version: V3.2019.06.1598
### Updated: Tue Jun 25 15:25:43 SAST 2019 ### Updated: Tue Jun 25 15:42:44 SAST 2019
### Bad Referrer Count: 6711 ### Bad Referrer Count: 6711
### Bad Bot Count: 556 ### Bad Bot Count: 556
################################################### ###################################################

114
.dev-tools/globalblacklist.template
View file

@ -4,8 +4,8 @@
### VERSION INFORMATION # ### VERSION INFORMATION #
################################################### ###################################################
### Version: V4.2019.06.1597 ### Version: V4.2019.06.1598
### Updated: Tue Jun 25 15:25:42 SAST 2019 ### Updated: Tue Jun 25 15:42:43 SAST 2019
### Bad Referrer Count: 6711 ### Bad Referrer Count: 6711
### Bad Bot Count: 556 ### Bad Bot Count: 556
################################################### ###################################################
@ -693,38 +693,38 @@ map $http_user_agent $bad_bot {
# *********************************************** # ***********************************************
# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###
"~*\badidxbot\b" 0; "~*(?:\b)adidxbot(?:\b|)" 0;
"~*\bAdsBot-Google\b" 0; "~*(?:\b)AdsBot-Google(?:\b|)" 0;
"~*\baolbuild\b" 0; "~*(?:\b)aolbuild(?:\b|)" 0;
"~*\bbingbot\b" 0; "~*(?:\b)bingbot(?:\b|)" 0;
"~*\bbingpreview\b" 0; "~*(?:\b)bingpreview(?:\b|)" 0;
"~*\bDoCoMo\b" 0; "~*(?:\b)DoCoMo(?:\b|)" 0;
"~*\bduckduckgo\b" 0; "~*(?:\b)duckduckgo(?:\b|)" 0;
"~*\bfacebookexternalhit\b" 0; "~*(?:\b)facebookexternalhit(?:\b|)" 0;
"~*\bFeedfetcher-Google\b" 0; "~*(?:\b)Feedfetcher-Google(?:\b|)" 0;
"~*\bGooglebot\b" 0; "~*(?:\b)Googlebot(?:\b|)" 0;
"~*\bGooglebot-Image\b" 0; "~*(?:\b)Googlebot-Image(?:\b|)" 0;
"~*\bGooglebot-Mobile\b" 0; "~*(?:\b)Googlebot-Mobile(?:\b|)" 0;
"~*\bGooglebot-News\b" 0; "~*(?:\b)Googlebot-News(?:\b|)" 0;
"~*\bGooglebot/Test\b" 0; "~*(?:\b)Googlebot/Test(?:\b|)" 0;
"~*\bGooglebot-Video\b" 0; "~*(?:\b)Googlebot-Video(?:\b|)" 0;
"~*\bGoogle-HTTP-Java-Client\b" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0;
"~*\bGravityscan\b" 0; "~*(?:\b)Gravityscan(?:\b|)" 0;
"~*\bgsa-crawler\b" 0; "~*(?:\b)gsa-crawler(?:\b|)" 0;
"~*\bJakarta\ Commons\b" 0; "~*(?:\b)Jakarta\ Commons(?:\b|)" 0;
"~*\bKraken/0.1\b" 0; "~*(?:\b)Kraken/0.1(?:\b|)" 0;
"~*\bLinkedInBot\b" 0; "~*(?:\b)LinkedInBot(?:\b|)" 0;
"~*\bMediapartners-Google\b" 0; "~*(?:\b)Mediapartners-Google(?:\b|)" 0;
"~*\bmsnbot\b" 0; "~*(?:\b)msnbot(?:\b|)" 0;
"~*\bmsnbot-media\b" 0; "~*(?:\b)msnbot-media(?:\b|)" 0;
"~*\bSAMSUNG\b" 0; "~*(?:\b)SAMSUNG(?:\b|)" 0;
"~*\bSlackbot\b" 0; "~*(?:\b)Slackbot(?:\b|)" 0;
"~*\bSlackbot-LinkExpanding\b" 0; "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0;
"~*\bslurp\b" 0; "~*(?:\b)slurp(?:\b|)" 0;
"~*\bteoma\b" 0; "~*(?:\b)teoma(?:\b|)" 0;
"~*\bTwitterBot\b" 0; "~*(?:\b)TwitterBot(?:\b|)" 0;
"~*\bWordpress\b" 0; "~*(?:\b)Wordpress(?:\b|)" 0;
"~*\byahoo\b" 0; "~*(?:\b)yahoo(?:\b|)" 0;
# END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###
# *************************************************** # ***************************************************
@ -735,12 +735,12 @@ map $http_user_agent $bad_bot {
# I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited
# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
"~*\bjetmon\b" 1; "~*(?:\b)jetmon(?:\b|)" 1;
"~*\blibwww-perl\b" 1; "~*(?:\b)libwww-perl(?:\b|)" 1;
"~*\bLynx\b" 1; "~*(?:\b)Lynx(?:\b|)" 1;
"~*\bmunin\b" 1; "~*(?:\b)munin(?:\b|)" 1;
"~*\bPresto\b" 1; "~*(?:\b)Presto(?:\b|)" 1;
"~*\bWget/1.15\b" 1; "~*(?:\b)Wget/1.15(?:\b|)" 1;
# END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
# ************************************************************** # **************************************************************
@ -748,23 +748,23 @@ map $http_user_agent $bad_bot {
# ************************************************************** # **************************************************************
# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
"~*\bAlexa\b" 2; "~*(?:\b)Alexa(?:\b|)" 2;
"~*\barchive.org\b" 2; "~*(?:\b)archive.org(?:\b|)" 2;
"~*\bBaidu\b" 2; "~*(?:\b)Baidu(?:\b|)" 2;
"~*\bBUbiNG\b" 2; "~*(?:\b)BUbiNG(?:\b|)" 2;
"~*\bFlipboardProxy\b" 2; "~*(?:\b)FlipboardProxy(?:\b|)" 2;
"~*\bia_archiver\b" 2; "~*(?:\b)ia_archiver(?:\b|)" 2;
"~*\bMSIE\ 7.0\b" 2; "~*(?:\b)MSIE\ 7.0(?:\b|)" 2;
"~*\bProximic\b" 2; "~*(?:\b)Proximic(?:\b|)" 2;
"~*\bR6_CommentReader\b" 2; "~*(?:\b)R6_CommentReader(?:\b|)" 2;
"~*\bR6_FeedFetcher\b" 2; "~*(?:\b)R6_FeedFetcher(?:\b|)" 2;
"~*\bRED/1\b" 2; "~*(?:\b)RED/1(?:\b|)" 2;
"~*\bRPT-HTTPClient\b" 2; "~*(?:\b)RPT-HTTPClient(?:\b|)" 2;
"~*\bsfFeedReader/0.9\b" 2; "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2;
"~*\bSpaidu\b" 2; "~*(?:\b)Spaidu(?:\b|)" 2;
"~*\bUptimeRobot/2.0\b" 2; "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2;
"~*\bYandexBot\b" 2; "~*(?:\b)YandexBot(?:\b|)" 2;
"~*\bYandexImages\b" 2; "~*(?:\b)YandexImages(?:\b|)" 2;
# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
} }

96
.dev-tools/test_units/random-bots-for-test-quick.list
View file

@ -1,50 +1,50 @@
360Spider Aboundex
ADmantX AiHitBot
Asterias BetaBot
Bigfoot BlackWidow
BLEXBot Brandwatch
Blow Discoverybot
Bullseye DnyzBot
ChinaClaw EasyDL
Chlooe Ecxi
coccocbot-web EirGrabber
cognitiveseo Flunky
DataCha0s Fyrebot
DomainCrawler GridBot
ExtLinksBot
Extreme Picture Finder
HTTP::Lite
InfoTekies
InternetSeer
internetVista monitor
JennyBot JennyBot
JustView JetCar
Kenjin Spider JikeSpider
Lanshanbot Joomla
Likse Jyxobot
LinqiaMetadataDownloaderBot Keyword Density
Lipperhey LibWeb
Mail.RU_Bot lwp-trivial
Metauri Magnet
Name Intelligence Mata Hari
Nameprotect MauiBot
netEstate NE Crawler Microsoft Data Access
NetZIP Microsoft URL Control
PeoplePal Nmap
Picsearch OpenVAS
PxBroker RankurBot
RealDownload SEOlyticsCrawler
Screaming seoscanners
SearchmetricsBot SEOstats
SocialRankIOBot sexsearcher
Sosospider Siteimprove
sysscan Sucker
TightTwatBot tAkeOut
Voltron Teleport
WallpapersHD Trendictionbot
WebAuto Turingos
Web Auto VCI
Web Enhancer Webalta
Web Pix WebCopier
Web Sucker WebmasterWorldForumBot
Webshag
Webster
WWW-Mechanize
Xenu Xenu
Zauba
Zeus
ZmEu

290
.dev-tools/test_units/random-bots-for-test.list
View file

@ -1,250 +1,250 @@
360Spider
404checker 404checker
404enemy 404enemy
80legs 80legs
Abonti Aboundex
Aboundexbot Aboundexbot
AfD-Verbotsverfahren AfD-Verbotsverfahren
AIBOT AIBOT
AiHitBot AiHitBot
Aipbot
Alligator Alligator
AllSubmitter
AlphaBot
Apexoo
archive.org_bot
ASPSeek ASPSeek
Asterias
autoemailspider
Backlink-Ceck
backlink-check backlink-check
BacklinkCrawler BacklinkCrawler
BackStreet BackWeb
BatchFTP Badass
BBBike Bandit
Blackboard Barkrowler
Black Hole Battleztar Bazinga
BDCbot
BetaBot
Bitacle
BlackWidow BlackWidow
BLEXBot BlowFish
Bolt Boardreader
Brandprotect BotALot
Buddy Brandwatch
BunnySlippers BuiltWith
CazoodleBot Cegbfeieh
CheeseBot CheeseBot
CherryPicker
CheTeam
ChinaClaw ChinaClaw
Chlooe Cloud mapping
Claritybot coccocbot-web
Cogentbot Cogentbot
CopyRightCheck cognitiveseo
Copyscape com.plumanalytics
Copier
crawler4j
crawler.feedback
crawl.sogou.com
Crescent Crescent
Custo DBLBot
DatabaseDriverMysqli demandbase-bot
Demon
Deusu Deusu
DigitalPebble Devil
Dirbuster Digincore
Disco DIIbot
DnyzBot Discoverybot
DomainCrawler
DomainSigmaCrawler
DomainStatsBot
Dragonfly Dragonfly
EasyDL
eCatch eCatch
ECCP/1.0 Ecxi
EMail Siphon EroCrawler
EMail Wolf evc-batch
Evil Evil
Express WebPictures
ExtLinksBot ExtLinksBot
Extractor Extractor
ExtractorPro
EyeNetIE EyeNetIE
facebookscraper Ezooms
FDM GalaxyBot
FemtosearchBot GermCrawler
FlashGet Getintent
FrontPage GetRight
Genieo GetWeb
GoZilla Gigablast
Gigabot
Go-Ahead-Got-It
Go!Zilla Go!Zilla
Grabber Grafula
GrapeFX GrapeFX
GridBot GT::WWW
HaosouSpider HaosouSpider
Harvest
Havij
Heritrix Heritrix
Hloader Hloader
HMView
HTMLparser
HTTP::Lite HTTP::Lite
HTTrack HTTrack
Humanlinks Humanlinks
HybridBot
IDBot IDBot
Id-search
IlseBot
Image Fetch
Image Sucker Image Sucker
Intelliseek Indy Library
InterGET InfoNaviRobot
internetVista monitor InfoTekies
Iria InternetSeer
Iskanie ips-agent
JennyBot JennyBot
Joomla Joomla
Jyxobot Jyxobot
Kenjin Spider Kozmosbot
Lanshanbot Larbin
Lftp LeechFTP
LinkWalker Libwhisker
Lightspeedsystems
LinkextractorPro
LinqiaMetadataDownloaderBot
LinqiaRSSBot LinqiaRSSBot
Lipperhey LinqiaScrapeBot
Lipperhey Spider
Litemage_walker Litemage_walker
Lmspider Lmspider
LWP::Simple Ltx71
Mag-Net Masscan
magpie-crawler Mass Downloader
Majestic12 meanpathbot
MarkMonitor Meanpathbot
MarkWatch
MauiBot
MeanPath Bot
Mediatoolkitbot Mediatoolkitbot
mediawords
MegaIndex.ru MegaIndex.ru
Metauri
Microsoft Data Access Microsoft Data Access
Microsoft URL Control
MIDown tool MIDown tool
MIIxpc MIIxpc
Mister PiX Morfeus Fucking Scanner
Mr.4x3 MSIECrawler
MSFrontPage
Msrabot
MS Web Services Client Protocol MS Web Services Client Protocol
Name Intelligence Musobot
Nameprotect Navroad
NearSite NearSite
Needle
Nessus Nessus
Netcraft Netcraft
NetSpider NetMechanic
Nettrack
Net Vampire
Netvibes Netvibes
NetZIP
Nibbler
NICErsPRO
Niki-bot
Nikto
NimbleCrawler NimbleCrawler
Nimbostratus
Nmap Nmap
Octopus NPbot
Offline Explorer Offline Navigator
OnCrawl OnCrawl
Openfind
OpenLinkProfiler
Openvas Openvas
OpenVAS OpenVAS
OrangeSpider OutfoxBot
OutclicksBot PageAnalyzer
PageGrabber
page scorer page scorer
PageScorer
Pandalytics
Panscient Panscient
pcBrowser pcBrowser
PECL::HTTP PeoplePal
PHPCrawl Picsearch
PictureFinder
Pi-Monster Pi-Monster
PleaseCrawl
plumanalytics plumanalytics
Pockey Pockey
POE-Component-Client-HTTP
Probethenet Probethenet
ProWebWalker ProPowerBot
Psbot
Pump Pump
PyCurl PyCurl
QueryN Metasearch Quick-Crawler
RankActive RankActiveLinkBot
Rankivabot Rankivabot
RankurBot RankurBot
RealDownload Reaper
RebelMouse RebelMouse
ReGet RocketCrawler
Ripper s1z.ru
Rogerbot SalesIntelligent
ScanAlert SBIder
Scanbot Scanbot
ScoutJet
Scrapy Scrapy
Screaming ScreenerBot
Searchestate Semrush
SEOkicks SEOkicks
SEOkicks-Robot SEOlyticsCrawler
SEOprofiler SEOprofiler
seoscanners
SeoSiteCheckup
SEOstats SEOstats
Shodan serpstatbot
Siphon Siphon
Sitebeam
SiteExplorer
Siteimprove
SiteLockSpider SiteLockSpider
SiteSnagger SiteSucker
Site Sucker
SlySearch SlySearch
SmartDownload SmartDownload
Snake Snake
Snoopy Sogou web spider
SocialRankIOBot
Sociscraper
sogouspider
Sosospider Sosospider
Sottopop Sottopop
SpaceBison Spammen
SpankBot SpankBot
Spanner Spanner
sp_auditbot sp_auditbot
SputnikBot Spinn3r
spyfu spyfu
Sqlworm
Sqworm Sqworm
Steeler Stripper
Sucker
Sucuri Sucuri
SuperBot SuperBot
SuperHTTP SuperHTTP
Surfbot Swiftbot
sysscan T0PHackTeam
Teleport tAkeOut
TeleportPro TeleportPro
Telesoft
The Intraformant
TheNomad TheNomad
Thumbor
TightTwatBot TightTwatBot
trendiction.com Titan
trendiction.de Trendiction
Turnitin Turingos
TwengaBot TwengaBot
Upflow Upflow
URLy Warning
URLy.Warning URLy.Warning
Vagabondo Vagabondo
VB Project VB Project
VCI VidibleScraper
Virusdie WallpapersHD
Wallpapers/3.0 WASALive-Bot
Webalta WBSearchBot
WebAuto WebCollage
WebBandit WEBDAV
WebEnhancer
WebFuck WebFuck
WebImageCollector WebLeacher
WebPix WebPix
Web Pix
WebReaper
WebSauger WebSauger
Web Sauger Web Sauger
WebsiteExtractor
WebsiteQuester WebsiteQuester
Website Quester Website Quester
WebStripper Webster
WebSucker WebSucker
Whatweb Web Sucker
Who.is Bot WebZIP
Widow Whack
Woobot WinHTTrack
WiseGuys Robot
Wotbox Wotbox
WWW-Mechanize Wprecon
WWW::Mechanize WWW-Collector-E
x09Mozilla WWWOFFLE
Xenu Xaldon_WebSpider
zauba.io xpymep1.exe
Zade
Zauba
Zermelo
Zeus
ZmEu ZmEu
ZumBot ZumBot
ZyBorg ZyBorg

280
.dev-tools/test_units/random-bots-for-whitelist-test.list
View file

@ -1,250 +1,250 @@
360Spider
80legs 80legs
Abonti
Aboundex Aboundex
Acunetix Aboundexbot
AhrefsBot AfD-Verbotsverfahren
AIBOT AIBOT
Anarchie Alligator
Attach AllSubmitter
AlphaBot
autoemailspider autoemailspider
BackDoorBot BackDoorBot
Backlink-Ceck
backlink-check
BacklinkCrawler BacklinkCrawler
Badass BackWeb
Bandit
Barkrowler
BatchFTP BatchFTP
Battleztar Bazinga Battleztar Bazinga
BBBike BlackWidow
Blackboard
BLEXBot BLEXBot
BlowFish BlowFish
Boardreader Boardreader
BotALot Bolt
Brandwatch Brandprotect
BuiltBotTough BuiltBotTough
Bullseye Bullseye
BunnySlippers BunnySlippers
BuzzSumo BuzzSumo
Calculon
CATExplorador CATExplorador
CazoodleBot CazoodleBot
CCBot
Cegbfeieh Cegbfeieh
Chlooe CheeseBot
CheTeam
ChinaClaw
Claritybot
Cliqzbot Cliqzbot
coccocbot-web
cognitiveseo cognitiveseo
Collector CopyRightCheck
com.plumanalytics Cosmos
crawler4j
crawler.feedback crawler.feedback
CrazyWebCrawler CrazyWebCrawler
Crescent Crescent
CrunchBot CrunchBot
Curious
Custo Custo
DataCha0s DatabaseDriverMysqli
DBLBot
demandbase-bot demandbase-bot
Demon Demon
Digincore Deusu
DigitalPebble DIIbot
Dirbuster Dirbuster
Discoverybot
Dispatch
DittoSpyder DittoSpyder
DomainAppender DomainAppender
DomainSigmaCrawler DomainSigmaCrawler
DomainStatsBot
Dotbot
Dragonfly
Drip
Ebingbong
eCatch eCatch
EirGrabber EirGrabber
EMail Wolf FemtosearchBot
EroCrawler FHscan
evc-batch
Evil
ExtLinksBot
Extractor
ExtractorPro
EyeNetIE
facebookscraper
FDM
Fimap Fimap
FlashGet Flunky
FrontPage FrontPage
FyberSpider
Fyrebot
GalaxyBot
Genieo Genieo
Gigablast GermCrawler
Gigabot Getintent
GetRight
G-i-g-a-b-o-t G-i-g-a-b-o-t
Go!Zilla Go!Zilla
Grabber GrapeFX
GrabNet Haansoft
Grafula Harvest
HaosouSpider HMView
HEADMasterSEO HTTrack
Heritrix
HTTP::Lite
HybridBot HybridBot
Iblog
Id-search Id-search
IlseBot
Image Fetch Image Fetch
instabid IndeedBot
InfoTekies
Intelliseek Intelliseek
JamesBOT InternetSeer
JetCar internetVista monitor
IRLbot
Jetty Jetty
JikeSpider
JOC Web Spider JOC Web Spider
Keyword Density Kenjin Spider
Kozmosbot
Lanshanbot Lanshanbot
Lftp LeechFTP
LeechGet
LexiBot
LibWeb LibWeb
Likse Lightspeedsystems
Linkdexbot
LinkextractorPro
LinkpadBot LinkpadBot
LinksManager LinksManager
LinqiaMetadataDownloaderBot
LinqiaRSSBot
LinqiaScrapeBot LinqiaScrapeBot
Lipperhey Litemage_walker
Lmspider Lmspider
lwp-request lwp-request
LWP::Simple
Magnet Magnet
Mag-Net magpie-crawler
Majestic12 Mail.RU_Bot
Majestic SEO
MarkMonitor MarkMonitor
Masscan MarkWatch
Mass Downloader Mass Downloader
Mata Hari Mata Hari
MauiBot MauiBot
Meanpathbot
Mediatoolkitbot
mediawords
Metauri
MFC_Tear_Sample MFC_Tear_Sample
Microsoft Data Access MIDown tool
Mojeek Mister PiX
Morfeus Fucking Scanner
MSIECrawler MSIECrawler
Msrabot Musobot
muhstik-scan Name Intelligence
Navroad Nameprotect
Nessus
Netcraft
netEstate NE Crawler netEstate NE Crawler
NetLyzer Nettrack
NetMechanic
NextGenSearchBot NextGenSearchBot
Niki-bot NICErsPRO
Nikto
Nimbostratus
Ninja Ninja
Nmap
NPbot
Nutch Nutch
Octopus oBot
Offline Explorer Offline Explorer
Offline Navigator Openfind
OpenLinkProfiler
OpenVAS OpenVAS
OrangeBot OrangeBot
OutfoxBot OrangeSpider
PageAnalyzer OutclicksBot
PageGrabber PageGrabber
page scorer
Pandalytics Pandalytics
pcBrowser Panscient
Pavuk
PeoplePal PeoplePal
Picscout PictureFinder
Picsearch
Pimonster Pimonster
Pi-Monster
PleaseCrawl PleaseCrawl
plumanalytics
Pockey
POE-Component-Client-HTTP POE-Component-Client-HTTP
ProPowerBot Probethenet
Psbot
Pump Pump
PxBroker
PyCurl
RankActive
RankActiveLinkBot
RankingBot
RankingBot2
RealDownload RealDownload
RebelMouse Reaper
Recorder
RedesScrapy RedesScrapy
Ripper ReGet
RepoMonkey
RocketCrawler
Rogerbot
s1z.ru s1z.ru
SBIder Scrapy
SEOkicks Screaming
SEOkicks-Robot ScreenerBot
Searchestate
SearchmetricsBot
Semrush
SEOlyticsCrawler
SEOprofiler SEOprofiler
seoscanners
SeoSiteCheckup SeoSiteCheckup
SEOstats
serpstatbot
sexsearcher sexsearcher
Siphon SISTRIX
Sitebeam
SiteExplorer
Siteimprove
SiteSnagger SiteSnagger
SiteSucker SiteSucker
Site Sucker Site Sucker
SlySearch SocialRankIOBot
SMTBot Sociscraper
Snapbot
Snoopy
sogouspider sogouspider
Sosospider Sogou web spider
Sottopop Sottopop
SpaceBison SpaceBison
Spammen Spammen
Spanner
sp_auditbot sp_auditbot
Spbot Spinn3r
SputnikBot spyfu
Sqlworm Stripper
Steeler
Sucker
Sucuri
SuperBot SuperBot
SuperHTTP SurveyBot
sysscan sysscan
T0PHackTeam tAkeOut
Telesoft Teleport
Telesphoreo TeleportPro
Telesphorep The Intraformant
Toata TheNomad
Thumbor
TightTwatBot
Titan
Toweyabot Toweyabot
Tracemyfile Tracemyfile
Trendiction
Trendictionbot Trendictionbot
trendiction.com Turingos
trendiction.de
True_Robot
Turnitin
TwengaBot TwengaBot
Twice
UnisterBot
Upflow Upflow
URLy Warning
URLy.Warning URLy.Warning
Vacuum
Vagabondo Vagabondo
VCI VB Project
Virusdie VeriCiteCrawler
Voil VidibleScraper
WallpapersHD
WBSearchBot
Webalta Webalta
Web Auto
Web Collage Web Collage
WEBDAV WebCopier
WebEnhancer WebEnhancer
Web Enhancer
WebFetch WebFetch
Web Fetch Web Fetch
WebFuck
Web Fuck
WebGo IS WebGo IS
Web Pix WebReaper
WebSauger
Web Sauger Web Sauger
Webshag Website Quester
WebsiteQuester
Webster
WebStripper WebStripper
WebSucker Whack
WebWhacker
WebZIP
Whatweb Whatweb
WinHTTrack WinHTTrack
WiseGuys Robot WiseGuys Robot
WISENutbot WISENutbot
Wonderbot x09Mozilla
Wotbox Xaldon WebSpider
Wprecon
WWW::Mechanize
WWWOFFLE
xpymep1.exe xpymep1.exe
YoudaoBot
Zauba Zauba
Zermelo zauba.io
zgrab Zeus
ZumBot ZumBot

198
.dev-tools/test_units/random-referrers-for-test-quick.list
View file

@ -1,100 +1,100 @@
100searchengines.com 000free.us
3rbseyes.com 4pp13.com
activecampaign.dreamhosters.com accessoires-mode.in
adult-shop.com.ua anapa-inns.ru
airlandsea.info android-vsem.org
angry-fermi-7633.arukascloud.io aruplighting.com
apartmentratings.com autodo.info
apisecretsaucebi-a.akamaihd.net ayeartoforget.com
artefakct.com battle.net
autobrennero.it betteroffers.review
autonew.biz bizlist.com.de
bankiem.pl brandedleadgeneration.com
bannerads.de brk-rti.ru
beauty-bracelet.com cartechnic.ru
bezcmexa.ru cbox.ws
bitporno.sx cheapbarbour.online
bjanshee.ru cheapjerseysa.com
bonky.biz clips.ua.ac.be
break-the-chains.com compliance-john.top
btvn.ru cranly.net
compliance-alex.top druzhininevgeniy63.blogspot.com
compliance-josh.top esfchat.tk
conocer-sanabria.com filerockstar298.weebly.com
cpajunkies.com findclan.org
darmebel.com.ua fitfloponline.store
detsky-nabytek.info fivedwld.cf
dtnlyss.com freshwallpapers.info
ekto.ee getmiro.com
elektrozigarette1.dreamwidth.org gnuetella.com
eloconcream.blogspot.com graphics8.info
exoneration-project.us gratuitbaise.com
fitness-video.net gwebtools.com
floating-share-buttons.com hit-men.men
focalink.com hotblognetwork.com
foodcrafts.website infazavr.ru
freakycheats.com istizanidineanarcoticdrug.blogspot.com
froggytube.com jewelryandfiligree.com
gameprimary.com kabbalah-red-bracelets.com
gamewrath.com kerch.site
gayxperience.com kinobaks.com
gemgrab-a.akamaihd.net letmacworkfaster.site
girlgamerdaily.com letslowbefast.site
gkvector.ru liky.co.ua
glavtral.ru lineshops.biz
grand-chlen.ru luxmagazine.cf
greatfind-a.akamaihd.net makedo.ru
gsasearchengineranker.pw monclerjacketsoutlet.pw
hackers-crackers.tk montredemarque.nl
hanink.biz.ly
holidaypics.org
hotxnights.info
hoztorg-opt.ru
images.gyffu.com
imperia31.ru
ip-guide.com
istizanidineanopiate.blogspot.com
jav-fetish.com
kuzinsp.ru
laptop-4-less.com
likesuccess.com
mantramusic.ru
maridan.com.ua
meltwater.com
missclub.info
mobplayer.ru
myiptest.com myiptest.com
narkologiya-penza.ru myonlinepayday.co
niroo.info na-telefon.biz
pandarastore.top novodigs.com
partner-cdn.men offf.info
phpdevops.com onlinewritingjobs17.blogspot.ru
privatamateure.com o-o-8-o-o.ru
pts163.ru openmultipleurl.com
purchasepillsnorx.com orgasmatrix.com
rednise.com osnova3.ru
samo-soznanie.ru oxford-book.com.ua
scat.porn piccdata.com
security60-e.com porno.simple-image.com.ua
service.adtech.us prostitutki-odessa.org
sfd-chess.ru proxyradar.com
shoppytoolmac.com quality-traffic.com
space-worry.ml radiodigital.co
spy-sts.com razyboard.com
stuff-about-money.com refads.pro
suchenindeutschland.com sa-live.com
tattooha.com sankt-peterburg.nodup.ru
tiens2010.ru sdrescher.net
tizanidinehcl2mg.blogspot.com selectads.men
top250movies.ru seminarygeorgia59.ga
touchmods.fr seocheki.net
tubo360.com seogadget.ru
videospornogratisx.net seokicks.de
vilingstore.net sexcamamateurchat.com
warezaccess.com sharpchallenge.com
website-speed-up.site simple-image.com.ua
w-journal.ru sindragosa.comxa.com
ww1943.ru social-buttons.com
youporn-ru.com spammen.de
zeikopay.com svetlotorg.ru
zophim.me tabakur77.com
tagil.zrus.org
tech4master.com
thewebsitetemplate.info
tmearegion26.com
tuckermktg.com
ukrobstep.com
us-america.ru
vacances-voyages.info
vladimir.zrus.org
vote-up.ru
vysigy.su
wikes.20fr.com
xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai
xn--h1ahbi.com.ua
y8games-free.com
zoogdisany.com
zrus.org

1690
.dev-tools/test_units/random-referrers-for-test.list
View file

File diff suppressed because it is too large Load diff

478
.dev-tools/test_units/random-referrers-for-whitelist-test.list
View file

@ -1,250 +1,250 @@
007angels.com 007angels.com
20pascals.nl 1adult.com
24-ak.ru 2015god.org
2ads.co.uk 8lufu.com
7makemoneyonline.com 98oi.ru
aasoldes.fr aa14ab57a3339c4064bd9ae6fad7495b5f.com
academia-nsk.org aa8b68101d388c446389283820863176e7.com
acunetix-referrer.com aac52006ec82a24e08b665f4db2b5013f7.com
admitad.com aad1f4acb0a373420d9b0c4202d38d94fa.com
adultfriendfinder.com adidas.frwebs.fr
afmuseum.com ads-seo.men
agreda.pluto.ro advancedmassagebysara.com
ahrefs.com afftrack001.com
alldezservice.kz agardomains.com
all-streaming-media.com agecheckadult.com
apibetweenlinesn-a.akamaihd.net allsilver925.co.il
arate.ru ampower.me
arvut.org animebox.com.ua
autobudpostach.club anime.dougasouko.com
autotop.com.ua aquarium-pro.ru
avtointeres.ru arendatora.ru
awency.com audiofree.ru
ayanaspa.com aussie-prizes.com
bablonow.ru autochoixspinelli.com
balans.shahterworld.org avirasecureserver.com
besthomemadepornsites.com avon-severozapad.ru
bif-ru.info babyguns.ru
biznesluxe.ru baladur.ru
bloggerads.net balla-boo.se
brainboosting.club bannerspace.com
brakehawk.com best-games.today
buchananshardware.com bio-japan.net
bum.com.ru bioscorp.ru
buy-cheap-pills-order-online.com bio.trade-jp.net
cashkitten-a.akamaihd.net biteg.xyz
cctva.tv budilneg.xyz
celejihad.info buron.pw
cellfun.mobi canacopegdl.com
cheapcigarettesc.info canrole.com
chelnytruck.ru canzoni.ru
cherubinimobili.it carabela.com.do
chinacheapelitejerseys.com cartierbracelet.xyz
chlooe.com cartierlovestore.xyz
ciceros.co caulderwoodkennels.com
ci.ua cheapestjerseys-wholesale.com
clknsee.com ciekawinki.pl
club-lukojl.ru clickcash.com
colehaanoutlet.store clubfashionista.com
compliance-brian.top compliance-julianna.top
concordexoticrentals.com conboy.us
conocer-sanabria.com cool-wedding.net
cordstrap.cc cosmetic.donna7753191.ru
countercrazy.com counter.bloke.com
crd.clan.su creativehutindia.com
dealitright.click data-mining.tk
dichvuvesinhhanoi.com ddospanel.com
dlya-android.org deanmoore.ie
dnepr-avtospar.com.ua directrev.com
docstoc.com djonwatch.ru
domainsphoto.com domainroam.win
domashneeruporno.com domik-derevne.ru
donvito.unas.cz dtnlyss.com
dorratex.tn ecookna.com.ua
download-wallpaper.net ed-shop01.ru
dumpsmania24.com ekto.ee
elektrozigarette2.wordpressy.pl eng-lyrics.com
erectile.bid escort-russian.com
escortplius.com eurosamodelki.ru
esnm.ru everflora.ru
eyelike.com.ua evidencecleanergold.com
fable.in.ua exct.net
extlabs.io
familienzahnaerzte.com
farmingworm.com
fast-wordpress-start.com fast-wordpress-start.com
fillmewithhappiness.com fatmaelgarny.com
flowersbazar.com feeriaclub.ru
forum.tvmir.org filmci.pro
frameimage.org firma-legion.ru
foxweber.com
free-fbook-traffic.com free-fbook-traffic.com
freemaintenancesysforpcandmac.top free-share-buttons.top
free-traffic.xyz frequiry.com
geckoandfly.com fr.netlog.com
gg.zzyjxs.com fyxabomiw.ru
g33.org
gateway.zscalerone.net
ghazel.ru
girlporn.ru girlporn.ru
gnuetella.com gomusix.com
go2album.com goodprotein.ru
goatse.ru goodwriterssales.com
golden-praga.ru gosreg.amchs.ru
googlsucks.com grtyi.com
gotwebsite1.com gungamesz.com
gsasearchengineranker.top gz2.bbsoldes.fr
hazardky.net hannasolution.ru
hd-filmy.net hetmanship.xyz
hdimagegallery.net hmywwogw.bloger.index.hr
hidemyass.com homeart.space
hobbyhuren24.net homedecorpicture.us
hollywoodactress.info huhn.altervista.org
homemade.gq ideawheel.com
hostcritique.com iinstalll-fii1leis.jus0wil.pp.ua
hotchatdate.com imadedinner.net
hotkeys.com intelhdgraphicsgtdrive6w.metroblog.com
hoverboard360.at interferencer.ru
iccornacircri.cf iopeninghours.co.uk
igrovyeavtomaty777.ru jabimgo.pw
ilbe.club jackpotchances.com
immobiliaremassaro.com javitas.info
irkutsk.zrus.org jerseywholesalebizchina.com
iskalko.ru jikoman.info
isotoner.com jpcycles.com
istizanidineanopiate.blogspot.com jus0wil.pp.ua
jewelryandfiligree.com keywordhouse.com
johannesburgsingles.co.za kiinomaniak.pl
kam-dom.ru kino-key.info
kareliatobacco.ru konteiner24.com
kazinogames.lv koptims.tiu.ru
keywordsdoctor.com kraljeva-sutjeska.com
keywords-monitoring-success.com krasula.pp.ua
khafre.us laserpen.club
kino2018.club
kinogonew.ru
krasivoe-hd.net
labvis.host.sk
laptop-4-less.com
lecbter-relationships.ga lecbter-relationships.ga
lmrauction.com ledis.top
loginduepunti.it life-instyle.com
live-sexchat.ru
liveu.infoteka.hu
loadingpages.me
locimge.pw
lombardfinder.ru lombardfinder.ru
louboutinreplica.pw lovelycraftyhome.com
louboutinreplica.xyz mail.allnews24.in
maju.bluesalt.co manipulyator-peterburg.ru
marcoislandvacations.net manve.info
michaelkorsoutletstore.net maranbrinfo.com.br
mindbox.co.za marblestyle.ru
minecraft-rus.org mcadamssupplyco.com
missvietnam.org medispainstitute.com.au
mixx.com meet-flirt-dating.com
mmostrike.ru minneapoliscopiers.com
mobstarr.com mirtorrent.net
morlat.altervista.org moneyzzz.ru
moscow-clining.ru mosdverka.ru
moscow.online-podarki.com myblogregistercm.tk
myonlinepayday.co mypets.by
myxdate.info na-telefon.biz
nacap.ru njpalletremoval.com
narutonaruto.ru nodup.ru
net-profits.xyz novosti-hi-tech.ru
normalegal.ru oconto.ru
of-ireland.info oktube.ru
ok-ua.info o-o-6-o-o.ru
online-sbank.ru openmediasoft.com
ontargetseo.us orsonet.ru
oops-cinema.ru otbelivanie-zubov.com
pekori.to page2rss.com
pensplan4u.com paperwritingservice17.blogspot.ru
pensplan.com parlament.biz
photosaga.info pastaleads.com
piccshare.com pechikamini.ru
pizda.lol percin.biz.ly
pl.aasoldes.fr picsforkeywordsuggestion.com
poptool.net picturesboss.com
pizza-imperia.com
playlott.com
ploenjitmedia.azurewebsites.net
plohaya-kreditnaya-istoriya.ru
pl-top.pl
pontiacsolstice.info
popup-hgd.xyz
porno-play.net
positive2b.ru
preg.marketingvici.com
presleycollectibles.com
pricheski-video.com pricheski-video.com
priscilarodrigues.com.br priscilarodrigues.com.br
prodvigator.ua prohoster.info
prombudpostach.com.ua pron.pro
promodj.com proposal-engine.com
q-moto.ru pro-tec.kz
quangcaons.com pussysaga.com
raschtextil.com.ua qor360.com
real-time-analytics.com rapevideosmovies.com
reining.lovasszovetseg.hu redbottomheels.xyz
remedyotc.com redirectme.net
replicalouboutin.xyz registrydomainservices.com
ritlweb.com rfd-split.hr
ruscoininvest.company robot-forex.biz
sabizonline.com royalcar-ufa.ru
saltspray.ru rusprostitute.com
sandhillsonline.com russkoe-zdorovie.ru
sape.top sanjosestartups.com
scanner-julianna.top
scanner-mark.top
sdsjweb.com
searchkut.com
seeingmeerkat.com
sei80.com sei80.com
semalt.com seinterface.com
seoboxes.com serpstat.com
sex.hotblog.top shop-electron.ru
sexpornotales.com signx.info
sexvideo-sex.com siteexpress.co.il
shell-pmr.ru softomix.net
ship-marvel.co.ua songplanet.ru
shitmovs.com spacash.com
sildenafil-tadalafil.info spb-plitka.ru
simple-share-buttons.com sprttrack.com
s-iwantyou.com starwars.wikia.com
skincrate.net stop-gepatit.te.ua
skyway24.ru studiofaca.com
snowplanes.com sugarlyflex.pw
socialmadesimple.com survival.betteroffers.review
socialmonkee.com svensk-poesi.com
start.myplaycity.com svtrd.com
stockspmb.info telefonsexsofort.tk
strv.se titangel-vietnam.com
swingerseiten.com torrnada.ru
takethatad.com toursmaps.com
tdsing.ru trahvid.com
templates.franklinfire.co traxdom.ru
thebluffs.com typer.one
thegamerznetwork.com u17795.netangels.ru
theguardlan.com undergroundcityphoto.com
timecrimea.ru underthesite.com
tizanidinehclsideeffects.blogspot.com univerfiles.com
tizanidineoral.blogspot.com uroffer.link
tizanidinevsvalium.blogspot.com urzedowski.eu
top-l2.com varikozdok.ru
tourismvictoria.com venta-prom.ru
tozup.com vereo.eu
tpu.ru video-production.com
tracksz.co videotuber.ru
turbabitload.weebly.com
twu.com.ua
tyumen.xrus.org
validccseller.com
vegascosmetics.ru
video-chat.love
viel.su
vip.51.la vip.51.la
visionwell.com.cn
vkonche.com vkonche.com
voloomoney.com vote-up.ru
voprosotvet24.ru w7s.ru
wemedinc.com wait3sec.org
whatsappdownload10.com webads.co.nz
wildcattube.com website-analytics.online
wladimirpayen.com where-toget.com
womama.ru wholinkstome.com
woman-h.ru wiosenny-bon-1500.pl
womensplay.net wstroika.ru
wosik-dach.service-for-web.de www.event-tracking.com
wpsecurity.website www.kabbalah-red-bracelets.com
xgftnlrt.bloger.index.hr xingzi-vision.com
xn--80ahdheogk5l.xn--p1ai x-lime.net
xuki.us xn----7sbaphztdjeboffeiof6c.xn--p1ai
yellowads.men xn--oogle-wmc.com
yginekologa.com zahvat.ru
your-bearings.com zaim-pod-zalog-krasnodar.ru
zakonobosago.ru zarepta.com
zerocash.msk.ru zeg-distribution.com
zigzog.ru
zoomovies.org
zrizvtrnpale.tk
zx6.ru

BIN
.latest_release/bots.d.tar.gz
View file

Binary file not shown.

BIN
.latest_release/conf.d.tar.gz
View file

Binary file not shown.

2
AUTO-CONFIGURATION.md
View file

@ -4,7 +4,7 @@
### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation:
_______________ _______________
#### Version: V4.2019.06.1597 #### Version: V4.2019.06.1598
#### Bad Referrer Count: 6711 #### Bad Referrer Count: 6711
#### Bad Bot Count: 556 #### Bad Bot Count: 556
____________________ ____________________

2
MANUAL-CONFIGURATION.md
View file

@ -4,7 +4,7 @@
### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation:
_______________ _______________
#### Version: V4.2019.06.1597 #### Version: V4.2019.06.1598
#### Bad Referrer Count: 6711 #### Bad Referrer Count: 6711
#### Bad Bot Count: 556 #### Bad Bot Count: 556
____________________ ____________________

2
README.md
View file

@ -11,7 +11,7 @@
##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots)
_______________ _______________
#### Version: V4.2019.06.1597 #### Version: V4.2019.06.1598
#### Bad Referrer Count: 6711 #### Bad Referrer Count: 6711
#### Bad Bot Count: 556 #### Bad Bot Count: 556
____________________ ____________________

2
_google_analytics_ghost_spam/README.md
View file

@ -3,7 +3,7 @@
# EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM
_______________ _______________
#### Version: V4.2019.06.1597 #### Version: V4.2019.06.1598
#### Bad Referrer Count: 6711 #### Bad Referrer Count: 6711
#### Bad Bot Count: 556 #### Bad Bot Count: 556
____________________ ____________________

114
_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf
View file

@ -4,8 +4,8 @@
### VERSION INFORMATION # ### VERSION INFORMATION #
################################################### ###################################################
### Version: V4.2019.06.1597 ### Version: V4.2019.06.1598
### Updated: Tue Jun 25 15:25:42 SAST 2019 ### Updated: Tue Jun 25 15:42:43 SAST 2019
### Bad Referrer Count: 6711 ### Bad Referrer Count: 6711
### Bad Bot Count: 556 ### Bad Bot Count: 556
################################################### ###################################################
@ -693,38 +693,38 @@ map $http_user_agent $bad_bot {
# *********************************************** # ***********************************************
# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###
"~*\badidxbot\b" 0; "~*(?:\b)adidxbot(?:\b|)" 0;
"~*\bAdsBot-Google\b" 0; "~*(?:\b)AdsBot-Google(?:\b|)" 0;
"~*\baolbuild\b" 0; "~*(?:\b)aolbuild(?:\b|)" 0;
"~*\bbingbot\b" 0; "~*(?:\b)bingbot(?:\b|)" 0;
"~*\bbingpreview\b" 0; "~*(?:\b)bingpreview(?:\b|)" 0;
"~*\bDoCoMo\b" 0; "~*(?:\b)DoCoMo(?:\b|)" 0;
"~*\bduckduckgo\b" 0; "~*(?:\b)duckduckgo(?:\b|)" 0;
"~*\bfacebookexternalhit\b" 0; "~*(?:\b)facebookexternalhit(?:\b|)" 0;
"~*\bFeedfetcher-Google\b" 0; "~*(?:\b)Feedfetcher-Google(?:\b|)" 0;
"~*\bGooglebot\b" 0; "~*(?:\b)Googlebot(?:\b|)" 0;
"~*\bGooglebot-Image\b" 0; "~*(?:\b)Googlebot-Image(?:\b|)" 0;
"~*\bGooglebot-Mobile\b" 0; "~*(?:\b)Googlebot-Mobile(?:\b|)" 0;
"~*\bGooglebot-News\b" 0; "~*(?:\b)Googlebot-News(?:\b|)" 0;
"~*\bGooglebot/Test\b" 0; "~*(?:\b)Googlebot/Test(?:\b|)" 0;
"~*\bGooglebot-Video\b" 0; "~*(?:\b)Googlebot-Video(?:\b|)" 0;
"~*\bGoogle-HTTP-Java-Client\b" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0;
"~*\bGravityscan\b" 0; "~*(?:\b)Gravityscan(?:\b|)" 0;
"~*\bgsa-crawler\b" 0; "~*(?:\b)gsa-crawler(?:\b|)" 0;
"~*\bJakarta\ Commons\b" 0; "~*(?:\b)Jakarta\ Commons(?:\b|)" 0;
"~*\bKraken/0.1\b" 0; "~*(?:\b)Kraken/0.1(?:\b|)" 0;
"~*\bLinkedInBot\b" 0; "~*(?:\b)LinkedInBot(?:\b|)" 0;
"~*\bMediapartners-Google\b" 0; "~*(?:\b)Mediapartners-Google(?:\b|)" 0;
"~*\bmsnbot\b" 0; "~*(?:\b)msnbot(?:\b|)" 0;
"~*\bmsnbot-media\b" 0; "~*(?:\b)msnbot-media(?:\b|)" 0;
"~*\bSAMSUNG\b" 0; "~*(?:\b)SAMSUNG(?:\b|)" 0;
"~*\bSlackbot\b" 0; "~*(?:\b)Slackbot(?:\b|)" 0;
"~*\bSlackbot-LinkExpanding\b" 0; "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0;
"~*\bslurp\b" 0; "~*(?:\b)slurp(?:\b|)" 0;
"~*\bteoma\b" 0; "~*(?:\b)teoma(?:\b|)" 0;
"~*\bTwitterBot\b" 0; "~*(?:\b)TwitterBot(?:\b|)" 0;
"~*\bWordpress\b" 0; "~*(?:\b)Wordpress(?:\b|)" 0;
"~*\byahoo\b" 0; "~*(?:\b)yahoo(?:\b|)" 0;
# END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###
# *************************************************** # ***************************************************
@ -735,12 +735,12 @@ map $http_user_agent $bad_bot {
# I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited
# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
"~*\bjetmon\b" 1; "~*(?:\b)jetmon(?:\b|)" 1;
"~*\blibwww-perl\b" 1; "~*(?:\b)libwww-perl(?:\b|)" 1;
"~*\bLynx\b" 1; "~*(?:\b)Lynx(?:\b|)" 1;
"~*\bmunin\b" 1; "~*(?:\b)munin(?:\b|)" 1;
"~*\bPresto\b" 1; "~*(?:\b)Presto(?:\b|)" 1;
"~*\bWget/1.15\b" 1; "~*(?:\b)Wget/1.15(?:\b|)" 1;
# END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
# ************************************************************** # **************************************************************
@ -748,23 +748,23 @@ map $http_user_agent $bad_bot {
# ************************************************************** # **************************************************************
# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
"~*\bAlexa\b" 2; "~*(?:\b)Alexa(?:\b|)" 2;
"~*\barchive.org\b" 2; "~*(?:\b)archive.org(?:\b|)" 2;
"~*\bBaidu\b" 2; "~*(?:\b)Baidu(?:\b|)" 2;
"~*\bBUbiNG\b" 2; "~*(?:\b)BUbiNG(?:\b|)" 2;
"~*\bFlipboardProxy\b" 2; "~*(?:\b)FlipboardProxy(?:\b|)" 2;
"~*\bia_archiver\b" 2; "~*(?:\b)ia_archiver(?:\b|)" 2;
"~*\bMSIE\ 7.0\b" 2; "~*(?:\b)MSIE\ 7.0(?:\b|)" 2;
"~*\bProximic\b" 2; "~*(?:\b)Proximic(?:\b|)" 2;
"~*\bR6_CommentReader\b" 2; "~*(?:\b)R6_CommentReader(?:\b|)" 2;
"~*\bR6_FeedFetcher\b" 2; "~*(?:\b)R6_FeedFetcher(?:\b|)" 2;
"~*\bRED/1\b" 2; "~*(?:\b)RED/1(?:\b|)" 2;
"~*\bRPT-HTTPClient\b" 2; "~*(?:\b)RPT-HTTPClient(?:\b|)" 2;
"~*\bsfFeedReader/0.9\b" 2; "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2;
"~*\bSpaidu\b" 2; "~*(?:\b)Spaidu(?:\b|)" 2;
"~*\bUptimeRobot/2.0\b" 2; "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2;
"~*\bYandexBot\b" 2; "~*(?:\b)YandexBot(?:\b|)" 2;
"~*\bYandexImages\b" 2; "~*(?:\b)YandexImages(?:\b|)" 2;
# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
} }

4
conf.d/globalblacklist-testing-version.conf
View file

@ -10,8 +10,8 @@
### VERSION INFORMATION # ### VERSION INFORMATION #
################################################### ###################################################
### Version: V3.2019.06.1597 ### Version: V3.2019.06.1598
### Updated: Tue Jun 25 15:25:43 SAST 2019 ### Updated: Tue Jun 25 15:42:44 SAST 2019
### Bad Referrer Count: 6711 ### Bad Referrer Count: 6711
### Bad Bot Count: 556 ### Bad Bot Count: 556
################################################### ###################################################

114
conf.d/globalblacklist.conf
View file

@ -4,8 +4,8 @@
### VERSION INFORMATION # ### VERSION INFORMATION #
################################################### ###################################################
### Version: V4.2019.06.1597 ### Version: V4.2019.06.1598
### Updated: Tue Jun 25 15:25:42 SAST 2019 ### Updated: Tue Jun 25 15:42:43 SAST 2019
### Bad Referrer Count: 6711 ### Bad Referrer Count: 6711
### Bad Bot Count: 556 ### Bad Bot Count: 556
################################################### ###################################################
@ -693,38 +693,38 @@ map $http_user_agent $bad_bot {
# *********************************************** # ***********************************************
# START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###
"~*\badidxbot\b" 0; "~*(?:\b)adidxbot(?:\b|)" 0;
"~*\bAdsBot-Google\b" 0; "~*(?:\b)AdsBot-Google(?:\b|)" 0;
"~*\baolbuild\b" 0; "~*(?:\b)aolbuild(?:\b|)" 0;
"~*\bbingbot\b" 0; "~*(?:\b)bingbot(?:\b|)" 0;
"~*\bbingpreview\b" 0; "~*(?:\b)bingpreview(?:\b|)" 0;
"~*\bDoCoMo\b" 0; "~*(?:\b)DoCoMo(?:\b|)" 0;
"~*\bduckduckgo\b" 0; "~*(?:\b)duckduckgo(?:\b|)" 0;
"~*\bfacebookexternalhit\b" 0; "~*(?:\b)facebookexternalhit(?:\b|)" 0;
"~*\bFeedfetcher-Google\b" 0; "~*(?:\b)Feedfetcher-Google(?:\b|)" 0;
"~*\bGooglebot\b" 0; "~*(?:\b)Googlebot(?:\b|)" 0;
"~*\bGooglebot-Image\b" 0; "~*(?:\b)Googlebot-Image(?:\b|)" 0;
"~*\bGooglebot-Mobile\b" 0; "~*(?:\b)Googlebot-Mobile(?:\b|)" 0;
"~*\bGooglebot-News\b" 0; "~*(?:\b)Googlebot-News(?:\b|)" 0;
"~*\bGooglebot/Test\b" 0; "~*(?:\b)Googlebot/Test(?:\b|)" 0;
"~*\bGooglebot-Video\b" 0; "~*(?:\b)Googlebot-Video(?:\b|)" 0;
"~*\bGoogle-HTTP-Java-Client\b" 0; "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0;
"~*\bGravityscan\b" 0; "~*(?:\b)Gravityscan(?:\b|)" 0;
"~*\bgsa-crawler\b" 0; "~*(?:\b)gsa-crawler(?:\b|)" 0;
"~*\bJakarta\ Commons\b" 0; "~*(?:\b)Jakarta\ Commons(?:\b|)" 0;
"~*\bKraken/0.1\b" 0; "~*(?:\b)Kraken/0.1(?:\b|)" 0;
"~*\bLinkedInBot\b" 0; "~*(?:\b)LinkedInBot(?:\b|)" 0;
"~*\bMediapartners-Google\b" 0; "~*(?:\b)Mediapartners-Google(?:\b|)" 0;
"~*\bmsnbot\b" 0; "~*(?:\b)msnbot(?:\b|)" 0;
"~*\bmsnbot-media\b" 0; "~*(?:\b)msnbot-media(?:\b|)" 0;
"~*\bSAMSUNG\b" 0; "~*(?:\b)SAMSUNG(?:\b|)" 0;
"~*\bSlackbot\b" 0; "~*(?:\b)Slackbot(?:\b|)" 0;
"~*\bSlackbot-LinkExpanding\b" 0; "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0;
"~*\bslurp\b" 0; "~*(?:\b)slurp(?:\b|)" 0;
"~*\bteoma\b" 0; "~*(?:\b)teoma(?:\b|)" 0;
"~*\bTwitterBot\b" 0; "~*(?:\b)TwitterBot(?:\b|)" 0;
"~*\bWordpress\b" 0; "~*(?:\b)Wordpress(?:\b|)" 0;
"~*\byahoo\b" 0; "~*(?:\b)yahoo(?:\b|)" 0;
# END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ###
# *************************************************** # ***************************************************
@ -735,12 +735,12 @@ map $http_user_agent $bad_bot {
# I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited
# START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
"~*\bjetmon\b" 1; "~*(?:\b)jetmon(?:\b|)" 1;
"~*\blibwww-perl\b" 1; "~*(?:\b)libwww-perl(?:\b|)" 1;
"~*\bLynx\b" 1; "~*(?:\b)Lynx(?:\b|)" 1;
"~*\bmunin\b" 1; "~*(?:\b)munin(?:\b|)" 1;
"~*\bPresto\b" 1; "~*(?:\b)Presto(?:\b|)" 1;
"~*\bWget/1.15\b" 1; "~*(?:\b)Wget/1.15(?:\b|)" 1;
# END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
# ************************************************************** # **************************************************************
@ -748,23 +748,23 @@ map $http_user_agent $bad_bot {
# ************************************************************** # **************************************************************
# START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
"~*\bAlexa\b" 2; "~*(?:\b)Alexa(?:\b|)" 2;
"~*\barchive.org\b" 2; "~*(?:\b)archive.org(?:\b|)" 2;
"~*\bBaidu\b" 2; "~*(?:\b)Baidu(?:\b|)" 2;
"~*\bBUbiNG\b" 2; "~*(?:\b)BUbiNG(?:\b|)" 2;
"~*\bFlipboardProxy\b" 2; "~*(?:\b)FlipboardProxy(?:\b|)" 2;
"~*\bia_archiver\b" 2; "~*(?:\b)ia_archiver(?:\b|)" 2;
"~*\bMSIE\ 7.0\b" 2; "~*(?:\b)MSIE\ 7.0(?:\b|)" 2;
"~*\bProximic\b" 2; "~*(?:\b)Proximic(?:\b|)" 2;
"~*\bR6_CommentReader\b" 2; "~*(?:\b)R6_CommentReader(?:\b|)" 2;
"~*\bR6_FeedFetcher\b" 2; "~*(?:\b)R6_FeedFetcher(?:\b|)" 2;
"~*\bRED/1\b" 2; "~*(?:\b)RED/1(?:\b|)" 2;
"~*\bRPT-HTTPClient\b" 2; "~*(?:\b)RPT-HTTPClient(?:\b|)" 2;
"~*\bsfFeedReader/0.9\b" 2; "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2;
"~*\bSpaidu\b" 2; "~*(?:\b)Spaidu(?:\b|)" 2;
"~*\bUptimeRobot/2.0\b" 2; "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2;
"~*\bYandexBot\b" 2; "~*(?:\b)YandexBot(?:\b|)" 2;
"~*\bYandexImages\b" 2; "~*(?:\b)YandexImages(?:\b|)" 2;
# END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ###
} }

4
robots.txt/robots.txt
View file

@ -6,8 +6,8 @@
### Version Information # ### Version Information #
################################################### ###################################################
### Version: V4.2019.06.1597 ### Version: V4.2019.06.1598
### Updated: Tue Jun 25 15:25:44 SAST 2019 ### Updated: Tue Jun 25 15:42:45 SAST 2019
### Bad Bot Count: 556 ### Bad Bot Count: 556
################################################### ###################################################
### Version Information ## ### Version Information ##