diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 7321a3612..d4482041b 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -17,11 +17,11 @@ ### - https://github.com/oohnoitz/nginx-blacklist ### Last Updated -### Wed Apr 19 13:00:08 SAST 2017 +### Wed Apr 19 13:11:25 SAST 2017 ### End Last Updated ### Generated in -### 0.318285942078 seconds +### 0.330133199692 seconds ### End Generated in ### Tested on: nginx/1.10.0 (Ubuntu 16.04) @@ -128,35 +128,35 @@ map $http_user_agent $bad_bot { # *********************************************** # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "^adidxbot" 0; - "^AdsBot-Google" 0; - "^aolbuild" 0; - "^bingbot" 0; - "^bingpreview" 0; - "^DoCoMo" 0; - "^duckduckgo" 0; - "^facebookexternalhit" 0; - "^Feedfetcher-Google" 0; - "^Googlebot" 0; - "^Googlebot-Image" 0; - "^Googlebot-Mobile" 0; - "^Googlebot-News" 0; - "^Googlebot/Test" 0; - "^Googlebot-Video" 0; - "^Google-HTTP-Java-Client" 0; - "^gsa-crawler" 0; - "^Jakarta\ Commons" 0; - "^Kraken/0.1" 0; - "^LinkedInBot" 0; - "^Mediapartners-Google" 0; - "^msnbot" 0; - "^msnbot-media" 0; - "^SAMSUNG" 0; - "^slurp" 0; - "^teoma" 0; - "^TwitterBot" 0; - "^Wordpress" 0; - "^yahoo" 0; + "~adidxbot" 0; + "~AdsBot-Google" 0; + "~aolbuild" 0; + "~bingbot" 0; + "~bingpreview" 0; + "~DoCoMo" 0; + "~duckduckgo" 0; + "~facebookexternalhit" 0; + "~Feedfetcher-Google" 0; + "~Googlebot" 0; + "~Googlebot-Image" 0; + "~Googlebot-Mobile" 0; + "~Googlebot-News" 0; + "~Googlebot/Test" 0; + "~Googlebot-Video" 0; + "~Google-HTTP-Java-Client" 0; + "~gsa-crawler" 0; + "~Jakarta\ Commons" 0; + "~Kraken/0.1" 0; + "~LinkedInBot" 0; + "~Mediapartners-Google" 0; + "~msnbot" 0; + "~msnbot-media" 0; + "~SAMSUNG" 0; + "~slurp" 0; + "~teoma" 0; + "~TwitterBot" 0; + "~Wordpress" 0; + "~yahoo" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************** @@ -166,11 +166,11 @@ map $http_user_agent $bad_bot { # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "^jetmon" 1; - "^libwww-perl" 1; - "^Lynx" 1; - "^munin" 1; - "^Wget/1.15" 1; + "~jetmon" 1; + "~libwww-perl" 1; + "~Lynx" 1; + "~munin" 1; + "~Wget/1.15" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************************** @@ -178,22 +178,22 @@ map $http_user_agent $bad_bot { # ************************************************************** # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "^Alexa" 2; - "^archive.org" 2; - "^Baidu" 2; - "^FlipboardProxy" 2; - "^ia_archiver" 2; - "^Mozilla/4.0" 2; - "^MSIE\ 7.0" 2; - "^Presto" 2; - "^R6_CommentReader" 2; - "^R6_FeedFetcher" 2; - "^RPT-HTTPClient" 2; - "^sfFeedReader/0.9" 2; - "^Spaidu" 2; - "^UptimeRobot/2.0" 2; - "^YandexBot" 2; - "^YandexImages" 2; + "~Alexa" 2; + "~archive.org" 2; + "~Baidu" 2; + "~FlipboardProxy" 2; + "~ia_archiver" 2; + "~Mozilla/4.0" 2; + "~MSIE\ 7.0" 2; + "~Presto" 2; + "~R6_CommentReader" 2; + "~R6_FeedFetcher" 2; + "~RPT-HTTPClient" 2; + "~sfFeedReader/0.9" 2; + "~Spaidu" 2; + "~UptimeRobot/2.0" 2; + "~YandexBot" 2; + "~YandexImages" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ********************************************* @@ -211,476 +211,476 @@ map $http_user_agent $bad_bot { # END BLACKLISTED USER AGENTS ### DO NOT EDIT THIS LINE AT ALL ### # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "^360Spider" 3; - "^80legs" 3; - "^Abonti" 3; - "^Aboundex" 3; - "^Acunetix" 3; - "^AhrefsBot" 3; - "^AIBOT" 3; - "^AiHitBot" 3; - "^Aipbot" 3; - "^Alexibot" 3; - "^Alligator" 3; - "^AllSubmitter" 3; - "^Anarchie" 3; - "^Apexoo" 3; - "^ASPSeek" 3; - "^Asterias" 3; - "^Attach" 3; - "^autoemailspider" 3; - "^BackDoorBot" 3; - "^BackStreet" 3; - "^BackWeb" 3; - "^Badass" 3; - "^Bandit" 3; - "^BatchFTP" 3; - "^Battleztar\ Bazinga" 3; - "^BBBike" 3; - "^BDFetch" 3; - "^BetaBot" 3; - "^Bigfoot" 3; - "^Bitacle" 3; - "^Black\ Hole" 3; - "^BlackWidow" 3; - "^BLEXBot" 3; - "^Blow" 3; - "^BlowFish" 3; - "^Boardreader" 3; - "^Bolt" 3; - "^BotALot" 3; - "^Brandprotect" 3; - "^Brandwatch" 3; - "^Bubing" 3; - "^Buddy" 3; - "^BuiltBotTough" 3; - "^BuiltWith" 3; - "^Bullseye" 3; - "^BunnySlippers" 3; - "^BuzzSumo" 3; - "^Calculon" 3; - "^CazoodleBot" 3; - "^CCBot" 3; - "^Cegbfeieh" 3; - "^CheeseBot" 3; - "^CherryPicker" 3; - "^ChinaClaw" 3; - "^Chlooe" 3; - "^Claritybot" 3; - "^Cliqzbot" 3; - "^Cogentbot" 3; - "^Collector" 3; - "^Copier" 3; - "^CopyRightCheck" 3; - "^Copyscape" 3; - "^Cosmos" 3; - "^Craftbot" 3; - "^CrazyWebCrawler" 3; - "^Crescent" 3; - "^CSHttp" 3; - "^Curious" 3; - "^Custo" 3; - "^DBLBot" 3; - "^Demon" 3; - "^Deusu" 3; - "^Devil" 3; - "^DIIbot" 3; - "^Dirbuster" 3; - "^Disco" 3; - "^Discobot" 3; - "^Discoverybot" 3; - "^DittoSpyder" 3; - "^DomainAppender" 3; - "^DomainSigmaCrawler" 3; - "^Dotbot" 3; - "^Download\ Demon" 3; - "^Download\ Devil" 3; - "^Download\ Wonder" 3; - "^Dragonfly" 3; - "^Drip" 3; - "^DTS\ Agent" 3; - "^EasyDL" 3; - "^Ebingbong" 3; - "^eCatch" 3; - "^ECCP/1.0" 3; - "^Ecxi" 3; - "^EirGrabber" 3; - "^EMail\ Collector" 3; - "^EMail\ Extractor" 3; - "^EMail\ Siphon" 3; - "^EMail\ Wolf" 3; - "^EroCrawler" 3; - "^Evil" 3; - "^Exabot" 3; - "^Express\ WebPictures" 3; - "^Extractor" 3; - "^ExtractorPro" 3; - "^EyeNetIE" 3; - "^Ezooms" 3; - "^FHscan" 3; - "^Fimap" 3; - "^Findxbot" 3; - "^Firefox/7.0" 3; - "^FlashGet" 3; - "^Flunky" 3; - "^Foobot" 3; - "^Freeuploader" 3; - "^FrontPage" 3; - "^GalaxyBot" 3; - "^Genieo" 3; - "^Getintent" 3; - "^GetRight" 3; - "^GetWeb" 3; - "^Gigablast" 3; - "^Gigabot" 3; - "^Go-Ahead-Got-It" 3; - "^Gotit" 3; - "^GoZilla" 3; - "^Go!Zilla" 3; - "^Grabber" 3; - "^GrabNet" 3; - "^Grafula" 3; - "^GrapeFX" 3; - "^GrapeshotCrawler" 3; - "^GridBot" 3; - "^GT::WWW" 3; - "^GuzzleHttp" 3; - "^HaosouSpider" 3; - "^Harvest" 3; - "^Havij" 3; - "^Heritrix" 3; - "^Hloader" 3; - "^HMView" 3; - "^HTMLparser" 3; - "^HTTP::Lite" 3; - "^HTTrack" 3; - "^Humanlinks" 3; - "^Iblog" 3; - "^IDBot" 3; - "^Id-search" 3; - "^IlseBot" 3; - "^Image\ Fetch" 3; - "^Image\ Stripper" 3; - "^Image\ Sucker" 3; - "^Indy\ Library" 3; - "^InfoNaviRobot" 3; - "^InfoTekies" 3; - "^Intelliseek" 3; - "^InterGET" 3; - "^Internet\ Ninja" 3; - "^InternetSeer" 3; - "^internetVista\ monitor" 3; - "^Iria" 3; - "^IRLbot" 3; - "^Iskanie" 3; - "^JamesBOT" 3; - "^Jbrofuzz" 3; - "^JennyBot" 3; - "^JetCar" 3; - "^JikeSpider" 3; - "^JOC\ Web\ Spider" 3; - "^Joomla" 3; - "^JustView" 3; - "^Jyxobot" 3; - "^Kenjin\ Spider" 3; - "^Keyword\ Density" 3; - "^Kraken" 3; - "^Lanshanbot" 3; - "^Larbin" 3; - "^LeechFTP" 3; - "^LeechGet" 3; - "^LexiBot" 3; - "^Lftp" 3; - "^LibWeb" 3; - "^Libwhisker" 3; - "^Lightspeedsystems" 3; - "^Likse" 3; - "^Linkdexbot" 3; - "^LinkextractorPro" 3; - "^LinkpadBot" 3; - "^LinkScan" 3; - "^LinksManager" 3; - "^LinkWalker" 3; - "^LinqiaMetadataDownloaderBot" 3; - "^LinqiaRSSBot" 3; - "^LinqiaScrapeBot" 3; - "^Lipperhey" 3; - "^Litemage_walker" 3; - "^Lmspider" 3; - "^LNSpiderguy" 3; - "^Ltx71" 3; - "^lwp-request" 3; - "^LWP::Simple" 3; - "^lwp-trivial" 3; - "^Magnet" 3; - "^Mag-Net" 3; - "^magpie-crawler" 3; - "^Mail.ru" 3; - "^Majestic12" 3; - "^MarkMonitor" 3; - "^MarkWatch" 3; - "^Masscan" 3; - "^Mass\ Downloader" 3; - "^Mata\ Hari" 3; - "^Meanpathbot" 3; - "^MegaIndex.ru" 3; - "^Metauri" 3; - "^MFC_Tear_Sample" 3; - "^Microsoft\ Data\ Access" 3; - "^Microsoft\ URL\ Control" 3; - "^MIDown\ tool" 3; - "^MIIxpc" 3; - "^Mister\ PiX" 3; - "^MJ12bot" 3; - "^Mojeek" 3; - "^MSFrontPage" 3; - "^MSIE\ 6.0" 3; - "^MSIECrawler" 3; - "^Msrabot" 3; - "^MS\ Web\ Services\ Client\ Protocol" 3; - "^Musobot" 3; - "^Name\ Intelligence" 3; - "^Nameprotect" 3; - "^Navroad" 3; - "^NearSite" 3; - "^Needle" 3; - "^Nessus" 3; - "^NetAnts" 3; - "^Netcraft" 3; - "^netEstate\ NE\ Crawler" 3; - "^NetLyzer" 3; - "^NetMechanic" 3; - "^NetSpider" 3; - "^Nettrack" 3; - "^Net\ Vampire" 3; - "^NetZIP" 3; - "^NextGenSearchBot" 3; - "^Nibbler" 3; - "^NICErsPRO" 3; - "^Niki-bot" 3; - "^Nikto" 3; - "^NimbleCrawler" 3; - "^Ninja" 3; - "^Nmap" 3; - "^NPbot" 3; - "^Nutch" 3; - "^Octopus" 3; - "^Offline\ Explorer" 3; - "^Offline\ Navigator" 3; - "^Openfind" 3; - "^OpenLinkProfiler" 3; - "^Openvas" 3; - "^OrangeBot" 3; - "^OrangeSpider" 3; - "^OutfoxBot" 3; - "^PageAnalyzer" 3; - "^Page\ Analyzer" 3; - "^PageGrabber" 3; - "^Page\ Grabber" 3; - "^page\ scorer" 3; - "^PageScorer" 3; - "^Panscient" 3; - "^Papa\ Foto" 3; - "^Pavuk" 3; - "^pcBrowser" 3; - "^PECL::HTTP" 3; - "^PeoplePal" 3; - "^PHPCrawl" 3; - "^Picscout" 3; - "^Picsearch" 3; - "^Pimonster" 3; - "^Pi-Monster" 3; - "^Pixray" 3; - "^PleaseCrawl" 3; - "^Pockey" 3; - "^POE-Component-Client-HTTP" 3; - "^Probethenet" 3; - "^ProPowerBot" 3; - "^ProWebWalker" 3; - "^Proximic" 3; - "^Psbot" 3; - "^Pump" 3; - "^PyCurl" 3; - "^QueryN\ Metasearch" 3; - "^Qwantify" 3; - "^RankActiveLinkBot" 3; - "^Rankivabot" 3; - "^RealDownload" 3; - "^Reaper" 3; - "^Recorder" 3; - "^RedesScrapy" 3; - "^ReGet" 3; - "^RepoMonkey" 3; - "^Ripper" 3; - "^RocketCrawler" 3; - "^Rogerbot" 3; - "^SBIder" 3; - "^ScanAlert" 3; - "^Scanbot" 3; - "^Scrapy" 3; - "^Screaming\ Frog\ SEO\ Spider" 3; - "^ScreenerBot" 3; - "^Searchestate" 3; - "^SearchmetricsBot" 3; - "^Semrush" 3; - "^SemrushBot" 3; - "^SEOkicks" 3; - "^SEOkicks-Robot" 3; - "^SEOlyticsCrawler" 3; - "^Seomoz" 3; - "^SEOprofiler" 3; - "^SEOstats" 3; - "^SeznamBot" 3; - "^Siphon" 3; - "^SISTRIX" 3; - "^SISTRIX\ Crawler" 3; - "^Sitebeam" 3; - "^SiteExplorer" 3; - "^Siteimprove" 3; - "^SiteLockSpider" 3; - "^SiteSnagger" 3; - "^SiteSucker" 3; - "^Site\ Sucker" 3; - "^Sitevigil" 3; - "^Slackbot-LinkExpanding" 3; - "^SlySearch" 3; - "^SmartDownload" 3; - "^Snake" 3; - "^Snapbot" 3; - "^Snoopy" 3; - "^SocialRankIOBot" 3; - "^Sogou\ web\ spider" 3; - "^Sosospider" 3; - "^SpaceBison" 3; - "^Spammen" 3; - "^SpankBot" 3; - "^Spanner" 3; - "^Spbot" 3; - "^Spinn3r" 3; - "^SputnikBot" 3; - "^Sqlmap" 3; - "^Sqlworm" 3; - "^Sqworm" 3; - "^Steeler" 3; - "^Stripper" 3; - "^Sucker" 3; - "^Sucuri" 3; - "^SuperBot" 3; - "^SuperHTTP" 3; - "^Surfbot" 3; - "^SurveyBot" 3; - "^Suzuran" 3; - "^Swiftbot" 3; - "^Szukacz" 3; - "^T0PHackTeam" 3; - "^T8Abot" 3; - "^tAkeOut" 3; - "^Teleport" 3; - "^TeleportPro" 3; - "^Telesoft" 3; - "^Telesphoreo" 3; - "^Telesphorep" 3; - "^The\ Intraformant" 3; - "^TheNomad" 3; - "^TightTwatBot" 3; - "^Titan" 3; - "^Toata" 3; - "^Toweyabot" 3; - "^Trendictionbot" 3; - "^True_Robot" 3; - "^Turingos" 3; - "^TurnitinBot" 3; - "^Turnitin\ Bot" 3; - "^Turnitin\ Robot" 3; - "^TwengaBot" 3; - "^Twice" 3; - "^Typhoeus" 3; - "^URLy.Warning" 3; - "^URLy\ Warning" 3; - "^Vacuum" 3; - "^Vagabondo" 3; - "^VB\ Project" 3; - "^VCI" 3; - "^VeriCiteCrawler" 3; - "^VidibleScraper" 3; - "^VoidEYE" 3; - "^Voil" 3; - "^Voltron" 3; - "^WallpapersHD" 3; - "^WASALive-Bot" 3; - "^WBSearchBot" 3; - "^Webalta" 3; - "^WebAuto" 3; - "^Web\ Auto" 3; - "^WebBandit" 3; - "^Web\ Bandit" 3; - "^WebCollage" 3; - "^Web\ Collage" 3; - "^WebCopier" 3; - "^Web\ Copier" 3; - "^WEBDAV" 3; - "^WEBDAV\ Client" 3; - "^WebEnhancer" 3; - "^Web\ Enhancer" 3; - "^WebFetch" 3; - "^Web\ Fetch" 3; - "^WebFuck" 3; - "^Web\ Fuck" 3; - "^WebGo\ IS" 3; - "^WebImageCollector" 3; - "^Web\ Image\ Collector" 3; - "^WebLeacher" 3; - "^WebmasterWorldForumBot" 3; - "^webmeup-crawler" 3; - "^WebPix" 3; - "^Web\ Pix" 3; - "^WebReaper" 3; - "^Web\ Reaper" 3; - "^WebSauger" 3; - "^Web\ Sauger" 3; - "^Webshag" 3; - "^WebsiteExtractor" 3; - "^Website\ Extractor" 3; - "^WebsiteQuester" 3; - "^Website\ Quester" 3; - "^Webster" 3; - "^WebStripper" 3; - "^Web\ Stripper" 3; - "^WebSucker" 3; - "^Web\ Sucker" 3; - "^WebWhacker" 3; - "^Web\ Whacker" 3; - "^WebZIP" 3; - "^WeSEE" 3; - "^Whack" 3; - "^Whacker" 3; - "^Whatweb" 3; - "^Widow" 3; - "^WinHTTrack" 3; - "^WiseGuys\ Robot" 3; - "^WISENutbot" 3; - "^Wonderbot" 3; - "^Woobot" 3; - "^Wotbox" 3; - "^Wprecon" 3; - "^WPScan" 3; - "^WWW-Collector-E" 3; - "^WWW-Mechanize" 3; - "^WWW::Mechanize" 3; - "^WWWOFFLE" 3; - "^Xaldon_WebSpider" 3; - "^Xaldon\ WebSpider" 3; - "^Xenu" 3; - "^Y!J-ASR" 3; - "^YoudaoBot" 3; - "^Zade" 3; - "^Zermelo" 3; - "^Zeus" 3; - "^Zgrab" 3; - "^Zitebot" 3; - "^ZmEu" 3; - "^ZumBot" 3; - "^ZyBorg" 3; + "~360Spider" 3; + "~80legs" 3; + "~Abonti" 3; + "~Aboundex" 3; + "~Acunetix" 3; + "~AhrefsBot" 3; + "~AIBOT" 3; + "~AiHitBot" 3; + "~Aipbot" 3; + "~Alexibot" 3; + "~Alligator" 3; + "~AllSubmitter" 3; + "~Anarchie" 3; + "~Apexoo" 3; + "~ASPSeek" 3; + "~Asterias" 3; + "~Attach" 3; + "~autoemailspider" 3; + "~BackDoorBot" 3; + "~BackStreet" 3; + "~BackWeb" 3; + "~Badass" 3; + "~Bandit" 3; + "~BatchFTP" 3; + "~Battleztar\ Bazinga" 3; + "~BBBike" 3; + "~BDFetch" 3; + "~BetaBot" 3; + "~Bigfoot" 3; + "~Bitacle" 3; + "~Black\ Hole" 3; + "~BlackWidow" 3; + "~BLEXBot" 3; + "~Blow" 3; + "~BlowFish" 3; + "~Boardreader" 3; + "~Bolt" 3; + "~BotALot" 3; + "~Brandprotect" 3; + "~Brandwatch" 3; + "~Bubing" 3; + "~Buddy" 3; + "~BuiltBotTough" 3; + "~BuiltWith" 3; + "~Bullseye" 3; + "~BunnySlippers" 3; + "~BuzzSumo" 3; + "~Calculon" 3; + "~CazoodleBot" 3; + "~CCBot" 3; + "~Cegbfeieh" 3; + "~CheeseBot" 3; + "~CherryPicker" 3; + "~ChinaClaw" 3; + "~Chlooe" 3; + "~Claritybot" 3; + "~Cliqzbot" 3; + "~Cogentbot" 3; + "~Collector" 3; + "~Copier" 3; + "~CopyRightCheck" 3; + "~Copyscape" 3; + "~Cosmos" 3; + "~Craftbot" 3; + "~CrazyWebCrawler" 3; + "~Crescent" 3; + "~CSHttp" 3; + "~Curious" 3; + "~Custo" 3; + "~DBLBot" 3; + "~Demon" 3; + "~Deusu" 3; + "~Devil" 3; + "~DIIbot" 3; + "~Dirbuster" 3; + "~Disco" 3; + "~Discobot" 3; + "~Discoverybot" 3; + "~DittoSpyder" 3; + "~DomainAppender" 3; + "~DomainSigmaCrawler" 3; + "~Dotbot" 3; + "~Download\ Demon" 3; + "~Download\ Devil" 3; + "~Download\ Wonder" 3; + "~Dragonfly" 3; + "~Drip" 3; + "~DTS\ Agent" 3; + "~EasyDL" 3; + "~Ebingbong" 3; + "~eCatch" 3; + "~ECCP/1.0" 3; + "~Ecxi" 3; + "~EirGrabber" 3; + "~EMail\ Collector" 3; + "~EMail\ Extractor" 3; + "~EMail\ Siphon" 3; + "~EMail\ Wolf" 3; + "~EroCrawler" 3; + "~Evil" 3; + "~Exabot" 3; + "~Express\ WebPictures" 3; + "~Extractor" 3; + "~ExtractorPro" 3; + "~EyeNetIE" 3; + "~Ezooms" 3; + "~FHscan" 3; + "~Fimap" 3; + "~Findxbot" 3; + "~Firefox/7.0" 3; + "~FlashGet" 3; + "~Flunky" 3; + "~Foobot" 3; + "~Freeuploader" 3; + "~FrontPage" 3; + "~GalaxyBot" 3; + "~Genieo" 3; + "~Getintent" 3; + "~GetRight" 3; + "~GetWeb" 3; + "~Gigablast" 3; + "~Gigabot" 3; + "~Go-Ahead-Got-It" 3; + "~Gotit" 3; + "~GoZilla" 3; + "~Go!Zilla" 3; + "~Grabber" 3; + "~GrabNet" 3; + "~Grafula" 3; + "~GrapeFX" 3; + "~GrapeshotCrawler" 3; + "~GridBot" 3; + "~GT::WWW" 3; + "~GuzzleHttp" 3; + "~HaosouSpider" 3; + "~Harvest" 3; + "~Havij" 3; + "~Heritrix" 3; + "~Hloader" 3; + "~HMView" 3; + "~HTMLparser" 3; + "~HTTP::Lite" 3; + "~HTTrack" 3; + "~Humanlinks" 3; + "~Iblog" 3; + "~IDBot" 3; + "~Id-search" 3; + "~IlseBot" 3; + "~Image\ Fetch" 3; + "~Image\ Stripper" 3; + "~Image\ Sucker" 3; + "~Indy\ Library" 3; + "~InfoNaviRobot" 3; + "~InfoTekies" 3; + "~Intelliseek" 3; + "~InterGET" 3; + "~Internet\ Ninja" 3; + "~InternetSeer" 3; + "~internetVista\ monitor" 3; + "~Iria" 3; + "~IRLbot" 3; + "~Iskanie" 3; + "~JamesBOT" 3; + "~Jbrofuzz" 3; + "~JennyBot" 3; + "~JetCar" 3; + "~JikeSpider" 3; + "~JOC\ Web\ Spider" 3; + "~Joomla" 3; + "~JustView" 3; + "~Jyxobot" 3; + "~Kenjin\ Spider" 3; + "~Keyword\ Density" 3; + "~Kraken" 3; + "~Lanshanbot" 3; + "~Larbin" 3; + "~LeechFTP" 3; + "~LeechGet" 3; + "~LexiBot" 3; + "~Lftp" 3; + "~LibWeb" 3; + "~Libwhisker" 3; + "~Lightspeedsystems" 3; + "~Likse" 3; + "~Linkdexbot" 3; + "~LinkextractorPro" 3; + "~LinkpadBot" 3; + "~LinkScan" 3; + "~LinksManager" 3; + "~LinkWalker" 3; + "~LinqiaMetadataDownloaderBot" 3; + "~LinqiaRSSBot" 3; + "~LinqiaScrapeBot" 3; + "~Lipperhey" 3; + "~Litemage_walker" 3; + "~Lmspider" 3; + "~LNSpiderguy" 3; + "~Ltx71" 3; + "~lwp-request" 3; + "~LWP::Simple" 3; + "~lwp-trivial" 3; + "~Magnet" 3; + "~Mag-Net" 3; + "~magpie-crawler" 3; + "~Mail.ru" 3; + "~Majestic12" 3; + "~MarkMonitor" 3; + "~MarkWatch" 3; + "~Masscan" 3; + "~Mass\ Downloader" 3; + "~Mata\ Hari" 3; + "~Meanpathbot" 3; + "~MegaIndex.ru" 3; + "~Metauri" 3; + "~MFC_Tear_Sample" 3; + "~Microsoft\ Data\ Access" 3; + "~Microsoft\ URL\ Control" 3; + "~MIDown\ tool" 3; + "~MIIxpc" 3; + "~Mister\ PiX" 3; + "~MJ12bot" 3; + "~Mojeek" 3; + "~MSFrontPage" 3; + "~MSIE\ 6.0" 3; + "~MSIECrawler" 3; + "~Msrabot" 3; + "~MS\ Web\ Services\ Client\ Protocol" 3; + "~Musobot" 3; + "~Name\ Intelligence" 3; + "~Nameprotect" 3; + "~Navroad" 3; + "~NearSite" 3; + "~Needle" 3; + "~Nessus" 3; + "~NetAnts" 3; + "~Netcraft" 3; + "~netEstate\ NE\ Crawler" 3; + "~NetLyzer" 3; + "~NetMechanic" 3; + "~NetSpider" 3; + "~Nettrack" 3; + "~Net\ Vampire" 3; + "~NetZIP" 3; + "~NextGenSearchBot" 3; + "~Nibbler" 3; + "~NICErsPRO" 3; + "~Niki-bot" 3; + "~Nikto" 3; + "~NimbleCrawler" 3; + "~Ninja" 3; + "~Nmap" 3; + "~NPbot" 3; + "~Nutch" 3; + "~Octopus" 3; + "~Offline\ Explorer" 3; + "~Offline\ Navigator" 3; + "~Openfind" 3; + "~OpenLinkProfiler" 3; + "~Openvas" 3; + "~OrangeBot" 3; + "~OrangeSpider" 3; + "~OutfoxBot" 3; + "~PageAnalyzer" 3; + "~Page\ Analyzer" 3; + "~PageGrabber" 3; + "~Page\ Grabber" 3; + "~page\ scorer" 3; + "~PageScorer" 3; + "~Panscient" 3; + "~Papa\ Foto" 3; + "~Pavuk" 3; + "~pcBrowser" 3; + "~PECL::HTTP" 3; + "~PeoplePal" 3; + "~PHPCrawl" 3; + "~Picscout" 3; + "~Picsearch" 3; + "~Pimonster" 3; + "~Pi-Monster" 3; + "~Pixray" 3; + "~PleaseCrawl" 3; + "~Pockey" 3; + "~POE-Component-Client-HTTP" 3; + "~Probethenet" 3; + "~ProPowerBot" 3; + "~ProWebWalker" 3; + "~Proximic" 3; + "~Psbot" 3; + "~Pump" 3; + "~PyCurl" 3; + "~QueryN\ Metasearch" 3; + "~Qwantify" 3; + "~RankActiveLinkBot" 3; + "~Rankivabot" 3; + "~RealDownload" 3; + "~Reaper" 3; + "~Recorder" 3; + "~RedesScrapy" 3; + "~ReGet" 3; + "~RepoMonkey" 3; + "~Ripper" 3; + "~RocketCrawler" 3; + "~Rogerbot" 3; + "~SBIder" 3; + "~ScanAlert" 3; + "~Scanbot" 3; + "~Scrapy" 3; + "~Screaming\ Frog\ SEO\ Spider" 3; + "~ScreenerBot" 3; + "~Searchestate" 3; + "~SearchmetricsBot" 3; + "~Semrush" 3; + "~SemrushBot" 3; + "~SEOkicks" 3; + "~SEOkicks-Robot" 3; + "~SEOlyticsCrawler" 3; + "~Seomoz" 3; + "~SEOprofiler" 3; + "~SEOstats" 3; + "~SeznamBot" 3; + "~Siphon" 3; + "~SISTRIX" 3; + "~SISTRIX\ Crawler" 3; + "~Sitebeam" 3; + "~SiteExplorer" 3; + "~Siteimprove" 3; + "~SiteLockSpider" 3; + "~SiteSnagger" 3; + "~SiteSucker" 3; + "~Site\ Sucker" 3; + "~Sitevigil" 3; + "~Slackbot-LinkExpanding" 3; + "~SlySearch" 3; + "~SmartDownload" 3; + "~Snake" 3; + "~Snapbot" 3; + "~Snoopy" 3; + "~SocialRankIOBot" 3; + "~Sogou\ web\ spider" 3; + "~Sosospider" 3; + "~SpaceBison" 3; + "~Spammen" 3; + "~SpankBot" 3; + "~Spanner" 3; + "~Spbot" 3; + "~Spinn3r" 3; + "~SputnikBot" 3; + "~Sqlmap" 3; + "~Sqlworm" 3; + "~Sqworm" 3; + "~Steeler" 3; + "~Stripper" 3; + "~Sucker" 3; + "~Sucuri" 3; + "~SuperBot" 3; + "~SuperHTTP" 3; + "~Surfbot" 3; + "~SurveyBot" 3; + "~Suzuran" 3; + "~Swiftbot" 3; + "~Szukacz" 3; + "~T0PHackTeam" 3; + "~T8Abot" 3; + "~tAkeOut" 3; + "~Teleport" 3; + "~TeleportPro" 3; + "~Telesoft" 3; + "~Telesphoreo" 3; + "~Telesphorep" 3; + "~The\ Intraformant" 3; + "~TheNomad" 3; + "~TightTwatBot" 3; + "~Titan" 3; + "~Toata" 3; + "~Toweyabot" 3; + "~Trendictionbot" 3; + "~True_Robot" 3; + "~Turingos" 3; + "~TurnitinBot" 3; + "~Turnitin\ Bot" 3; + "~Turnitin\ Robot" 3; + "~TwengaBot" 3; + "~Twice" 3; + "~Typhoeus" 3; + "~URLy.Warning" 3; + "~URLy\ Warning" 3; + "~Vacuum" 3; + "~Vagabondo" 3; + "~VB\ Project" 3; + "~VCI" 3; + "~VeriCiteCrawler" 3; + "~VidibleScraper" 3; + "~VoidEYE" 3; + "~Voil" 3; + "~Voltron" 3; + "~WallpapersHD" 3; + "~WASALive-Bot" 3; + "~WBSearchBot" 3; + "~Webalta" 3; + "~WebAuto" 3; + "~Web\ Auto" 3; + "~WebBandit" 3; + "~Web\ Bandit" 3; + "~WebCollage" 3; + "~Web\ Collage" 3; + "~WebCopier" 3; + "~Web\ Copier" 3; + "~WEBDAV" 3; + "~WEBDAV\ Client" 3; + "~WebEnhancer" 3; + "~Web\ Enhancer" 3; + "~WebFetch" 3; + "~Web\ Fetch" 3; + "~WebFuck" 3; + "~Web\ Fuck" 3; + "~WebGo\ IS" 3; + "~WebImageCollector" 3; + "~Web\ Image\ Collector" 3; + "~WebLeacher" 3; + "~WebmasterWorldForumBot" 3; + "~webmeup-crawler" 3; + "~WebPix" 3; + "~Web\ Pix" 3; + "~WebReaper" 3; + "~Web\ Reaper" 3; + "~WebSauger" 3; + "~Web\ Sauger" 3; + "~Webshag" 3; + "~WebsiteExtractor" 3; + "~Website\ Extractor" 3; + "~WebsiteQuester" 3; + "~Website\ Quester" 3; + "~Webster" 3; + "~WebStripper" 3; + "~Web\ Stripper" 3; + "~WebSucker" 3; + "~Web\ Sucker" 3; + "~WebWhacker" 3; + "~Web\ Whacker" 3; + "~WebZIP" 3; + "~WeSEE" 3; + "~Whack" 3; + "~Whacker" 3; + "~Whatweb" 3; + "~Widow" 3; + "~WinHTTrack" 3; + "~WiseGuys\ Robot" 3; + "~WISENutbot" 3; + "~Wonderbot" 3; + "~Woobot" 3; + "~Wotbox" 3; + "~Wprecon" 3; + "~WPScan" 3; + "~WWW-Collector-E" 3; + "~WWW-Mechanize" 3; + "~WWW::Mechanize" 3; + "~WWWOFFLE" 3; + "~Xaldon_WebSpider" 3; + "~Xaldon\ WebSpider" 3; + "~Xenu" 3; + "~Y!J-ASR" 3; + "~YoudaoBot" 3; + "~Zade" 3; + "~Zermelo" 3; + "~Zeus" 3; + "~Zgrab" 3; + "~Zitebot" 3; + "~ZmEu" 3; + "~ZumBot" 3; + "~ZyBorg" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/travisCI/globalblacklist.tpl.conf b/travisCI/globalblacklist.tpl.conf index 2c8eb45e6..494413537 100644 --- a/travisCI/globalblacklist.tpl.conf +++ b/travisCI/globalblacklist.tpl.conf @@ -17,11 +17,11 @@ ### - https://github.com/oohnoitz/nginx-blacklist ### Last Updated -### Wed Apr 19 13:00:08 SAST 2017 +### Wed Apr 19 13:11:25 SAST 2017 ### End Last Updated ### Generated in -### 0.318285942078 seconds +### 0.330133199692 seconds ### End Generated in ### Tested on: nginx/1.10.0 (Ubuntu 16.04) @@ -128,35 +128,35 @@ map $http_user_agent $bad_bot { # *********************************************** # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "^adidxbot" 0; - "^AdsBot-Google" 0; - "^aolbuild" 0; - "^bingbot" 0; - "^bingpreview" 0; - "^DoCoMo" 0; - "^duckduckgo" 0; - "^facebookexternalhit" 0; - "^Feedfetcher-Google" 0; - "^Googlebot" 0; - "^Googlebot-Image" 0; - "^Googlebot-Mobile" 0; - "^Googlebot-News" 0; - "^Googlebot/Test" 0; - "^Googlebot-Video" 0; - "^Google-HTTP-Java-Client" 0; - "^gsa-crawler" 0; - "^Jakarta\ Commons" 0; - "^Kraken/0.1" 0; - "^LinkedInBot" 0; - "^Mediapartners-Google" 0; - "^msnbot" 0; - "^msnbot-media" 0; - "^SAMSUNG" 0; - "^slurp" 0; - "^teoma" 0; - "^TwitterBot" 0; - "^Wordpress" 0; - "^yahoo" 0; + "~adidxbot" 0; + "~AdsBot-Google" 0; + "~aolbuild" 0; + "~bingbot" 0; + "~bingpreview" 0; + "~DoCoMo" 0; + "~duckduckgo" 0; + "~facebookexternalhit" 0; + "~Feedfetcher-Google" 0; + "~Googlebot" 0; + "~Googlebot-Image" 0; + "~Googlebot-Mobile" 0; + "~Googlebot-News" 0; + "~Googlebot/Test" 0; + "~Googlebot-Video" 0; + "~Google-HTTP-Java-Client" 0; + "~gsa-crawler" 0; + "~Jakarta\ Commons" 0; + "~Kraken/0.1" 0; + "~LinkedInBot" 0; + "~Mediapartners-Google" 0; + "~msnbot" 0; + "~msnbot-media" 0; + "~SAMSUNG" 0; + "~slurp" 0; + "~teoma" 0; + "~TwitterBot" 0; + "~Wordpress" 0; + "~yahoo" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************** @@ -166,11 +166,11 @@ map $http_user_agent $bad_bot { # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "^jetmon" 1; - "^libwww-perl" 1; - "^Lynx" 1; - "^munin" 1; - "^Wget/1.15" 1; + "~jetmon" 1; + "~libwww-perl" 1; + "~Lynx" 1; + "~munin" 1; + "~Wget/1.15" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************************** @@ -178,22 +178,22 @@ map $http_user_agent $bad_bot { # ************************************************************** # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "^Alexa" 2; - "^archive.org" 2; - "^Baidu" 2; - "^FlipboardProxy" 2; - "^ia_archiver" 2; - "^Mozilla/4.0" 2; - "^MSIE\ 7.0" 2; - "^Presto" 2; - "^R6_CommentReader" 2; - "^R6_FeedFetcher" 2; - "^RPT-HTTPClient" 2; - "^sfFeedReader/0.9" 2; - "^Spaidu" 2; - "^UptimeRobot/2.0" 2; - "^YandexBot" 2; - "^YandexImages" 2; + "~Alexa" 2; + "~archive.org" 2; + "~Baidu" 2; + "~FlipboardProxy" 2; + "~ia_archiver" 2; + "~Mozilla/4.0" 2; + "~MSIE\ 7.0" 2; + "~Presto" 2; + "~R6_CommentReader" 2; + "~R6_FeedFetcher" 2; + "~RPT-HTTPClient" 2; + "~sfFeedReader/0.9" 2; + "~Spaidu" 2; + "~UptimeRobot/2.0" 2; + "~YandexBot" 2; + "~YandexImages" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ********************************************* @@ -211,476 +211,476 @@ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisC # END BLACKLISTED USER AGENTS ### DO NOT EDIT THIS LINE AT ALL ### # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "^360Spider" 3; - "^80legs" 3; - "^Abonti" 3; - "^Aboundex" 3; - "^Acunetix" 3; - "^AhrefsBot" 3; - "^AIBOT" 3; - "^AiHitBot" 3; - "^Aipbot" 3; - "^Alexibot" 3; - "^Alligator" 3; - "^AllSubmitter" 3; - "^Anarchie" 3; - "^Apexoo" 3; - "^ASPSeek" 3; - "^Asterias" 3; - "^Attach" 3; - "^autoemailspider" 3; - "^BackDoorBot" 3; - "^BackStreet" 3; - "^BackWeb" 3; - "^Badass" 3; - "^Bandit" 3; - "^BatchFTP" 3; - "^Battleztar\ Bazinga" 3; - "^BBBike" 3; - "^BDFetch" 3; - "^BetaBot" 3; - "^Bigfoot" 3; - "^Bitacle" 3; - "^Black\ Hole" 3; - "^BlackWidow" 3; - "^BLEXBot" 3; - "^Blow" 3; - "^BlowFish" 3; - "^Boardreader" 3; - "^Bolt" 3; - "^BotALot" 3; - "^Brandprotect" 3; - "^Brandwatch" 3; - "^Bubing" 3; - "^Buddy" 3; - "^BuiltBotTough" 3; - "^BuiltWith" 3; - "^Bullseye" 3; - "^BunnySlippers" 3; - "^BuzzSumo" 3; - "^Calculon" 3; - "^CazoodleBot" 3; - "^CCBot" 3; - "^Cegbfeieh" 3; - "^CheeseBot" 3; - "^CherryPicker" 3; - "^ChinaClaw" 3; - "^Chlooe" 3; - "^Claritybot" 3; - "^Cliqzbot" 3; - "^Cogentbot" 3; - "^Collector" 3; - "^Copier" 3; - "^CopyRightCheck" 3; - "^Copyscape" 3; - "^Cosmos" 3; - "^Craftbot" 3; - "^CrazyWebCrawler" 3; - "^Crescent" 3; - "^CSHttp" 3; - "^Curious" 3; - "^Custo" 3; - "^DBLBot" 3; - "^Demon" 3; - "^Deusu" 3; - "^Devil" 3; - "^DIIbot" 3; - "^Dirbuster" 3; - "^Disco" 3; - "^Discobot" 3; - "^Discoverybot" 3; - "^DittoSpyder" 3; - "^DomainAppender" 3; - "^DomainSigmaCrawler" 3; - "^Dotbot" 3; - "^Download\ Demon" 3; - "^Download\ Devil" 3; - "^Download\ Wonder" 3; - "^Dragonfly" 3; - "^Drip" 3; - "^DTS\ Agent" 3; - "^EasyDL" 3; - "^Ebingbong" 3; - "^eCatch" 3; - "^ECCP/1.0" 3; - "^Ecxi" 3; - "^EirGrabber" 3; - "^EMail\ Collector" 3; - "^EMail\ Extractor" 3; - "^EMail\ Siphon" 3; - "^EMail\ Wolf" 3; - "^EroCrawler" 3; - "^Evil" 3; - "^Exabot" 3; - "^Express\ WebPictures" 3; - "^Extractor" 3; - "^ExtractorPro" 3; - "^EyeNetIE" 3; - "^Ezooms" 3; - "^FHscan" 3; - "^Fimap" 3; - "^Findxbot" 3; - "^Firefox/7.0" 3; - "^FlashGet" 3; - "^Flunky" 3; - "^Foobot" 3; - "^Freeuploader" 3; - "^FrontPage" 3; - "^GalaxyBot" 3; - "^Genieo" 3; - "^Getintent" 3; - "^GetRight" 3; - "^GetWeb" 3; - "^Gigablast" 3; - "^Gigabot" 3; - "^Go-Ahead-Got-It" 3; - "^Gotit" 3; - "^GoZilla" 3; - "^Go!Zilla" 3; - "^Grabber" 3; - "^GrabNet" 3; - "^Grafula" 3; - "^GrapeFX" 3; - "^GrapeshotCrawler" 3; - "^GridBot" 3; - "^GT::WWW" 3; - "^GuzzleHttp" 3; - "^HaosouSpider" 3; - "^Harvest" 3; - "^Havij" 3; - "^Heritrix" 3; - "^Hloader" 3; - "^HMView" 3; - "^HTMLparser" 3; - "^HTTP::Lite" 3; - "^HTTrack" 3; - "^Humanlinks" 3; - "^Iblog" 3; - "^IDBot" 3; - "^Id-search" 3; - "^IlseBot" 3; - "^Image\ Fetch" 3; - "^Image\ Stripper" 3; - "^Image\ Sucker" 3; - "^Indy\ Library" 3; - "^InfoNaviRobot" 3; - "^InfoTekies" 3; - "^Intelliseek" 3; - "^InterGET" 3; - "^Internet\ Ninja" 3; - "^InternetSeer" 3; - "^internetVista\ monitor" 3; - "^Iria" 3; - "^IRLbot" 3; - "^Iskanie" 3; - "^JamesBOT" 3; - "^Jbrofuzz" 3; - "^JennyBot" 3; - "^JetCar" 3; - "^JikeSpider" 3; - "^JOC\ Web\ Spider" 3; - "^Joomla" 3; - "^JustView" 3; - "^Jyxobot" 3; - "^Kenjin\ Spider" 3; - "^Keyword\ Density" 3; - "^Kraken" 3; - "^Lanshanbot" 3; - "^Larbin" 3; - "^LeechFTP" 3; - "^LeechGet" 3; - "^LexiBot" 3; - "^Lftp" 3; - "^LibWeb" 3; - "^Libwhisker" 3; - "^Lightspeedsystems" 3; - "^Likse" 3; - "^Linkdexbot" 3; - "^LinkextractorPro" 3; - "^LinkpadBot" 3; - "^LinkScan" 3; - "^LinksManager" 3; - "^LinkWalker" 3; - "^LinqiaMetadataDownloaderBot" 3; - "^LinqiaRSSBot" 3; - "^LinqiaScrapeBot" 3; - "^Lipperhey" 3; - "^Litemage_walker" 3; - "^Lmspider" 3; - "^LNSpiderguy" 3; - "^Ltx71" 3; - "^lwp-request" 3; - "^LWP::Simple" 3; - "^lwp-trivial" 3; - "^Magnet" 3; - "^Mag-Net" 3; - "^magpie-crawler" 3; - "^Mail.ru" 3; - "^Majestic12" 3; - "^MarkMonitor" 3; - "^MarkWatch" 3; - "^Masscan" 3; - "^Mass\ Downloader" 3; - "^Mata\ Hari" 3; - "^Meanpathbot" 3; - "^MegaIndex.ru" 3; - "^Metauri" 3; - "^MFC_Tear_Sample" 3; - "^Microsoft\ Data\ Access" 3; - "^Microsoft\ URL\ Control" 3; - "^MIDown\ tool" 3; - "^MIIxpc" 3; - "^Mister\ PiX" 3; - "^MJ12bot" 3; - "^Mojeek" 3; - "^MSFrontPage" 3; - "^MSIE\ 6.0" 3; - "^MSIECrawler" 3; - "^Msrabot" 3; - "^MS\ Web\ Services\ Client\ Protocol" 3; - "^Musobot" 3; - "^Name\ Intelligence" 3; - "^Nameprotect" 3; - "^Navroad" 3; - "^NearSite" 3; - "^Needle" 3; - "^Nessus" 3; - "^NetAnts" 3; - "^Netcraft" 3; - "^netEstate\ NE\ Crawler" 3; - "^NetLyzer" 3; - "^NetMechanic" 3; - "^NetSpider" 3; - "^Nettrack" 3; - "^Net\ Vampire" 3; - "^NetZIP" 3; - "^NextGenSearchBot" 3; - "^Nibbler" 3; - "^NICErsPRO" 3; - "^Niki-bot" 3; - "^Nikto" 3; - "^NimbleCrawler" 3; - "^Ninja" 3; - "^Nmap" 3; - "^NPbot" 3; - "^Nutch" 3; - "^Octopus" 3; - "^Offline\ Explorer" 3; - "^Offline\ Navigator" 3; - "^Openfind" 3; - "^OpenLinkProfiler" 3; - "^Openvas" 3; - "^OrangeBot" 3; - "^OrangeSpider" 3; - "^OutfoxBot" 3; - "^PageAnalyzer" 3; - "^Page\ Analyzer" 3; - "^PageGrabber" 3; - "^Page\ Grabber" 3; - "^page\ scorer" 3; - "^PageScorer" 3; - "^Panscient" 3; - "^Papa\ Foto" 3; - "^Pavuk" 3; - "^pcBrowser" 3; - "^PECL::HTTP" 3; - "^PeoplePal" 3; - "^PHPCrawl" 3; - "^Picscout" 3; - "^Picsearch" 3; - "^Pimonster" 3; - "^Pi-Monster" 3; - "^Pixray" 3; - "^PleaseCrawl" 3; - "^Pockey" 3; - "^POE-Component-Client-HTTP" 3; - "^Probethenet" 3; - "^ProPowerBot" 3; - "^ProWebWalker" 3; - "^Proximic" 3; - "^Psbot" 3; - "^Pump" 3; - "^PyCurl" 3; - "^QueryN\ Metasearch" 3; - "^Qwantify" 3; - "^RankActiveLinkBot" 3; - "^Rankivabot" 3; - "^RealDownload" 3; - "^Reaper" 3; - "^Recorder" 3; - "^RedesScrapy" 3; - "^ReGet" 3; - "^RepoMonkey" 3; - "^Ripper" 3; - "^RocketCrawler" 3; - "^Rogerbot" 3; - "^SBIder" 3; - "^ScanAlert" 3; - "^Scanbot" 3; - "^Scrapy" 3; - "^Screaming\ Frog\ SEO\ Spider" 3; - "^ScreenerBot" 3; - "^Searchestate" 3; - "^SearchmetricsBot" 3; - "^Semrush" 3; - "^SemrushBot" 3; - "^SEOkicks" 3; - "^SEOkicks-Robot" 3; - "^SEOlyticsCrawler" 3; - "^Seomoz" 3; - "^SEOprofiler" 3; - "^SEOstats" 3; - "^SeznamBot" 3; - "^Siphon" 3; - "^SISTRIX" 3; - "^SISTRIX\ Crawler" 3; - "^Sitebeam" 3; - "^SiteExplorer" 3; - "^Siteimprove" 3; - "^SiteLockSpider" 3; - "^SiteSnagger" 3; - "^SiteSucker" 3; - "^Site\ Sucker" 3; - "^Sitevigil" 3; - "^Slackbot-LinkExpanding" 3; - "^SlySearch" 3; - "^SmartDownload" 3; - "^Snake" 3; - "^Snapbot" 3; - "^Snoopy" 3; - "^SocialRankIOBot" 3; - "^Sogou\ web\ spider" 3; - "^Sosospider" 3; - "^SpaceBison" 3; - "^Spammen" 3; - "^SpankBot" 3; - "^Spanner" 3; - "^Spbot" 3; - "^Spinn3r" 3; - "^SputnikBot" 3; - "^Sqlmap" 3; - "^Sqlworm" 3; - "^Sqworm" 3; - "^Steeler" 3; - "^Stripper" 3; - "^Sucker" 3; - "^Sucuri" 3; - "^SuperBot" 3; - "^SuperHTTP" 3; - "^Surfbot" 3; - "^SurveyBot" 3; - "^Suzuran" 3; - "^Swiftbot" 3; - "^Szukacz" 3; - "^T0PHackTeam" 3; - "^T8Abot" 3; - "^tAkeOut" 3; - "^Teleport" 3; - "^TeleportPro" 3; - "^Telesoft" 3; - "^Telesphoreo" 3; - "^Telesphorep" 3; - "^The\ Intraformant" 3; - "^TheNomad" 3; - "^TightTwatBot" 3; - "^Titan" 3; - "^Toata" 3; - "^Toweyabot" 3; - "^Trendictionbot" 3; - "^True_Robot" 3; - "^Turingos" 3; - "^TurnitinBot" 3; - "^Turnitin\ Bot" 3; - "^Turnitin\ Robot" 3; - "^TwengaBot" 3; - "^Twice" 3; - "^Typhoeus" 3; - "^URLy.Warning" 3; - "^URLy\ Warning" 3; - "^Vacuum" 3; - "^Vagabondo" 3; - "^VB\ Project" 3; - "^VCI" 3; - "^VeriCiteCrawler" 3; - "^VidibleScraper" 3; - "^VoidEYE" 3; - "^Voil" 3; - "^Voltron" 3; - "^WallpapersHD" 3; - "^WASALive-Bot" 3; - "^WBSearchBot" 3; - "^Webalta" 3; - "^WebAuto" 3; - "^Web\ Auto" 3; - "^WebBandit" 3; - "^Web\ Bandit" 3; - "^WebCollage" 3; - "^Web\ Collage" 3; - "^WebCopier" 3; - "^Web\ Copier" 3; - "^WEBDAV" 3; - "^WEBDAV\ Client" 3; - "^WebEnhancer" 3; - "^Web\ Enhancer" 3; - "^WebFetch" 3; - "^Web\ Fetch" 3; - "^WebFuck" 3; - "^Web\ Fuck" 3; - "^WebGo\ IS" 3; - "^WebImageCollector" 3; - "^Web\ Image\ Collector" 3; - "^WebLeacher" 3; - "^WebmasterWorldForumBot" 3; - "^webmeup-crawler" 3; - "^WebPix" 3; - "^Web\ Pix" 3; - "^WebReaper" 3; - "^Web\ Reaper" 3; - "^WebSauger" 3; - "^Web\ Sauger" 3; - "^Webshag" 3; - "^WebsiteExtractor" 3; - "^Website\ Extractor" 3; - "^WebsiteQuester" 3; - "^Website\ Quester" 3; - "^Webster" 3; - "^WebStripper" 3; - "^Web\ Stripper" 3; - "^WebSucker" 3; - "^Web\ Sucker" 3; - "^WebWhacker" 3; - "^Web\ Whacker" 3; - "^WebZIP" 3; - "^WeSEE" 3; - "^Whack" 3; - "^Whacker" 3; - "^Whatweb" 3; - "^Widow" 3; - "^WinHTTrack" 3; - "^WiseGuys\ Robot" 3; - "^WISENutbot" 3; - "^Wonderbot" 3; - "^Woobot" 3; - "^Wotbox" 3; - "^Wprecon" 3; - "^WPScan" 3; - "^WWW-Collector-E" 3; - "^WWW-Mechanize" 3; - "^WWW::Mechanize" 3; - "^WWWOFFLE" 3; - "^Xaldon_WebSpider" 3; - "^Xaldon\ WebSpider" 3; - "^Xenu" 3; - "^Y!J-ASR" 3; - "^YoudaoBot" 3; - "^Zade" 3; - "^Zermelo" 3; - "^Zeus" 3; - "^Zgrab" 3; - "^Zitebot" 3; - "^ZmEu" 3; - "^ZumBot" 3; - "^ZyBorg" 3; + "~360Spider" 3; + "~80legs" 3; + "~Abonti" 3; + "~Aboundex" 3; + "~Acunetix" 3; + "~AhrefsBot" 3; + "~AIBOT" 3; + "~AiHitBot" 3; + "~Aipbot" 3; + "~Alexibot" 3; + "~Alligator" 3; + "~AllSubmitter" 3; + "~Anarchie" 3; + "~Apexoo" 3; + "~ASPSeek" 3; + "~Asterias" 3; + "~Attach" 3; + "~autoemailspider" 3; + "~BackDoorBot" 3; + "~BackStreet" 3; + "~BackWeb" 3; + "~Badass" 3; + "~Bandit" 3; + "~BatchFTP" 3; + "~Battleztar\ Bazinga" 3; + "~BBBike" 3; + "~BDFetch" 3; + "~BetaBot" 3; + "~Bigfoot" 3; + "~Bitacle" 3; + "~Black\ Hole" 3; + "~BlackWidow" 3; + "~BLEXBot" 3; + "~Blow" 3; + "~BlowFish" 3; + "~Boardreader" 3; + "~Bolt" 3; + "~BotALot" 3; + "~Brandprotect" 3; + "~Brandwatch" 3; + "~Bubing" 3; + "~Buddy" 3; + "~BuiltBotTough" 3; + "~BuiltWith" 3; + "~Bullseye" 3; + "~BunnySlippers" 3; + "~BuzzSumo" 3; + "~Calculon" 3; + "~CazoodleBot" 3; + "~CCBot" 3; + "~Cegbfeieh" 3; + "~CheeseBot" 3; + "~CherryPicker" 3; + "~ChinaClaw" 3; + "~Chlooe" 3; + "~Claritybot" 3; + "~Cliqzbot" 3; + "~Cogentbot" 3; + "~Collector" 3; + "~Copier" 3; + "~CopyRightCheck" 3; + "~Copyscape" 3; + "~Cosmos" 3; + "~Craftbot" 3; + "~CrazyWebCrawler" 3; + "~Crescent" 3; + "~CSHttp" 3; + "~Curious" 3; + "~Custo" 3; + "~DBLBot" 3; + "~Demon" 3; + "~Deusu" 3; + "~Devil" 3; + "~DIIbot" 3; + "~Dirbuster" 3; + "~Disco" 3; + "~Discobot" 3; + "~Discoverybot" 3; + "~DittoSpyder" 3; + "~DomainAppender" 3; + "~DomainSigmaCrawler" 3; + "~Dotbot" 3; + "~Download\ Demon" 3; + "~Download\ Devil" 3; + "~Download\ Wonder" 3; + "~Dragonfly" 3; + "~Drip" 3; + "~DTS\ Agent" 3; + "~EasyDL" 3; + "~Ebingbong" 3; + "~eCatch" 3; + "~ECCP/1.0" 3; + "~Ecxi" 3; + "~EirGrabber" 3; + "~EMail\ Collector" 3; + "~EMail\ Extractor" 3; + "~EMail\ Siphon" 3; + "~EMail\ Wolf" 3; + "~EroCrawler" 3; + "~Evil" 3; + "~Exabot" 3; + "~Express\ WebPictures" 3; + "~Extractor" 3; + "~ExtractorPro" 3; + "~EyeNetIE" 3; + "~Ezooms" 3; + "~FHscan" 3; + "~Fimap" 3; + "~Findxbot" 3; + "~Firefox/7.0" 3; + "~FlashGet" 3; + "~Flunky" 3; + "~Foobot" 3; + "~Freeuploader" 3; + "~FrontPage" 3; + "~GalaxyBot" 3; + "~Genieo" 3; + "~Getintent" 3; + "~GetRight" 3; + "~GetWeb" 3; + "~Gigablast" 3; + "~Gigabot" 3; + "~Go-Ahead-Got-It" 3; + "~Gotit" 3; + "~GoZilla" 3; + "~Go!Zilla" 3; + "~Grabber" 3; + "~GrabNet" 3; + "~Grafula" 3; + "~GrapeFX" 3; + "~GrapeshotCrawler" 3; + "~GridBot" 3; + "~GT::WWW" 3; + "~GuzzleHttp" 3; + "~HaosouSpider" 3; + "~Harvest" 3; + "~Havij" 3; + "~Heritrix" 3; + "~Hloader" 3; + "~HMView" 3; + "~HTMLparser" 3; + "~HTTP::Lite" 3; + "~HTTrack" 3; + "~Humanlinks" 3; + "~Iblog" 3; + "~IDBot" 3; + "~Id-search" 3; + "~IlseBot" 3; + "~Image\ Fetch" 3; + "~Image\ Stripper" 3; + "~Image\ Sucker" 3; + "~Indy\ Library" 3; + "~InfoNaviRobot" 3; + "~InfoTekies" 3; + "~Intelliseek" 3; + "~InterGET" 3; + "~Internet\ Ninja" 3; + "~InternetSeer" 3; + "~internetVista\ monitor" 3; + "~Iria" 3; + "~IRLbot" 3; + "~Iskanie" 3; + "~JamesBOT" 3; + "~Jbrofuzz" 3; + "~JennyBot" 3; + "~JetCar" 3; + "~JikeSpider" 3; + "~JOC\ Web\ Spider" 3; + "~Joomla" 3; + "~JustView" 3; + "~Jyxobot" 3; + "~Kenjin\ Spider" 3; + "~Keyword\ Density" 3; + "~Kraken" 3; + "~Lanshanbot" 3; + "~Larbin" 3; + "~LeechFTP" 3; + "~LeechGet" 3; + "~LexiBot" 3; + "~Lftp" 3; + "~LibWeb" 3; + "~Libwhisker" 3; + "~Lightspeedsystems" 3; + "~Likse" 3; + "~Linkdexbot" 3; + "~LinkextractorPro" 3; + "~LinkpadBot" 3; + "~LinkScan" 3; + "~LinksManager" 3; + "~LinkWalker" 3; + "~LinqiaMetadataDownloaderBot" 3; + "~LinqiaRSSBot" 3; + "~LinqiaScrapeBot" 3; + "~Lipperhey" 3; + "~Litemage_walker" 3; + "~Lmspider" 3; + "~LNSpiderguy" 3; + "~Ltx71" 3; + "~lwp-request" 3; + "~LWP::Simple" 3; + "~lwp-trivial" 3; + "~Magnet" 3; + "~Mag-Net" 3; + "~magpie-crawler" 3; + "~Mail.ru" 3; + "~Majestic12" 3; + "~MarkMonitor" 3; + "~MarkWatch" 3; + "~Masscan" 3; + "~Mass\ Downloader" 3; + "~Mata\ Hari" 3; + "~Meanpathbot" 3; + "~MegaIndex.ru" 3; + "~Metauri" 3; + "~MFC_Tear_Sample" 3; + "~Microsoft\ Data\ Access" 3; + "~Microsoft\ URL\ Control" 3; + "~MIDown\ tool" 3; + "~MIIxpc" 3; + "~Mister\ PiX" 3; + "~MJ12bot" 3; + "~Mojeek" 3; + "~MSFrontPage" 3; + "~MSIE\ 6.0" 3; + "~MSIECrawler" 3; + "~Msrabot" 3; + "~MS\ Web\ Services\ Client\ Protocol" 3; + "~Musobot" 3; + "~Name\ Intelligence" 3; + "~Nameprotect" 3; + "~Navroad" 3; + "~NearSite" 3; + "~Needle" 3; + "~Nessus" 3; + "~NetAnts" 3; + "~Netcraft" 3; + "~netEstate\ NE\ Crawler" 3; + "~NetLyzer" 3; + "~NetMechanic" 3; + "~NetSpider" 3; + "~Nettrack" 3; + "~Net\ Vampire" 3; + "~NetZIP" 3; + "~NextGenSearchBot" 3; + "~Nibbler" 3; + "~NICErsPRO" 3; + "~Niki-bot" 3; + "~Nikto" 3; + "~NimbleCrawler" 3; + "~Ninja" 3; + "~Nmap" 3; + "~NPbot" 3; + "~Nutch" 3; + "~Octopus" 3; + "~Offline\ Explorer" 3; + "~Offline\ Navigator" 3; + "~Openfind" 3; + "~OpenLinkProfiler" 3; + "~Openvas" 3; + "~OrangeBot" 3; + "~OrangeSpider" 3; + "~OutfoxBot" 3; + "~PageAnalyzer" 3; + "~Page\ Analyzer" 3; + "~PageGrabber" 3; + "~Page\ Grabber" 3; + "~page\ scorer" 3; + "~PageScorer" 3; + "~Panscient" 3; + "~Papa\ Foto" 3; + "~Pavuk" 3; + "~pcBrowser" 3; + "~PECL::HTTP" 3; + "~PeoplePal" 3; + "~PHPCrawl" 3; + "~Picscout" 3; + "~Picsearch" 3; + "~Pimonster" 3; + "~Pi-Monster" 3; + "~Pixray" 3; + "~PleaseCrawl" 3; + "~Pockey" 3; + "~POE-Component-Client-HTTP" 3; + "~Probethenet" 3; + "~ProPowerBot" 3; + "~ProWebWalker" 3; + "~Proximic" 3; + "~Psbot" 3; + "~Pump" 3; + "~PyCurl" 3; + "~QueryN\ Metasearch" 3; + "~Qwantify" 3; + "~RankActiveLinkBot" 3; + "~Rankivabot" 3; + "~RealDownload" 3; + "~Reaper" 3; + "~Recorder" 3; + "~RedesScrapy" 3; + "~ReGet" 3; + "~RepoMonkey" 3; + "~Ripper" 3; + "~RocketCrawler" 3; + "~Rogerbot" 3; + "~SBIder" 3; + "~ScanAlert" 3; + "~Scanbot" 3; + "~Scrapy" 3; + "~Screaming\ Frog\ SEO\ Spider" 3; + "~ScreenerBot" 3; + "~Searchestate" 3; + "~SearchmetricsBot" 3; + "~Semrush" 3; + "~SemrushBot" 3; + "~SEOkicks" 3; + "~SEOkicks-Robot" 3; + "~SEOlyticsCrawler" 3; + "~Seomoz" 3; + "~SEOprofiler" 3; + "~SEOstats" 3; + "~SeznamBot" 3; + "~Siphon" 3; + "~SISTRIX" 3; + "~SISTRIX\ Crawler" 3; + "~Sitebeam" 3; + "~SiteExplorer" 3; + "~Siteimprove" 3; + "~SiteLockSpider" 3; + "~SiteSnagger" 3; + "~SiteSucker" 3; + "~Site\ Sucker" 3; + "~Sitevigil" 3; + "~Slackbot-LinkExpanding" 3; + "~SlySearch" 3; + "~SmartDownload" 3; + "~Snake" 3; + "~Snapbot" 3; + "~Snoopy" 3; + "~SocialRankIOBot" 3; + "~Sogou\ web\ spider" 3; + "~Sosospider" 3; + "~SpaceBison" 3; + "~Spammen" 3; + "~SpankBot" 3; + "~Spanner" 3; + "~Spbot" 3; + "~Spinn3r" 3; + "~SputnikBot" 3; + "~Sqlmap" 3; + "~Sqlworm" 3; + "~Sqworm" 3; + "~Steeler" 3; + "~Stripper" 3; + "~Sucker" 3; + "~Sucuri" 3; + "~SuperBot" 3; + "~SuperHTTP" 3; + "~Surfbot" 3; + "~SurveyBot" 3; + "~Suzuran" 3; + "~Swiftbot" 3; + "~Szukacz" 3; + "~T0PHackTeam" 3; + "~T8Abot" 3; + "~tAkeOut" 3; + "~Teleport" 3; + "~TeleportPro" 3; + "~Telesoft" 3; + "~Telesphoreo" 3; + "~Telesphorep" 3; + "~The\ Intraformant" 3; + "~TheNomad" 3; + "~TightTwatBot" 3; + "~Titan" 3; + "~Toata" 3; + "~Toweyabot" 3; + "~Trendictionbot" 3; + "~True_Robot" 3; + "~Turingos" 3; + "~TurnitinBot" 3; + "~Turnitin\ Bot" 3; + "~Turnitin\ Robot" 3; + "~TwengaBot" 3; + "~Twice" 3; + "~Typhoeus" 3; + "~URLy.Warning" 3; + "~URLy\ Warning" 3; + "~Vacuum" 3; + "~Vagabondo" 3; + "~VB\ Project" 3; + "~VCI" 3; + "~VeriCiteCrawler" 3; + "~VidibleScraper" 3; + "~VoidEYE" 3; + "~Voil" 3; + "~Voltron" 3; + "~WallpapersHD" 3; + "~WASALive-Bot" 3; + "~WBSearchBot" 3; + "~Webalta" 3; + "~WebAuto" 3; + "~Web\ Auto" 3; + "~WebBandit" 3; + "~Web\ Bandit" 3; + "~WebCollage" 3; + "~Web\ Collage" 3; + "~WebCopier" 3; + "~Web\ Copier" 3; + "~WEBDAV" 3; + "~WEBDAV\ Client" 3; + "~WebEnhancer" 3; + "~Web\ Enhancer" 3; + "~WebFetch" 3; + "~Web\ Fetch" 3; + "~WebFuck" 3; + "~Web\ Fuck" 3; + "~WebGo\ IS" 3; + "~WebImageCollector" 3; + "~Web\ Image\ Collector" 3; + "~WebLeacher" 3; + "~WebmasterWorldForumBot" 3; + "~webmeup-crawler" 3; + "~WebPix" 3; + "~Web\ Pix" 3; + "~WebReaper" 3; + "~Web\ Reaper" 3; + "~WebSauger" 3; + "~Web\ Sauger" 3; + "~Webshag" 3; + "~WebsiteExtractor" 3; + "~Website\ Extractor" 3; + "~WebsiteQuester" 3; + "~Website\ Quester" 3; + "~Webster" 3; + "~WebStripper" 3; + "~Web\ Stripper" 3; + "~WebSucker" 3; + "~Web\ Sucker" 3; + "~WebWhacker" 3; + "~Web\ Whacker" 3; + "~WebZIP" 3; + "~WeSEE" 3; + "~Whack" 3; + "~Whacker" 3; + "~Whatweb" 3; + "~Widow" 3; + "~WinHTTrack" 3; + "~WiseGuys\ Robot" 3; + "~WISENutbot" 3; + "~Wonderbot" 3; + "~Woobot" 3; + "~Wotbox" 3; + "~Wprecon" 3; + "~WPScan" 3; + "~WWW-Collector-E" 3; + "~WWW-Mechanize" 3; + "~WWW::Mechanize" 3; + "~WWWOFFLE" 3; + "~Xaldon_WebSpider" 3; + "~Xaldon\ WebSpider" 3; + "~Xenu" 3; + "~Y!J-ASR" 3; + "~YoudaoBot" 3; + "~Zade" 3; + "~Zermelo" 3; + "~Zeus" 3; + "~Zgrab" 3; + "~Zitebot" 3; + "~ZmEu" 3; + "~ZumBot" 3; + "~ZyBorg" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### }