diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index afa89da56..4488686ba 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -17,11 +17,11 @@ ### - https://github.com/oohnoitz/nginx-blacklist ### Last Updated -### Wed Apr 19 12:42:01 SAST 2017 +### Wed Apr 19 12:53:26 SAST 2017 ### End Last Updated ### Generated in -### 0.301798343658 seconds +### 0.299472808838 seconds ### End Generated in ### Tested on: nginx/1.10.0 (Ubuntu 16.04) @@ -128,35 +128,35 @@ map $http_user_agent $bad_bot { # *********************************************** # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*adidxbot" 0; - "~*AdsBot-Google" 0; - "~*aolbuild" 0; - "~*bingbot" 0; - "~*bingpreview" 0; - "~*DoCoMo" 0; - "~*duckduckgo" 0; - "~*facebookexternalhit" 0; - "~*Feedfetcher-Google" 0; - "~*Googlebot" 0; - "~*Googlebot-Image" 0; - "~*Googlebot-Mobile" 0; - "~*Googlebot-News" 0; - "~*Googlebot/Test" 0; - "~*Googlebot-Video" 0; - "~*Google-HTTP-Java-Client" 0; - "~*gsa-crawler" 0; - "~*Jakarta\ Commons" 0; - "~*Kraken/0.1" 0; - "~*LinkedInBot" 0; - "~*Mediapartners-Google" 0; - "~*msnbot" 0; - "~*msnbot-media" 0; - "~*SAMSUNG" 0; - "~*slurp" 0; - "~*teoma" 0; - "~*TwitterBot" 0; - "~*Wordpress" 0; - "~*yahoo" 0; + "^adidxbot" 0; + "^AdsBot-Google" 0; + "^aolbuild" 0; + "^bingbot" 0; + "^bingpreview" 0; + "^DoCoMo" 0; + "^duckduckgo" 0; + "^facebookexternalhit" 0; + "^Feedfetcher-Google" 0; + "^Googlebot" 0; + "^Googlebot-Image" 0; + "^Googlebot-Mobile" 0; + "^Googlebot-News" 0; + "^Googlebot/Test" 0; + "^Googlebot-Video" 0; + "^Google-HTTP-Java-Client" 0; + "^gsa-crawler" 0; + "^Jakarta\ Commons" 0; + "^Kraken/0.1" 0; + "^LinkedInBot" 0; + "^Mediapartners-Google" 0; + "^msnbot" 0; + "^msnbot-media" 0; + "^SAMSUNG" 0; + "^slurp" 0; + "^teoma" 0; + "^TwitterBot" 0; + "^Wordpress" 0; + "^yahoo" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************** @@ -166,12 +166,12 @@ map $http_user_agent $bad_bot { # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*jetmon" 1; - "~*libwww-perl" 1; - "~*Lynx" 1; - "~*munin" 1; - "~*Wget/1.15" 1; - "~*WordPress" 1; + "^jetmon" 1; + "^libwww-perl" 1; + "^Lynx" 1; + "^munin" 1; + "^Wget/1.15" 1; + "^WordPress" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************************** @@ -179,22 +179,22 @@ map $http_user_agent $bad_bot { # ************************************************************** # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*Alexa" 2; - "~*archive.org" 2; - "~*Baidu" 2; - "~*FlipboardProxy" 2; - "~*ia_archiver" 2; - "~*Mozilla/4.0" 2; - "~*MSIE\ 7.0" 2; - "~*Presto" 2; - "~*R6_CommentReader" 2; - "~*R6_FeedFetcher" 2; - "~*RPT-HTTPClient" 2; - "~*sfFeedReader/0.9" 2; - "~*Spaidu" 2; - "~*UptimeRobot/2.0" 2; - "~*YandexBot" 2; - "~*YandexImages" 2; + "^Alexa" 2; + "^archive.org" 2; + "^Baidu" 2; + "^FlipboardProxy" 2; + "^ia_archiver" 2; + "^Mozilla/4.0" 2; + "^MSIE\ 7.0" 2; + "^Presto" 2; + "^R6_CommentReader" 2; + "^R6_FeedFetcher" 2; + "^RPT-HTTPClient" 2; + "^sfFeedReader/0.9" 2; + "^Spaidu" 2; + "^UptimeRobot/2.0" 2; + "^YandexBot" 2; + "^YandexImages" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ********************************************* @@ -212,476 +212,476 @@ map $http_user_agent $bad_bot { # END BLACKLISTED USER AGENTS ### DO NOT EDIT THIS LINE AT ALL ### # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*360Spider" 3; - "~*80legs" 3; - "~*Abonti" 3; - "~*Aboundex" 3; - "~*Acunetix" 3; - "~*AhrefsBot" 3; - "~*AIBOT" 3; - "~*AiHitBot" 3; - "~*Aipbot" 3; - "~*Alexibot" 3; - "~*Alligator" 3; - "~*AllSubmitter" 3; - "~*Anarchie" 3; - "~*Apexoo" 3; - "~*ASPSeek" 3; - "~*Asterias" 3; - "~*Attach" 3; - "~*autoemailspider" 3; - "~*BackDoorBot" 3; - "~*BackStreet" 3; - "~*BackWeb" 3; - "~*Badass" 3; - "~*Bandit" 3; - "~*BatchFTP" 3; - "~*Battleztar\ Bazinga" 3; - "~*BBBike" 3; - "~*BDFetch" 3; - "~*BetaBot" 3; - "~*Bigfoot" 3; - "~*Bitacle" 3; - "~*Black\ Hole" 3; - "~*BlackWidow" 3; - "~*BLEXBot" 3; - "~*Blow" 3; - "~*BlowFish" 3; - "~*Boardreader" 3; - "~*Bolt" 3; - "~*BotALot" 3; - "~*Brandprotect" 3; - "~*Brandwatch" 3; - "~*Bubing" 3; - "~*Buddy" 3; - "~*BuiltBotTough" 3; - "~*BuiltWith" 3; - "~*Bullseye" 3; - "~*BunnySlippers" 3; - "~*BuzzSumo" 3; - "~*Calculon" 3; - "~*CazoodleBot" 3; - "~*CCBot" 3; - "~*Cegbfeieh" 3; - "~*CheeseBot" 3; - "~*CherryPicker" 3; - "~*ChinaClaw" 3; - "~*Chlooe" 3; - "~*Claritybot" 3; - "~*Cliqzbot" 3; - "~*Cogentbot" 3; - "~*Collector" 3; - "~*Copier" 3; - "~*CopyRightCheck" 3; - "~*Copyscape" 3; - "~*Cosmos" 3; - "~*Craftbot" 3; - "~*CrazyWebCrawler" 3; - "~*Crescent" 3; - "~*CSHttp" 3; - "~*Curious" 3; - "~*Custo" 3; - "~*DBLBot" 3; - "~*Demon" 3; - "~*Deusu" 3; - "~*Devil" 3; - "~*DIIbot" 3; - "~*Dirbuster" 3; - "~*Disco" 3; - "~*Discobot" 3; - "~*Discoverybot" 3; - "~*DittoSpyder" 3; - "~*DomainAppender" 3; - "~*DomainSigmaCrawler" 3; - "~*Dotbot" 3; - "~*Download\ Demon" 3; - "~*Download\ Devil" 3; - "~*Download\ Wonder" 3; - "~*Dragonfly" 3; - "~*Drip" 3; - "~*DTS\ Agent" 3; - "~*EasyDL" 3; - "~*Ebingbong" 3; - "~*eCatch" 3; - "~*ECCP/1.0" 3; - "~*Ecxi" 3; - "~*EirGrabber" 3; - "~*EMail\ Collector" 3; - "~*EMail\ Extractor" 3; - "~*EMail\ Siphon" 3; - "~*EMail\ Wolf" 3; - "~*EroCrawler" 3; - "~*Evil" 3; - "~*Exabot" 3; - "~*Express\ WebPictures" 3; - "~*Extractor" 3; - "~*ExtractorPro" 3; - "~*EyeNetIE" 3; - "~*Ezooms" 3; - "~*FHscan" 3; - "~*Fimap" 3; - "~*Findxbot" 3; - "~*Firefox/7.0" 3; - "~*FlashGet" 3; - "~*Flunky" 3; - "~*Foobot" 3; - "~*Freeuploader" 3; - "~*FrontPage" 3; - "~*GalaxyBot" 3; - "~*Genieo" 3; - "~*Getintent" 3; - "~*GetRight" 3; - "~*GetWeb" 3; - "~*Gigablast" 3; - "~*Gigabot" 3; - "~*Go-Ahead-Got-It" 3; - "~*Gotit" 3; - "~*GoZilla" 3; - "~*Go!Zilla" 3; - "~*Grabber" 3; - "~*GrabNet" 3; - "~*Grafula" 3; - "~*GrapeFX" 3; - "~*GrapeshotCrawler" 3; - "~*GridBot" 3; - "~*GT::WWW" 3; - "~*GuzzleHttp" 3; - "~*HaosouSpider" 3; - "~*Harvest" 3; - "~*Havij" 3; - "~*Heritrix" 3; - "~*Hloader" 3; - "~*HMView" 3; - "~*HTMLparser" 3; - "~*HTTP::Lite" 3; - "~*HTTrack" 3; - "~*Humanlinks" 3; - "~*Iblog" 3; - "~*IDBot" 3; - "~*Id-search" 3; - "~*IlseBot" 3; - "~*Image\ Fetch" 3; - "~*Image\ Stripper" 3; - "~*Image\ Sucker" 3; - "~*Indy\ Library" 3; - "~*InfoNaviRobot" 3; - "~*InfoTekies" 3; - "~*Intelliseek" 3; - "~*InterGET" 3; - "~*Internet\ Ninja" 3; - "~*InternetSeer" 3; - "~*internetVista\ monitor" 3; - "~*Iria" 3; - "~*IRLbot" 3; - "~*Iskanie" 3; - "~*JamesBOT" 3; - "~*Jbrofuzz" 3; - "~*JennyBot" 3; - "~*JetCar" 3; - "~*JikeSpider" 3; - "~*JOC\ Web\ Spider" 3; - "~*Joomla" 3; - "~*JustView" 3; - "~*Jyxobot" 3; - "~*Kenjin\ Spider" 3; - "~*Keyword\ Density" 3; - "~*Kraken" 3; - "~*Lanshanbot" 3; - "~*Larbin" 3; - "~*LeechFTP" 3; - "~*LeechGet" 3; - "~*LexiBot" 3; - "~*Lftp" 3; - "~*LibWeb" 3; - "~*Libwhisker" 3; - "~*Lightspeedsystems" 3; - "~*Likse" 3; - "~*Linkdexbot" 3; - "~*LinkextractorPro" 3; - "~*LinkpadBot" 3; - "~*LinkScan" 3; - "~*LinksManager" 3; - "~*LinkWalker" 3; - "~*LinqiaMetadataDownloaderBot" 3; - "~*LinqiaRSSBot" 3; - "~*LinqiaScrapeBot" 3; - "~*Lipperhey" 3; - "~*Litemage_walker" 3; - "~*Lmspider" 3; - "~*LNSpiderguy" 3; - "~*Ltx71" 3; - "~*lwp-request" 3; - "~*LWP::Simple" 3; - "~*lwp-trivial" 3; - "~*Magnet" 3; - "~*Mag-Net" 3; - "~*magpie-crawler" 3; - "~*Mail.ru" 3; - "~*Majestic12" 3; - "~*MarkMonitor" 3; - "~*MarkWatch" 3; - "~*Masscan" 3; - "~*Mass\ Downloader" 3; - "~*Mata\ Hari" 3; - "~*Meanpathbot" 3; - "~*MegaIndex.ru" 3; - "~*Metauri" 3; - "~*MFC_Tear_Sample" 3; - "~*Microsoft\ Data\ Access" 3; - "~*Microsoft\ URL\ Control" 3; - "~*MIDown\ tool" 3; - "~*MIIxpc" 3; - "~*Mister\ PiX" 3; - "~*MJ12bot" 3; - "~*Mojeek" 3; - "~*MSFrontPage" 3; - "~*MSIE\ 6.0" 3; - "~*MSIECrawler" 3; - "~*Msrabot" 3; - "~*MS\ Web\ Services\ Client\ Protocol" 3; - "~*Musobot" 3; - "~*Name\ Intelligence" 3; - "~*Nameprotect" 3; - "~*Navroad" 3; - "~*NearSite" 3; - "~*Needle" 3; - "~*Nessus" 3; - "~*NetAnts" 3; - "~*Netcraft" 3; - "~*netEstate\ NE\ Crawler" 3; - "~*NetLyzer" 3; - "~*NetMechanic" 3; - "~*NetSpider" 3; - "~*Nettrack" 3; - "~*Net\ Vampire" 3; - "~*NetZIP" 3; - "~*NextGenSearchBot" 3; - "~*Nibbler" 3; - "~*NICErsPRO" 3; - "~*Niki-bot" 3; - "~*Nikto" 3; - "~*NimbleCrawler" 3; - "~*Ninja" 3; - "~*Nmap" 3; - "~*NPbot" 3; - "~*Nutch" 3; - "~*Octopus" 3; - "~*Offline\ Explorer" 3; - "~*Offline\ Navigator" 3; - "~*Openfind" 3; - "~*OpenLinkProfiler" 3; - "~*Openvas" 3; - "~*OrangeBot" 3; - "~*OrangeSpider" 3; - "~*OutfoxBot" 3; - "~*PageAnalyzer" 3; - "~*Page\ Analyzer" 3; - "~*PageGrabber" 3; - "~*Page\ Grabber" 3; - "~*page\ scorer" 3; - "~*PageScorer" 3; - "~*Panscient" 3; - "~*Papa\ Foto" 3; - "~*Pavuk" 3; - "~*pcBrowser" 3; - "~*PECL::HTTP" 3; - "~*PeoplePal" 3; - "~*PHPCrawl" 3; - "~*Picscout" 3; - "~*Picsearch" 3; - "~*Pimonster" 3; - "~*Pi-Monster" 3; - "~*Pixray" 3; - "~*PleaseCrawl" 3; - "~*Pockey" 3; - "~*POE-Component-Client-HTTP" 3; - "~*Probethenet" 3; - "~*ProPowerBot" 3; - "~*ProWebWalker" 3; - "~*Proximic" 3; - "~*Psbot" 3; - "~*Pump" 3; - "~*PyCurl" 3; - "~*QueryN\ Metasearch" 3; - "~*Qwantify" 3; - "~*RankActiveLinkBot" 3; - "~*Rankivabot" 3; - "~*RealDownload" 3; - "~*Reaper" 3; - "~*Recorder" 3; - "~*RedesScrapy" 3; - "~*ReGet" 3; - "~*RepoMonkey" 3; - "~*Ripper" 3; - "~*RocketCrawler" 3; - "~*Rogerbot" 3; - "~*SBIder" 3; - "~*ScanAlert" 3; - "~*Scanbot" 3; - "~*Scrapy" 3; - "~*Screaming\ Frog\ SEO\ Spider" 3; - "~*ScreenerBot" 3; - "~*Searchestate" 3; - "~*SearchmetricsBot" 3; - "~*Semrush" 3; - "~*SemrushBot" 3; - "~*SEOkicks" 3; - "~*SEOkicks-Robot" 3; - "~*SEOlyticsCrawler" 3; - "~*Seomoz" 3; - "~*SEOprofiler" 3; - "~*SEOstats" 3; - "~*SeznamBot" 3; - "~*Siphon" 3; - "~*SISTRIX" 3; - "~*SISTRIX\ Crawler" 3; - "~*Sitebeam" 3; - "~*SiteExplorer" 3; - "~*Siteimprove" 3; - "~*SiteLockSpider" 3; - "~*SiteSnagger" 3; - "~*SiteSucker" 3; - "~*Site\ Sucker" 3; - "~*Sitevigil" 3; - "~*Slackbot-LinkExpanding" 3; - "~*SlySearch" 3; - "~*SmartDownload" 3; - "~*Snake" 3; - "~*Snapbot" 3; - "~*Snoopy" 3; - "~*SocialRankIOBot" 3; - "~*Sogou\ web\ spider" 3; - "~*Sosospider" 3; - "~*SpaceBison" 3; - "~*Spammen" 3; - "~*SpankBot" 3; - "~*Spanner" 3; - "~*Spbot" 3; - "~*Spinn3r" 3; - "~*SputnikBot" 3; - "~*Sqlmap" 3; - "~*Sqlworm" 3; - "~*Sqworm" 3; - "~*Steeler" 3; - "~*Stripper" 3; - "~*Sucker" 3; - "~*Sucuri" 3; - "~*SuperBot" 3; - "~*SuperHTTP" 3; - "~*Surfbot" 3; - "~*SurveyBot" 3; - "~*Suzuran" 3; - "~*Swiftbot" 3; - "~*Szukacz" 3; - "~*T0PHackTeam" 3; - "~*T8Abot" 3; - "~*tAkeOut" 3; - "~*Teleport" 3; - "~*TeleportPro" 3; - "~*Telesoft" 3; - "~*Telesphoreo" 3; - "~*Telesphorep" 3; - "~*The\ Intraformant" 3; - "~*TheNomad" 3; - "~*TightTwatBot" 3; - "~*Titan" 3; - "~*Toata" 3; - "~*Toweyabot" 3; - "~*Trendictionbot" 3; - "~*True_Robot" 3; - "~*Turingos" 3; - "~*TurnitinBot" 3; - "~*Turnitin\ Bot" 3; - "~*Turnitin\ Robot" 3; - "~*TwengaBot" 3; - "~*Twice" 3; - "~*Typhoeus" 3; - "~*URLy.Warning" 3; - "~*URLy\ Warning" 3; - "~*Vacuum" 3; - "~*Vagabondo" 3; - "~*VB\ Project" 3; - "~*VCI" 3; - "~*VeriCiteCrawler" 3; - "~*VidibleScraper" 3; - "~*VoidEYE" 3; - "~*Voil" 3; - "~*Voltron" 3; - "~*WallpapersHD" 3; - "~*WASALive-Bot" 3; - "~*WBSearchBot" 3; - "~*Webalta" 3; - "~*WebAuto" 3; - "~*Web\ Auto" 3; - "~*WebBandit" 3; - "~*Web\ Bandit" 3; - "~*WebCollage" 3; - "~*Web\ Collage" 3; - "~*WebCopier" 3; - "~*Web\ Copier" 3; - "~*WEBDAV" 3; - "~*WEBDAV\ Client" 3; - "~*WebEnhancer" 3; - "~*Web\ Enhancer" 3; - "~*WebFetch" 3; - "~*Web\ Fetch" 3; - "~*WebFuck" 3; - "~*Web\ Fuck" 3; - "~*WebGo\ IS" 3; - "~*WebImageCollector" 3; - "~*Web\ Image\ Collector" 3; - "~*WebLeacher" 3; - "~*WebmasterWorldForumBot" 3; - "~*webmeup-crawler" 3; - "~*WebPix" 3; - "~*Web\ Pix" 3; - "~*WebReaper" 3; - "~*Web\ Reaper" 3; - "~*WebSauger" 3; - "~*Web\ Sauger" 3; - "~*Webshag" 3; - "~*WebsiteExtractor" 3; - "~*Website\ Extractor" 3; - "~*WebsiteQuester" 3; - "~*Website\ Quester" 3; - "~*Webster" 3; - "~*WebStripper" 3; - "~*Web\ Stripper" 3; - "~*WebSucker" 3; - "~*Web\ Sucker" 3; - "~*WebWhacker" 3; - "~*Web\ Whacker" 3; - "~*WebZIP" 3; - "~*WeSEE" 3; - "~*Whack" 3; - "~*Whacker" 3; - "~*Whatweb" 3; - "~*Widow" 3; - "~*WinHTTrack" 3; - "~*WiseGuys\ Robot" 3; - "~*WISENutbot" 3; - "~*Wonderbot" 3; - "~*Woobot" 3; - "~*Wotbox" 3; - "~*Wprecon" 3; - "~*WPScan" 3; - "~*WWW-Collector-E" 3; - "~*WWW-Mechanize" 3; - "~*WWW::Mechanize" 3; - "~*WWWOFFLE" 3; - "~*Xaldon_WebSpider" 3; - "~*Xaldon\ WebSpider" 3; - "~*Xenu" 3; - "~*Y!J-ASR" 3; - "~*YoudaoBot" 3; - "~*Zade" 3; - "~*Zermelo" 3; - "~*Zeus" 3; - "~*Zgrab" 3; - "~*Zitebot" 3; - "~*ZmEu" 3; - "~*ZumBot" 3; - "~*ZyBorg" 3; + "^360Spider" 3; + "^80legs" 3; + "^Abonti" 3; + "^Aboundex" 3; + "^Acunetix" 3; + "^AhrefsBot" 3; + "^AIBOT" 3; + "^AiHitBot" 3; + "^Aipbot" 3; + "^Alexibot" 3; + "^Alligator" 3; + "^AllSubmitter" 3; + "^Anarchie" 3; + "^Apexoo" 3; + "^ASPSeek" 3; + "^Asterias" 3; + "^Attach" 3; + "^autoemailspider" 3; + "^BackDoorBot" 3; + "^BackStreet" 3; + "^BackWeb" 3; + "^Badass" 3; + "^Bandit" 3; + "^BatchFTP" 3; + "^Battleztar\ Bazinga" 3; + "^BBBike" 3; + "^BDFetch" 3; + "^BetaBot" 3; + "^Bigfoot" 3; + "^Bitacle" 3; + "^Black\ Hole" 3; + "^BlackWidow" 3; + "^BLEXBot" 3; + "^Blow" 3; + "^BlowFish" 3; + "^Boardreader" 3; + "^Bolt" 3; + "^BotALot" 3; + "^Brandprotect" 3; + "^Brandwatch" 3; + "^Bubing" 3; + "^Buddy" 3; + "^BuiltBotTough" 3; + "^BuiltWith" 3; + "^Bullseye" 3; + "^BunnySlippers" 3; + "^BuzzSumo" 3; + "^Calculon" 3; + "^CazoodleBot" 3; + "^CCBot" 3; + "^Cegbfeieh" 3; + "^CheeseBot" 3; + "^CherryPicker" 3; + "^ChinaClaw" 3; + "^Chlooe" 3; + "^Claritybot" 3; + "^Cliqzbot" 3; + "^Cogentbot" 3; + "^Collector" 3; + "^Copier" 3; + "^CopyRightCheck" 3; + "^Copyscape" 3; + "^Cosmos" 3; + "^Craftbot" 3; + "^CrazyWebCrawler" 3; + "^Crescent" 3; + "^CSHttp" 3; + "^Curious" 3; + "^Custo" 3; + "^DBLBot" 3; + "^Demon" 3; + "^Deusu" 3; + "^Devil" 3; + "^DIIbot" 3; + "^Dirbuster" 3; + "^Disco" 3; + "^Discobot" 3; + "^Discoverybot" 3; + "^DittoSpyder" 3; + "^DomainAppender" 3; + "^DomainSigmaCrawler" 3; + "^Dotbot" 3; + "^Download\ Demon" 3; + "^Download\ Devil" 3; + "^Download\ Wonder" 3; + "^Dragonfly" 3; + "^Drip" 3; + "^DTS\ Agent" 3; + "^EasyDL" 3; + "^Ebingbong" 3; + "^eCatch" 3; + "^ECCP/1.0" 3; + "^Ecxi" 3; + "^EirGrabber" 3; + "^EMail\ Collector" 3; + "^EMail\ Extractor" 3; + "^EMail\ Siphon" 3; + "^EMail\ Wolf" 3; + "^EroCrawler" 3; + "^Evil" 3; + "^Exabot" 3; + "^Express\ WebPictures" 3; + "^Extractor" 3; + "^ExtractorPro" 3; + "^EyeNetIE" 3; + "^Ezooms" 3; + "^FHscan" 3; + "^Fimap" 3; + "^Findxbot" 3; + "^Firefox/7.0" 3; + "^FlashGet" 3; + "^Flunky" 3; + "^Foobot" 3; + "^Freeuploader" 3; + "^FrontPage" 3; + "^GalaxyBot" 3; + "^Genieo" 3; + "^Getintent" 3; + "^GetRight" 3; + "^GetWeb" 3; + "^Gigablast" 3; + "^Gigabot" 3; + "^Go-Ahead-Got-It" 3; + "^Gotit" 3; + "^GoZilla" 3; + "^Go!Zilla" 3; + "^Grabber" 3; + "^GrabNet" 3; + "^Grafula" 3; + "^GrapeFX" 3; + "^GrapeshotCrawler" 3; + "^GridBot" 3; + "^GT::WWW" 3; + "^GuzzleHttp" 3; + "^HaosouSpider" 3; + "^Harvest" 3; + "^Havij" 3; + "^Heritrix" 3; + "^Hloader" 3; + "^HMView" 3; + "^HTMLparser" 3; + "^HTTP::Lite" 3; + "^HTTrack" 3; + "^Humanlinks" 3; + "^Iblog" 3; + "^IDBot" 3; + "^Id-search" 3; + "^IlseBot" 3; + "^Image\ Fetch" 3; + "^Image\ Stripper" 3; + "^Image\ Sucker" 3; + "^Indy\ Library" 3; + "^InfoNaviRobot" 3; + "^InfoTekies" 3; + "^Intelliseek" 3; + "^InterGET" 3; + "^Internet\ Ninja" 3; + "^InternetSeer" 3; + "^internetVista\ monitor" 3; + "^Iria" 3; + "^IRLbot" 3; + "^Iskanie" 3; + "^JamesBOT" 3; + "^Jbrofuzz" 3; + "^JennyBot" 3; + "^JetCar" 3; + "^JikeSpider" 3; + "^JOC\ Web\ Spider" 3; + "^Joomla" 3; + "^JustView" 3; + "^Jyxobot" 3; + "^Kenjin\ Spider" 3; + "^Keyword\ Density" 3; + "^Kraken" 3; + "^Lanshanbot" 3; + "^Larbin" 3; + "^LeechFTP" 3; + "^LeechGet" 3; + "^LexiBot" 3; + "^Lftp" 3; + "^LibWeb" 3; + "^Libwhisker" 3; + "^Lightspeedsystems" 3; + "^Likse" 3; + "^Linkdexbot" 3; + "^LinkextractorPro" 3; + "^LinkpadBot" 3; + "^LinkScan" 3; + "^LinksManager" 3; + "^LinkWalker" 3; + "^LinqiaMetadataDownloaderBot" 3; + "^LinqiaRSSBot" 3; + "^LinqiaScrapeBot" 3; + "^Lipperhey" 3; + "^Litemage_walker" 3; + "^Lmspider" 3; + "^LNSpiderguy" 3; + "^Ltx71" 3; + "^lwp-request" 3; + "^LWP::Simple" 3; + "^lwp-trivial" 3; + "^Magnet" 3; + "^Mag-Net" 3; + "^magpie-crawler" 3; + "^Mail.ru" 3; + "^Majestic12" 3; + "^MarkMonitor" 3; + "^MarkWatch" 3; + "^Masscan" 3; + "^Mass\ Downloader" 3; + "^Mata\ Hari" 3; + "^Meanpathbot" 3; + "^MegaIndex.ru" 3; + "^Metauri" 3; + "^MFC_Tear_Sample" 3; + "^Microsoft\ Data\ Access" 3; + "^Microsoft\ URL\ Control" 3; + "^MIDown\ tool" 3; + "^MIIxpc" 3; + "^Mister\ PiX" 3; + "^MJ12bot" 3; + "^Mojeek" 3; + "^MSFrontPage" 3; + "^MSIE\ 6.0" 3; + "^MSIECrawler" 3; + "^Msrabot" 3; + "^MS\ Web\ Services\ Client\ Protocol" 3; + "^Musobot" 3; + "^Name\ Intelligence" 3; + "^Nameprotect" 3; + "^Navroad" 3; + "^NearSite" 3; + "^Needle" 3; + "^Nessus" 3; + "^NetAnts" 3; + "^Netcraft" 3; + "^netEstate\ NE\ Crawler" 3; + "^NetLyzer" 3; + "^NetMechanic" 3; + "^NetSpider" 3; + "^Nettrack" 3; + "^Net\ Vampire" 3; + "^NetZIP" 3; + "^NextGenSearchBot" 3; + "^Nibbler" 3; + "^NICErsPRO" 3; + "^Niki-bot" 3; + "^Nikto" 3; + "^NimbleCrawler" 3; + "^Ninja" 3; + "^Nmap" 3; + "^NPbot" 3; + "^Nutch" 3; + "^Octopus" 3; + "^Offline\ Explorer" 3; + "^Offline\ Navigator" 3; + "^Openfind" 3; + "^OpenLinkProfiler" 3; + "^Openvas" 3; + "^OrangeBot" 3; + "^OrangeSpider" 3; + "^OutfoxBot" 3; + "^PageAnalyzer" 3; + "^Page\ Analyzer" 3; + "^PageGrabber" 3; + "^Page\ Grabber" 3; + "^page\ scorer" 3; + "^PageScorer" 3; + "^Panscient" 3; + "^Papa\ Foto" 3; + "^Pavuk" 3; + "^pcBrowser" 3; + "^PECL::HTTP" 3; + "^PeoplePal" 3; + "^PHPCrawl" 3; + "^Picscout" 3; + "^Picsearch" 3; + "^Pimonster" 3; + "^Pi-Monster" 3; + "^Pixray" 3; + "^PleaseCrawl" 3; + "^Pockey" 3; + "^POE-Component-Client-HTTP" 3; + "^Probethenet" 3; + "^ProPowerBot" 3; + "^ProWebWalker" 3; + "^Proximic" 3; + "^Psbot" 3; + "^Pump" 3; + "^PyCurl" 3; + "^QueryN\ Metasearch" 3; + "^Qwantify" 3; + "^RankActiveLinkBot" 3; + "^Rankivabot" 3; + "^RealDownload" 3; + "^Reaper" 3; + "^Recorder" 3; + "^RedesScrapy" 3; + "^ReGet" 3; + "^RepoMonkey" 3; + "^Ripper" 3; + "^RocketCrawler" 3; + "^Rogerbot" 3; + "^SBIder" 3; + "^ScanAlert" 3; + "^Scanbot" 3; + "^Scrapy" 3; + "^Screaming\ Frog\ SEO\ Spider" 3; + "^ScreenerBot" 3; + "^Searchestate" 3; + "^SearchmetricsBot" 3; + "^Semrush" 3; + "^SemrushBot" 3; + "^SEOkicks" 3; + "^SEOkicks-Robot" 3; + "^SEOlyticsCrawler" 3; + "^Seomoz" 3; + "^SEOprofiler" 3; + "^SEOstats" 3; + "^SeznamBot" 3; + "^Siphon" 3; + "^SISTRIX" 3; + "^SISTRIX\ Crawler" 3; + "^Sitebeam" 3; + "^SiteExplorer" 3; + "^Siteimprove" 3; + "^SiteLockSpider" 3; + "^SiteSnagger" 3; + "^SiteSucker" 3; + "^Site\ Sucker" 3; + "^Sitevigil" 3; + "^Slackbot-LinkExpanding" 3; + "^SlySearch" 3; + "^SmartDownload" 3; + "^Snake" 3; + "^Snapbot" 3; + "^Snoopy" 3; + "^SocialRankIOBot" 3; + "^Sogou\ web\ spider" 3; + "^Sosospider" 3; + "^SpaceBison" 3; + "^Spammen" 3; + "^SpankBot" 3; + "^Spanner" 3; + "^Spbot" 3; + "^Spinn3r" 3; + "^SputnikBot" 3; + "^Sqlmap" 3; + "^Sqlworm" 3; + "^Sqworm" 3; + "^Steeler" 3; + "^Stripper" 3; + "^Sucker" 3; + "^Sucuri" 3; + "^SuperBot" 3; + "^SuperHTTP" 3; + "^Surfbot" 3; + "^SurveyBot" 3; + "^Suzuran" 3; + "^Swiftbot" 3; + "^Szukacz" 3; + "^T0PHackTeam" 3; + "^T8Abot" 3; + "^tAkeOut" 3; + "^Teleport" 3; + "^TeleportPro" 3; + "^Telesoft" 3; + "^Telesphoreo" 3; + "^Telesphorep" 3; + "^The\ Intraformant" 3; + "^TheNomad" 3; + "^TightTwatBot" 3; + "^Titan" 3; + "^Toata" 3; + "^Toweyabot" 3; + "^Trendictionbot" 3; + "^True_Robot" 3; + "^Turingos" 3; + "^TurnitinBot" 3; + "^Turnitin\ Bot" 3; + "^Turnitin\ Robot" 3; + "^TwengaBot" 3; + "^Twice" 3; + "^Typhoeus" 3; + "^URLy.Warning" 3; + "^URLy\ Warning" 3; + "^Vacuum" 3; + "^Vagabondo" 3; + "^VB\ Project" 3; + "^VCI" 3; + "^VeriCiteCrawler" 3; + "^VidibleScraper" 3; + "^VoidEYE" 3; + "^Voil" 3; + "^Voltron" 3; + "^WallpapersHD" 3; + "^WASALive-Bot" 3; + "^WBSearchBot" 3; + "^Webalta" 3; + "^WebAuto" 3; + "^Web\ Auto" 3; + "^WebBandit" 3; + "^Web\ Bandit" 3; + "^WebCollage" 3; + "^Web\ Collage" 3; + "^WebCopier" 3; + "^Web\ Copier" 3; + "^WEBDAV" 3; + "^WEBDAV\ Client" 3; + "^WebEnhancer" 3; + "^Web\ Enhancer" 3; + "^WebFetch" 3; + "^Web\ Fetch" 3; + "^WebFuck" 3; + "^Web\ Fuck" 3; + "^WebGo\ IS" 3; + "^WebImageCollector" 3; + "^Web\ Image\ Collector" 3; + "^WebLeacher" 3; + "^WebmasterWorldForumBot" 3; + "^webmeup-crawler" 3; + "^WebPix" 3; + "^Web\ Pix" 3; + "^WebReaper" 3; + "^Web\ Reaper" 3; + "^WebSauger" 3; + "^Web\ Sauger" 3; + "^Webshag" 3; + "^WebsiteExtractor" 3; + "^Website\ Extractor" 3; + "^WebsiteQuester" 3; + "^Website\ Quester" 3; + "^Webster" 3; + "^WebStripper" 3; + "^Web\ Stripper" 3; + "^WebSucker" 3; + "^Web\ Sucker" 3; + "^WebWhacker" 3; + "^Web\ Whacker" 3; + "^WebZIP" 3; + "^WeSEE" 3; + "^Whack" 3; + "^Whacker" 3; + "^Whatweb" 3; + "^Widow" 3; + "^WinHTTrack" 3; + "^WiseGuys\ Robot" 3; + "^WISENutbot" 3; + "^Wonderbot" 3; + "^Woobot" 3; + "^Wotbox" 3; + "^Wprecon" 3; + "^WPScan" 3; + "^WWW-Collector-E" 3; + "^WWW-Mechanize" 3; + "^WWW::Mechanize" 3; + "^WWWOFFLE" 3; + "^Xaldon_WebSpider" 3; + "^Xaldon\ WebSpider" 3; + "^Xenu" 3; + "^Y!J-ASR" 3; + "^YoudaoBot" 3; + "^Zade" 3; + "^Zermelo" 3; + "^Zeus" 3; + "^Zgrab" 3; + "^Zitebot" 3; + "^ZmEu" 3; + "^ZumBot" 3; + "^ZyBorg" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/travisCI/globalblacklist.tpl.conf b/travisCI/globalblacklist.tpl.conf index fa5c80e64..ab778d2f4 100644 --- a/travisCI/globalblacklist.tpl.conf +++ b/travisCI/globalblacklist.tpl.conf @@ -17,11 +17,11 @@ ### - https://github.com/oohnoitz/nginx-blacklist ### Last Updated -### Wed Apr 19 12:42:01 SAST 2017 +### Wed Apr 19 12:53:26 SAST 2017 ### End Last Updated ### Generated in -### 0.301798343658 seconds +### 0.299472808838 seconds ### End Generated in ### Tested on: nginx/1.10.0 (Ubuntu 16.04) @@ -128,35 +128,35 @@ map $http_user_agent $bad_bot { # *********************************************** # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*adidxbot" 0; - "~*AdsBot-Google" 0; - "~*aolbuild" 0; - "~*bingbot" 0; - "~*bingpreview" 0; - "~*DoCoMo" 0; - "~*duckduckgo" 0; - "~*facebookexternalhit" 0; - "~*Feedfetcher-Google" 0; - "~*Googlebot" 0; - "~*Googlebot-Image" 0; - "~*Googlebot-Mobile" 0; - "~*Googlebot-News" 0; - "~*Googlebot/Test" 0; - "~*Googlebot-Video" 0; - "~*Google-HTTP-Java-Client" 0; - "~*gsa-crawler" 0; - "~*Jakarta\ Commons" 0; - "~*Kraken/0.1" 0; - "~*LinkedInBot" 0; - "~*Mediapartners-Google" 0; - "~*msnbot" 0; - "~*msnbot-media" 0; - "~*SAMSUNG" 0; - "~*slurp" 0; - "~*teoma" 0; - "~*TwitterBot" 0; - "~*Wordpress" 0; - "~*yahoo" 0; + "^adidxbot" 0; + "^AdsBot-Google" 0; + "^aolbuild" 0; + "^bingbot" 0; + "^bingpreview" 0; + "^DoCoMo" 0; + "^duckduckgo" 0; + "^facebookexternalhit" 0; + "^Feedfetcher-Google" 0; + "^Googlebot" 0; + "^Googlebot-Image" 0; + "^Googlebot-Mobile" 0; + "^Googlebot-News" 0; + "^Googlebot/Test" 0; + "^Googlebot-Video" 0; + "^Google-HTTP-Java-Client" 0; + "^gsa-crawler" 0; + "^Jakarta\ Commons" 0; + "^Kraken/0.1" 0; + "^LinkedInBot" 0; + "^Mediapartners-Google" 0; + "^msnbot" 0; + "^msnbot-media" 0; + "^SAMSUNG" 0; + "^slurp" 0; + "^teoma" 0; + "^TwitterBot" 0; + "^Wordpress" 0; + "^yahoo" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************** @@ -166,12 +166,12 @@ map $http_user_agent $bad_bot { # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*jetmon" 1; - "~*libwww-perl" 1; - "~*Lynx" 1; - "~*munin" 1; - "~*Wget/1.15" 1; - "~*WordPress" 1; + "^jetmon" 1; + "^libwww-perl" 1; + "^Lynx" 1; + "^munin" 1; + "^Wget/1.15" 1; + "^WordPress" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************************** @@ -179,22 +179,22 @@ map $http_user_agent $bad_bot { # ************************************************************** # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*Alexa" 2; - "~*archive.org" 2; - "~*Baidu" 2; - "~*FlipboardProxy" 2; - "~*ia_archiver" 2; - "~*Mozilla/4.0" 2; - "~*MSIE\ 7.0" 2; - "~*Presto" 2; - "~*R6_CommentReader" 2; - "~*R6_FeedFetcher" 2; - "~*RPT-HTTPClient" 2; - "~*sfFeedReader/0.9" 2; - "~*Spaidu" 2; - "~*UptimeRobot/2.0" 2; - "~*YandexBot" 2; - "~*YandexImages" 2; + "^Alexa" 2; + "^archive.org" 2; + "^Baidu" 2; + "^FlipboardProxy" 2; + "^ia_archiver" 2; + "^Mozilla/4.0" 2; + "^MSIE\ 7.0" 2; + "^Presto" 2; + "^R6_CommentReader" 2; + "^R6_FeedFetcher" 2; + "^RPT-HTTPClient" 2; + "^sfFeedReader/0.9" 2; + "^Spaidu" 2; + "^UptimeRobot/2.0" 2; + "^YandexBot" 2; + "^YandexImages" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ********************************************* @@ -212,476 +212,476 @@ include /home/travis/build/mitchellkrogza/nginx-ultimate-bad-bot-blocker/travisC # END BLACKLISTED USER AGENTS ### DO NOT EDIT THIS LINE AT ALL ### # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*360Spider" 3; - "~*80legs" 3; - "~*Abonti" 3; - "~*Aboundex" 3; - "~*Acunetix" 3; - "~*AhrefsBot" 3; - "~*AIBOT" 3; - "~*AiHitBot" 3; - "~*Aipbot" 3; - "~*Alexibot" 3; - "~*Alligator" 3; - "~*AllSubmitter" 3; - "~*Anarchie" 3; - "~*Apexoo" 3; - "~*ASPSeek" 3; - "~*Asterias" 3; - "~*Attach" 3; - "~*autoemailspider" 3; - "~*BackDoorBot" 3; - "~*BackStreet" 3; - "~*BackWeb" 3; - "~*Badass" 3; - "~*Bandit" 3; - "~*BatchFTP" 3; - "~*Battleztar\ Bazinga" 3; - "~*BBBike" 3; - "~*BDFetch" 3; - "~*BetaBot" 3; - "~*Bigfoot" 3; - "~*Bitacle" 3; - "~*Black\ Hole" 3; - "~*BlackWidow" 3; - "~*BLEXBot" 3; - "~*Blow" 3; - "~*BlowFish" 3; - "~*Boardreader" 3; - "~*Bolt" 3; - "~*BotALot" 3; - "~*Brandprotect" 3; - "~*Brandwatch" 3; - "~*Bubing" 3; - "~*Buddy" 3; - "~*BuiltBotTough" 3; - "~*BuiltWith" 3; - "~*Bullseye" 3; - "~*BunnySlippers" 3; - "~*BuzzSumo" 3; - "~*Calculon" 3; - "~*CazoodleBot" 3; - "~*CCBot" 3; - "~*Cegbfeieh" 3; - "~*CheeseBot" 3; - "~*CherryPicker" 3; - "~*ChinaClaw" 3; - "~*Chlooe" 3; - "~*Claritybot" 3; - "~*Cliqzbot" 3; - "~*Cogentbot" 3; - "~*Collector" 3; - "~*Copier" 3; - "~*CopyRightCheck" 3; - "~*Copyscape" 3; - "~*Cosmos" 3; - "~*Craftbot" 3; - "~*CrazyWebCrawler" 3; - "~*Crescent" 3; - "~*CSHttp" 3; - "~*Curious" 3; - "~*Custo" 3; - "~*DBLBot" 3; - "~*Demon" 3; - "~*Deusu" 3; - "~*Devil" 3; - "~*DIIbot" 3; - "~*Dirbuster" 3; - "~*Disco" 3; - "~*Discobot" 3; - "~*Discoverybot" 3; - "~*DittoSpyder" 3; - "~*DomainAppender" 3; - "~*DomainSigmaCrawler" 3; - "~*Dotbot" 3; - "~*Download\ Demon" 3; - "~*Download\ Devil" 3; - "~*Download\ Wonder" 3; - "~*Dragonfly" 3; - "~*Drip" 3; - "~*DTS\ Agent" 3; - "~*EasyDL" 3; - "~*Ebingbong" 3; - "~*eCatch" 3; - "~*ECCP/1.0" 3; - "~*Ecxi" 3; - "~*EirGrabber" 3; - "~*EMail\ Collector" 3; - "~*EMail\ Extractor" 3; - "~*EMail\ Siphon" 3; - "~*EMail\ Wolf" 3; - "~*EroCrawler" 3; - "~*Evil" 3; - "~*Exabot" 3; - "~*Express\ WebPictures" 3; - "~*Extractor" 3; - "~*ExtractorPro" 3; - "~*EyeNetIE" 3; - "~*Ezooms" 3; - "~*FHscan" 3; - "~*Fimap" 3; - "~*Findxbot" 3; - "~*Firefox/7.0" 3; - "~*FlashGet" 3; - "~*Flunky" 3; - "~*Foobot" 3; - "~*Freeuploader" 3; - "~*FrontPage" 3; - "~*GalaxyBot" 3; - "~*Genieo" 3; - "~*Getintent" 3; - "~*GetRight" 3; - "~*GetWeb" 3; - "~*Gigablast" 3; - "~*Gigabot" 3; - "~*Go-Ahead-Got-It" 3; - "~*Gotit" 3; - "~*GoZilla" 3; - "~*Go!Zilla" 3; - "~*Grabber" 3; - "~*GrabNet" 3; - "~*Grafula" 3; - "~*GrapeFX" 3; - "~*GrapeshotCrawler" 3; - "~*GridBot" 3; - "~*GT::WWW" 3; - "~*GuzzleHttp" 3; - "~*HaosouSpider" 3; - "~*Harvest" 3; - "~*Havij" 3; - "~*Heritrix" 3; - "~*Hloader" 3; - "~*HMView" 3; - "~*HTMLparser" 3; - "~*HTTP::Lite" 3; - "~*HTTrack" 3; - "~*Humanlinks" 3; - "~*Iblog" 3; - "~*IDBot" 3; - "~*Id-search" 3; - "~*IlseBot" 3; - "~*Image\ Fetch" 3; - "~*Image\ Stripper" 3; - "~*Image\ Sucker" 3; - "~*Indy\ Library" 3; - "~*InfoNaviRobot" 3; - "~*InfoTekies" 3; - "~*Intelliseek" 3; - "~*InterGET" 3; - "~*Internet\ Ninja" 3; - "~*InternetSeer" 3; - "~*internetVista\ monitor" 3; - "~*Iria" 3; - "~*IRLbot" 3; - "~*Iskanie" 3; - "~*JamesBOT" 3; - "~*Jbrofuzz" 3; - "~*JennyBot" 3; - "~*JetCar" 3; - "~*JikeSpider" 3; - "~*JOC\ Web\ Spider" 3; - "~*Joomla" 3; - "~*JustView" 3; - "~*Jyxobot" 3; - "~*Kenjin\ Spider" 3; - "~*Keyword\ Density" 3; - "~*Kraken" 3; - "~*Lanshanbot" 3; - "~*Larbin" 3; - "~*LeechFTP" 3; - "~*LeechGet" 3; - "~*LexiBot" 3; - "~*Lftp" 3; - "~*LibWeb" 3; - "~*Libwhisker" 3; - "~*Lightspeedsystems" 3; - "~*Likse" 3; - "~*Linkdexbot" 3; - "~*LinkextractorPro" 3; - "~*LinkpadBot" 3; - "~*LinkScan" 3; - "~*LinksManager" 3; - "~*LinkWalker" 3; - "~*LinqiaMetadataDownloaderBot" 3; - "~*LinqiaRSSBot" 3; - "~*LinqiaScrapeBot" 3; - "~*Lipperhey" 3; - "~*Litemage_walker" 3; - "~*Lmspider" 3; - "~*LNSpiderguy" 3; - "~*Ltx71" 3; - "~*lwp-request" 3; - "~*LWP::Simple" 3; - "~*lwp-trivial" 3; - "~*Magnet" 3; - "~*Mag-Net" 3; - "~*magpie-crawler" 3; - "~*Mail.ru" 3; - "~*Majestic12" 3; - "~*MarkMonitor" 3; - "~*MarkWatch" 3; - "~*Masscan" 3; - "~*Mass\ Downloader" 3; - "~*Mata\ Hari" 3; - "~*Meanpathbot" 3; - "~*MegaIndex.ru" 3; - "~*Metauri" 3; - "~*MFC_Tear_Sample" 3; - "~*Microsoft\ Data\ Access" 3; - "~*Microsoft\ URL\ Control" 3; - "~*MIDown\ tool" 3; - "~*MIIxpc" 3; - "~*Mister\ PiX" 3; - "~*MJ12bot" 3; - "~*Mojeek" 3; - "~*MSFrontPage" 3; - "~*MSIE\ 6.0" 3; - "~*MSIECrawler" 3; - "~*Msrabot" 3; - "~*MS\ Web\ Services\ Client\ Protocol" 3; - "~*Musobot" 3; - "~*Name\ Intelligence" 3; - "~*Nameprotect" 3; - "~*Navroad" 3; - "~*NearSite" 3; - "~*Needle" 3; - "~*Nessus" 3; - "~*NetAnts" 3; - "~*Netcraft" 3; - "~*netEstate\ NE\ Crawler" 3; - "~*NetLyzer" 3; - "~*NetMechanic" 3; - "~*NetSpider" 3; - "~*Nettrack" 3; - "~*Net\ Vampire" 3; - "~*NetZIP" 3; - "~*NextGenSearchBot" 3; - "~*Nibbler" 3; - "~*NICErsPRO" 3; - "~*Niki-bot" 3; - "~*Nikto" 3; - "~*NimbleCrawler" 3; - "~*Ninja" 3; - "~*Nmap" 3; - "~*NPbot" 3; - "~*Nutch" 3; - "~*Octopus" 3; - "~*Offline\ Explorer" 3; - "~*Offline\ Navigator" 3; - "~*Openfind" 3; - "~*OpenLinkProfiler" 3; - "~*Openvas" 3; - "~*OrangeBot" 3; - "~*OrangeSpider" 3; - "~*OutfoxBot" 3; - "~*PageAnalyzer" 3; - "~*Page\ Analyzer" 3; - "~*PageGrabber" 3; - "~*Page\ Grabber" 3; - "~*page\ scorer" 3; - "~*PageScorer" 3; - "~*Panscient" 3; - "~*Papa\ Foto" 3; - "~*Pavuk" 3; - "~*pcBrowser" 3; - "~*PECL::HTTP" 3; - "~*PeoplePal" 3; - "~*PHPCrawl" 3; - "~*Picscout" 3; - "~*Picsearch" 3; - "~*Pimonster" 3; - "~*Pi-Monster" 3; - "~*Pixray" 3; - "~*PleaseCrawl" 3; - "~*Pockey" 3; - "~*POE-Component-Client-HTTP" 3; - "~*Probethenet" 3; - "~*ProPowerBot" 3; - "~*ProWebWalker" 3; - "~*Proximic" 3; - "~*Psbot" 3; - "~*Pump" 3; - "~*PyCurl" 3; - "~*QueryN\ Metasearch" 3; - "~*Qwantify" 3; - "~*RankActiveLinkBot" 3; - "~*Rankivabot" 3; - "~*RealDownload" 3; - "~*Reaper" 3; - "~*Recorder" 3; - "~*RedesScrapy" 3; - "~*ReGet" 3; - "~*RepoMonkey" 3; - "~*Ripper" 3; - "~*RocketCrawler" 3; - "~*Rogerbot" 3; - "~*SBIder" 3; - "~*ScanAlert" 3; - "~*Scanbot" 3; - "~*Scrapy" 3; - "~*Screaming\ Frog\ SEO\ Spider" 3; - "~*ScreenerBot" 3; - "~*Searchestate" 3; - "~*SearchmetricsBot" 3; - "~*Semrush" 3; - "~*SemrushBot" 3; - "~*SEOkicks" 3; - "~*SEOkicks-Robot" 3; - "~*SEOlyticsCrawler" 3; - "~*Seomoz" 3; - "~*SEOprofiler" 3; - "~*SEOstats" 3; - "~*SeznamBot" 3; - "~*Siphon" 3; - "~*SISTRIX" 3; - "~*SISTRIX\ Crawler" 3; - "~*Sitebeam" 3; - "~*SiteExplorer" 3; - "~*Siteimprove" 3; - "~*SiteLockSpider" 3; - "~*SiteSnagger" 3; - "~*SiteSucker" 3; - "~*Site\ Sucker" 3; - "~*Sitevigil" 3; - "~*Slackbot-LinkExpanding" 3; - "~*SlySearch" 3; - "~*SmartDownload" 3; - "~*Snake" 3; - "~*Snapbot" 3; - "~*Snoopy" 3; - "~*SocialRankIOBot" 3; - "~*Sogou\ web\ spider" 3; - "~*Sosospider" 3; - "~*SpaceBison" 3; - "~*Spammen" 3; - "~*SpankBot" 3; - "~*Spanner" 3; - "~*Spbot" 3; - "~*Spinn3r" 3; - "~*SputnikBot" 3; - "~*Sqlmap" 3; - "~*Sqlworm" 3; - "~*Sqworm" 3; - "~*Steeler" 3; - "~*Stripper" 3; - "~*Sucker" 3; - "~*Sucuri" 3; - "~*SuperBot" 3; - "~*SuperHTTP" 3; - "~*Surfbot" 3; - "~*SurveyBot" 3; - "~*Suzuran" 3; - "~*Swiftbot" 3; - "~*Szukacz" 3; - "~*T0PHackTeam" 3; - "~*T8Abot" 3; - "~*tAkeOut" 3; - "~*Teleport" 3; - "~*TeleportPro" 3; - "~*Telesoft" 3; - "~*Telesphoreo" 3; - "~*Telesphorep" 3; - "~*The\ Intraformant" 3; - "~*TheNomad" 3; - "~*TightTwatBot" 3; - "~*Titan" 3; - "~*Toata" 3; - "~*Toweyabot" 3; - "~*Trendictionbot" 3; - "~*True_Robot" 3; - "~*Turingos" 3; - "~*TurnitinBot" 3; - "~*Turnitin\ Bot" 3; - "~*Turnitin\ Robot" 3; - "~*TwengaBot" 3; - "~*Twice" 3; - "~*Typhoeus" 3; - "~*URLy.Warning" 3; - "~*URLy\ Warning" 3; - "~*Vacuum" 3; - "~*Vagabondo" 3; - "~*VB\ Project" 3; - "~*VCI" 3; - "~*VeriCiteCrawler" 3; - "~*VidibleScraper" 3; - "~*VoidEYE" 3; - "~*Voil" 3; - "~*Voltron" 3; - "~*WallpapersHD" 3; - "~*WASALive-Bot" 3; - "~*WBSearchBot" 3; - "~*Webalta" 3; - "~*WebAuto" 3; - "~*Web\ Auto" 3; - "~*WebBandit" 3; - "~*Web\ Bandit" 3; - "~*WebCollage" 3; - "~*Web\ Collage" 3; - "~*WebCopier" 3; - "~*Web\ Copier" 3; - "~*WEBDAV" 3; - "~*WEBDAV\ Client" 3; - "~*WebEnhancer" 3; - "~*Web\ Enhancer" 3; - "~*WebFetch" 3; - "~*Web\ Fetch" 3; - "~*WebFuck" 3; - "~*Web\ Fuck" 3; - "~*WebGo\ IS" 3; - "~*WebImageCollector" 3; - "~*Web\ Image\ Collector" 3; - "~*WebLeacher" 3; - "~*WebmasterWorldForumBot" 3; - "~*webmeup-crawler" 3; - "~*WebPix" 3; - "~*Web\ Pix" 3; - "~*WebReaper" 3; - "~*Web\ Reaper" 3; - "~*WebSauger" 3; - "~*Web\ Sauger" 3; - "~*Webshag" 3; - "~*WebsiteExtractor" 3; - "~*Website\ Extractor" 3; - "~*WebsiteQuester" 3; - "~*Website\ Quester" 3; - "~*Webster" 3; - "~*WebStripper" 3; - "~*Web\ Stripper" 3; - "~*WebSucker" 3; - "~*Web\ Sucker" 3; - "~*WebWhacker" 3; - "~*Web\ Whacker" 3; - "~*WebZIP" 3; - "~*WeSEE" 3; - "~*Whack" 3; - "~*Whacker" 3; - "~*Whatweb" 3; - "~*Widow" 3; - "~*WinHTTrack" 3; - "~*WiseGuys\ Robot" 3; - "~*WISENutbot" 3; - "~*Wonderbot" 3; - "~*Woobot" 3; - "~*Wotbox" 3; - "~*Wprecon" 3; - "~*WPScan" 3; - "~*WWW-Collector-E" 3; - "~*WWW-Mechanize" 3; - "~*WWW::Mechanize" 3; - "~*WWWOFFLE" 3; - "~*Xaldon_WebSpider" 3; - "~*Xaldon\ WebSpider" 3; - "~*Xenu" 3; - "~*Y!J-ASR" 3; - "~*YoudaoBot" 3; - "~*Zade" 3; - "~*Zermelo" 3; - "~*Zeus" 3; - "~*Zgrab" 3; - "~*Zitebot" 3; - "~*ZmEu" 3; - "~*ZumBot" 3; - "~*ZyBorg" 3; + "^360Spider" 3; + "^80legs" 3; + "^Abonti" 3; + "^Aboundex" 3; + "^Acunetix" 3; + "^AhrefsBot" 3; + "^AIBOT" 3; + "^AiHitBot" 3; + "^Aipbot" 3; + "^Alexibot" 3; + "^Alligator" 3; + "^AllSubmitter" 3; + "^Anarchie" 3; + "^Apexoo" 3; + "^ASPSeek" 3; + "^Asterias" 3; + "^Attach" 3; + "^autoemailspider" 3; + "^BackDoorBot" 3; + "^BackStreet" 3; + "^BackWeb" 3; + "^Badass" 3; + "^Bandit" 3; + "^BatchFTP" 3; + "^Battleztar\ Bazinga" 3; + "^BBBike" 3; + "^BDFetch" 3; + "^BetaBot" 3; + "^Bigfoot" 3; + "^Bitacle" 3; + "^Black\ Hole" 3; + "^BlackWidow" 3; + "^BLEXBot" 3; + "^Blow" 3; + "^BlowFish" 3; + "^Boardreader" 3; + "^Bolt" 3; + "^BotALot" 3; + "^Brandprotect" 3; + "^Brandwatch" 3; + "^Bubing" 3; + "^Buddy" 3; + "^BuiltBotTough" 3; + "^BuiltWith" 3; + "^Bullseye" 3; + "^BunnySlippers" 3; + "^BuzzSumo" 3; + "^Calculon" 3; + "^CazoodleBot" 3; + "^CCBot" 3; + "^Cegbfeieh" 3; + "^CheeseBot" 3; + "^CherryPicker" 3; + "^ChinaClaw" 3; + "^Chlooe" 3; + "^Claritybot" 3; + "^Cliqzbot" 3; + "^Cogentbot" 3; + "^Collector" 3; + "^Copier" 3; + "^CopyRightCheck" 3; + "^Copyscape" 3; + "^Cosmos" 3; + "^Craftbot" 3; + "^CrazyWebCrawler" 3; + "^Crescent" 3; + "^CSHttp" 3; + "^Curious" 3; + "^Custo" 3; + "^DBLBot" 3; + "^Demon" 3; + "^Deusu" 3; + "^Devil" 3; + "^DIIbot" 3; + "^Dirbuster" 3; + "^Disco" 3; + "^Discobot" 3; + "^Discoverybot" 3; + "^DittoSpyder" 3; + "^DomainAppender" 3; + "^DomainSigmaCrawler" 3; + "^Dotbot" 3; + "^Download\ Demon" 3; + "^Download\ Devil" 3; + "^Download\ Wonder" 3; + "^Dragonfly" 3; + "^Drip" 3; + "^DTS\ Agent" 3; + "^EasyDL" 3; + "^Ebingbong" 3; + "^eCatch" 3; + "^ECCP/1.0" 3; + "^Ecxi" 3; + "^EirGrabber" 3; + "^EMail\ Collector" 3; + "^EMail\ Extractor" 3; + "^EMail\ Siphon" 3; + "^EMail\ Wolf" 3; + "^EroCrawler" 3; + "^Evil" 3; + "^Exabot" 3; + "^Express\ WebPictures" 3; + "^Extractor" 3; + "^ExtractorPro" 3; + "^EyeNetIE" 3; + "^Ezooms" 3; + "^FHscan" 3; + "^Fimap" 3; + "^Findxbot" 3; + "^Firefox/7.0" 3; + "^FlashGet" 3; + "^Flunky" 3; + "^Foobot" 3; + "^Freeuploader" 3; + "^FrontPage" 3; + "^GalaxyBot" 3; + "^Genieo" 3; + "^Getintent" 3; + "^GetRight" 3; + "^GetWeb" 3; + "^Gigablast" 3; + "^Gigabot" 3; + "^Go-Ahead-Got-It" 3; + "^Gotit" 3; + "^GoZilla" 3; + "^Go!Zilla" 3; + "^Grabber" 3; + "^GrabNet" 3; + "^Grafula" 3; + "^GrapeFX" 3; + "^GrapeshotCrawler" 3; + "^GridBot" 3; + "^GT::WWW" 3; + "^GuzzleHttp" 3; + "^HaosouSpider" 3; + "^Harvest" 3; + "^Havij" 3; + "^Heritrix" 3; + "^Hloader" 3; + "^HMView" 3; + "^HTMLparser" 3; + "^HTTP::Lite" 3; + "^HTTrack" 3; + "^Humanlinks" 3; + "^Iblog" 3; + "^IDBot" 3; + "^Id-search" 3; + "^IlseBot" 3; + "^Image\ Fetch" 3; + "^Image\ Stripper" 3; + "^Image\ Sucker" 3; + "^Indy\ Library" 3; + "^InfoNaviRobot" 3; + "^InfoTekies" 3; + "^Intelliseek" 3; + "^InterGET" 3; + "^Internet\ Ninja" 3; + "^InternetSeer" 3; + "^internetVista\ monitor" 3; + "^Iria" 3; + "^IRLbot" 3; + "^Iskanie" 3; + "^JamesBOT" 3; + "^Jbrofuzz" 3; + "^JennyBot" 3; + "^JetCar" 3; + "^JikeSpider" 3; + "^JOC\ Web\ Spider" 3; + "^Joomla" 3; + "^JustView" 3; + "^Jyxobot" 3; + "^Kenjin\ Spider" 3; + "^Keyword\ Density" 3; + "^Kraken" 3; + "^Lanshanbot" 3; + "^Larbin" 3; + "^LeechFTP" 3; + "^LeechGet" 3; + "^LexiBot" 3; + "^Lftp" 3; + "^LibWeb" 3; + "^Libwhisker" 3; + "^Lightspeedsystems" 3; + "^Likse" 3; + "^Linkdexbot" 3; + "^LinkextractorPro" 3; + "^LinkpadBot" 3; + "^LinkScan" 3; + "^LinksManager" 3; + "^LinkWalker" 3; + "^LinqiaMetadataDownloaderBot" 3; + "^LinqiaRSSBot" 3; + "^LinqiaScrapeBot" 3; + "^Lipperhey" 3; + "^Litemage_walker" 3; + "^Lmspider" 3; + "^LNSpiderguy" 3; + "^Ltx71" 3; + "^lwp-request" 3; + "^LWP::Simple" 3; + "^lwp-trivial" 3; + "^Magnet" 3; + "^Mag-Net" 3; + "^magpie-crawler" 3; + "^Mail.ru" 3; + "^Majestic12" 3; + "^MarkMonitor" 3; + "^MarkWatch" 3; + "^Masscan" 3; + "^Mass\ Downloader" 3; + "^Mata\ Hari" 3; + "^Meanpathbot" 3; + "^MegaIndex.ru" 3; + "^Metauri" 3; + "^MFC_Tear_Sample" 3; + "^Microsoft\ Data\ Access" 3; + "^Microsoft\ URL\ Control" 3; + "^MIDown\ tool" 3; + "^MIIxpc" 3; + "^Mister\ PiX" 3; + "^MJ12bot" 3; + "^Mojeek" 3; + "^MSFrontPage" 3; + "^MSIE\ 6.0" 3; + "^MSIECrawler" 3; + "^Msrabot" 3; + "^MS\ Web\ Services\ Client\ Protocol" 3; + "^Musobot" 3; + "^Name\ Intelligence" 3; + "^Nameprotect" 3; + "^Navroad" 3; + "^NearSite" 3; + "^Needle" 3; + "^Nessus" 3; + "^NetAnts" 3; + "^Netcraft" 3; + "^netEstate\ NE\ Crawler" 3; + "^NetLyzer" 3; + "^NetMechanic" 3; + "^NetSpider" 3; + "^Nettrack" 3; + "^Net\ Vampire" 3; + "^NetZIP" 3; + "^NextGenSearchBot" 3; + "^Nibbler" 3; + "^NICErsPRO" 3; + "^Niki-bot" 3; + "^Nikto" 3; + "^NimbleCrawler" 3; + "^Ninja" 3; + "^Nmap" 3; + "^NPbot" 3; + "^Nutch" 3; + "^Octopus" 3; + "^Offline\ Explorer" 3; + "^Offline\ Navigator" 3; + "^Openfind" 3; + "^OpenLinkProfiler" 3; + "^Openvas" 3; + "^OrangeBot" 3; + "^OrangeSpider" 3; + "^OutfoxBot" 3; + "^PageAnalyzer" 3; + "^Page\ Analyzer" 3; + "^PageGrabber" 3; + "^Page\ Grabber" 3; + "^page\ scorer" 3; + "^PageScorer" 3; + "^Panscient" 3; + "^Papa\ Foto" 3; + "^Pavuk" 3; + "^pcBrowser" 3; + "^PECL::HTTP" 3; + "^PeoplePal" 3; + "^PHPCrawl" 3; + "^Picscout" 3; + "^Picsearch" 3; + "^Pimonster" 3; + "^Pi-Monster" 3; + "^Pixray" 3; + "^PleaseCrawl" 3; + "^Pockey" 3; + "^POE-Component-Client-HTTP" 3; + "^Probethenet" 3; + "^ProPowerBot" 3; + "^ProWebWalker" 3; + "^Proximic" 3; + "^Psbot" 3; + "^Pump" 3; + "^PyCurl" 3; + "^QueryN\ Metasearch" 3; + "^Qwantify" 3; + "^RankActiveLinkBot" 3; + "^Rankivabot" 3; + "^RealDownload" 3; + "^Reaper" 3; + "^Recorder" 3; + "^RedesScrapy" 3; + "^ReGet" 3; + "^RepoMonkey" 3; + "^Ripper" 3; + "^RocketCrawler" 3; + "^Rogerbot" 3; + "^SBIder" 3; + "^ScanAlert" 3; + "^Scanbot" 3; + "^Scrapy" 3; + "^Screaming\ Frog\ SEO\ Spider" 3; + "^ScreenerBot" 3; + "^Searchestate" 3; + "^SearchmetricsBot" 3; + "^Semrush" 3; + "^SemrushBot" 3; + "^SEOkicks" 3; + "^SEOkicks-Robot" 3; + "^SEOlyticsCrawler" 3; + "^Seomoz" 3; + "^SEOprofiler" 3; + "^SEOstats" 3; + "^SeznamBot" 3; + "^Siphon" 3; + "^SISTRIX" 3; + "^SISTRIX\ Crawler" 3; + "^Sitebeam" 3; + "^SiteExplorer" 3; + "^Siteimprove" 3; + "^SiteLockSpider" 3; + "^SiteSnagger" 3; + "^SiteSucker" 3; + "^Site\ Sucker" 3; + "^Sitevigil" 3; + "^Slackbot-LinkExpanding" 3; + "^SlySearch" 3; + "^SmartDownload" 3; + "^Snake" 3; + "^Snapbot" 3; + "^Snoopy" 3; + "^SocialRankIOBot" 3; + "^Sogou\ web\ spider" 3; + "^Sosospider" 3; + "^SpaceBison" 3; + "^Spammen" 3; + "^SpankBot" 3; + "^Spanner" 3; + "^Spbot" 3; + "^Spinn3r" 3; + "^SputnikBot" 3; + "^Sqlmap" 3; + "^Sqlworm" 3; + "^Sqworm" 3; + "^Steeler" 3; + "^Stripper" 3; + "^Sucker" 3; + "^Sucuri" 3; + "^SuperBot" 3; + "^SuperHTTP" 3; + "^Surfbot" 3; + "^SurveyBot" 3; + "^Suzuran" 3; + "^Swiftbot" 3; + "^Szukacz" 3; + "^T0PHackTeam" 3; + "^T8Abot" 3; + "^tAkeOut" 3; + "^Teleport" 3; + "^TeleportPro" 3; + "^Telesoft" 3; + "^Telesphoreo" 3; + "^Telesphorep" 3; + "^The\ Intraformant" 3; + "^TheNomad" 3; + "^TightTwatBot" 3; + "^Titan" 3; + "^Toata" 3; + "^Toweyabot" 3; + "^Trendictionbot" 3; + "^True_Robot" 3; + "^Turingos" 3; + "^TurnitinBot" 3; + "^Turnitin\ Bot" 3; + "^Turnitin\ Robot" 3; + "^TwengaBot" 3; + "^Twice" 3; + "^Typhoeus" 3; + "^URLy.Warning" 3; + "^URLy\ Warning" 3; + "^Vacuum" 3; + "^Vagabondo" 3; + "^VB\ Project" 3; + "^VCI" 3; + "^VeriCiteCrawler" 3; + "^VidibleScraper" 3; + "^VoidEYE" 3; + "^Voil" 3; + "^Voltron" 3; + "^WallpapersHD" 3; + "^WASALive-Bot" 3; + "^WBSearchBot" 3; + "^Webalta" 3; + "^WebAuto" 3; + "^Web\ Auto" 3; + "^WebBandit" 3; + "^Web\ Bandit" 3; + "^WebCollage" 3; + "^Web\ Collage" 3; + "^WebCopier" 3; + "^Web\ Copier" 3; + "^WEBDAV" 3; + "^WEBDAV\ Client" 3; + "^WebEnhancer" 3; + "^Web\ Enhancer" 3; + "^WebFetch" 3; + "^Web\ Fetch" 3; + "^WebFuck" 3; + "^Web\ Fuck" 3; + "^WebGo\ IS" 3; + "^WebImageCollector" 3; + "^Web\ Image\ Collector" 3; + "^WebLeacher" 3; + "^WebmasterWorldForumBot" 3; + "^webmeup-crawler" 3; + "^WebPix" 3; + "^Web\ Pix" 3; + "^WebReaper" 3; + "^Web\ Reaper" 3; + "^WebSauger" 3; + "^Web\ Sauger" 3; + "^Webshag" 3; + "^WebsiteExtractor" 3; + "^Website\ Extractor" 3; + "^WebsiteQuester" 3; + "^Website\ Quester" 3; + "^Webster" 3; + "^WebStripper" 3; + "^Web\ Stripper" 3; + "^WebSucker" 3; + "^Web\ Sucker" 3; + "^WebWhacker" 3; + "^Web\ Whacker" 3; + "^WebZIP" 3; + "^WeSEE" 3; + "^Whack" 3; + "^Whacker" 3; + "^Whatweb" 3; + "^Widow" 3; + "^WinHTTrack" 3; + "^WiseGuys\ Robot" 3; + "^WISENutbot" 3; + "^Wonderbot" 3; + "^Woobot" 3; + "^Wotbox" 3; + "^Wprecon" 3; + "^WPScan" 3; + "^WWW-Collector-E" 3; + "^WWW-Mechanize" 3; + "^WWW::Mechanize" 3; + "^WWWOFFLE" 3; + "^Xaldon_WebSpider" 3; + "^Xaldon\ WebSpider" 3; + "^Xenu" 3; + "^Y!J-ASR" 3; + "^YoudaoBot" 3; + "^Zade" 3; + "^Zermelo" 3; + "^Zeus" 3; + "^Zgrab" 3; + "^Zitebot" 3; + "^ZmEu" 3; + "^ZumBot" 3; + "^ZyBorg" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### }