diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index dc76d751c..30cef6b39 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -39,7 +39,7 @@ ### Copyright Mitchell Krog - ### Contributors: Stuart Cardall - https://github.com/itoffshore -### Tested on: nginx/1.10.3 (Ubuntu 16.04) +### Tested on: nginx/1.10.3 up to latest Mainstream Version (Ubuntu 16.04) ### This list was developed and is in use on a live Nginx server running some very busy web sites. ### It was built from the ground up using real data from daily logs and is updated almost daily. diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 57605aa18..def84c20f 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -33,7 +33,7 @@ ### Copyright Mitchell Krog - ### Contributors: Stuart Cardall - https://github.com/itoffshore -### Tested on: nginx/1.10.3 (Ubuntu 16.04) +### Tested on: nginx/1.10.3 up to latest Mainstream Version (Ubuntu 16.04) ### This list was developed and is in use on a live Nginx server running some very busy web sites. ### It was built from the ground up using real data from daily logs and is updated almost daily. @@ -113,6 +113,7 @@ map $http_user_agent $bad_bot { # *********************************************** # Use the include file below to further customize your own list of additional user-agents you wish to permanently block +# This include file allows whitelisting and blacklisting of anything specified below it, meaning you can over-ride anything in this blocker to your liking. # *********************************************************************************** # START CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### @@ -124,6 +125,567 @@ map $http_user_agent $bad_bot { # END CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### # ********************************************************************************* +# ********************************************* +# Bad User-Agent Strings That We Block Outright +# ********************************************* + +# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### + "~*(?:\b)360Spider(?:\b|)" 3; + "~*(?:\b)404checker(?:\b|)" 3; + "~*(?:\b)404enemy(?:\b|)" 3; + "~*(?:\b)80legs(?:\b|)" 3; + "~*(?:\b)Abonti(?:\b|)" 3; + "~*(?:\b)Aboundex(?:\b|)" 3; + "~*(?:\b)Aboundexbot(?:\b|)" 3; + "~*(?:\b)Acunetix(?:\b|)" 3; + "~*(?:\b)ADmantX(?:\b|)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; + "~*(?:\b)AhrefsBot(?:\b|)" 3; + "~*(?:\b)AIBOT(?:\b|)" 3; + "~*(?:\b)AiHitBot(?:\b|)" 3; + "~*(?:\b)Aipbot(?:\b|)" 3; + "~*(?:\b)Alexibot(?:\b|)" 3; + "~*(?:\b)Alligator(?:\b|)" 3; + "~*(?:\b)AllSubmitter(?:\b|)" 3; + "~*(?:\b)AlphaBot(?:\b|)" 3; + "~*(?:\b)Anarchie(?:\b|)" 3; + "~*(?:\b)Apexoo(?:\b|)" 3; + "~*(?:\b)archive.org_bot(?:\b|)" 3; + "~*(?:\b)ASPSeek(?:\b|)" 3; + "~*(?:\b)Asterias(?:\b|)" 3; + "~*(?:\b)Attach(?:\b|)" 3; + "~*(?:\b)autoemailspider(?:\b|)" 3; + "~*(?:\b)BackDoorBot(?:\b|)" 3; + "~*(?:\b)Backlink-Ceck(?:\b|)" 3; + "~*(?:\b)backlink-check(?:\b|)" 3; + "~*(?:\b)BacklinkCrawler(?:\b|)" 3; + "~*(?:\b)BackStreet(?:\b|)" 3; + "~*(?:\b)BackWeb(?:\b|)" 3; + "~*(?:\b)Badass(?:\b|)" 3; + "~*(?:\b)Bandit(?:\b|)" 3; + "~*(?:\b)Barkrowler(?:\b|)" 3; + "~*(?:\b)BatchFTP(?:\b|)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; + "~*(?:\b)BBBike(?:\b|)" 3; + "~*(?:\b)BDCbot(?:\b|)" 3; + "~*(?:\b)BDFetch(?:\b|)" 3; + "~*(?:\b)BetaBot(?:\b|)" 3; + "~*(?:\b)Bigfoot(?:\b|)" 3; + "~*(?:\b)Bitacle(?:\b|)" 3; + "~*(?:\b)Blackboard(?:\b|)" 3; + "~*(?:\b)Black\ Hole(?:\b|)" 3; + "~*(?:\b)BlackWidow(?:\b|)" 3; + "~*(?:\b)BLEXBot(?:\b|)" 3; + "~*(?:\b)Blow(?:\b|)" 3; + "~*(?:\b)BlowFish(?:\b|)" 3; + "~*(?:\b)Boardreader(?:\b|)" 3; + "~*(?:\b)Bolt(?:\b|)" 3; + "~*(?:\b)BotALot(?:\b|)" 3; + "~*(?:\b)Brandprotect(?:\b|)" 3; + "~*(?:\b)Brandwatch(?:\b|)" 3; + "~*(?:\b)Buddy(?:\b|)" 3; + "~*(?:\b)BuiltBotTough(?:\b|)" 3; + "~*(?:\b)BuiltWith(?:\b|)" 3; + "~*(?:\b)Bullseye(?:\b|)" 3; + "~*(?:\b)BunnySlippers(?:\b|)" 3; + "~*(?:\b)BuzzSumo(?:\b|)" 3; + "~*(?:\b)Calculon(?:\b|)" 3; + "~*(?:\b)CATExplorador(?:\b|)" 3; + "~*(?:\b)CazoodleBot(?:\b|)" 3; + "~*(?:\b)CCBot(?:\b|)" 3; + "~*(?:\b)Cegbfeieh(?:\b|)" 3; + "~*(?:\b)CheeseBot(?:\b|)" 3; + "~*(?:\b)CherryPicker(?:\b|)" 3; + "~*(?:\b)CheTeam(?:\b|)" 3; + "~*(?:\b)ChinaClaw(?:\b|)" 3; + "~*(?:\b)Chlooe(?:\b|)" 3; + "~*(?:\b)Claritybot(?:\b|)" 3; + "~*(?:\b)Cliqzbot(?:\b|)" 3; + "~*(?:\b)Cloud\ mapping(?:\b|)" 3; + "~*(?:\b)coccocbot-web(?:\b|)" 3; + "~*(?:\b)Cogentbot(?:\b|)" 3; + "~*(?:\b)cognitiveseo(?:\b|)" 3; + "~*(?:\b)Collector(?:\b|)" 3; + "~*(?:\b)com.plumanalytics(?:\b|)" 3; + "~*(?:\b)Copier(?:\b|)" 3; + "~*(?:\b)CopyRightCheck(?:\b|)" 3; + "~*(?:\b)Copyscape(?:\b|)" 3; + "~*(?:\b)Cosmos(?:\b|)" 3; + "~*(?:\b)Craftbot(?:\b|)" 3; + "~*(?:\b)crawler4j(?:\b|)" 3; + "~*(?:\b)crawler.feedback(?:\b|)" 3; + "~*(?:\b)crawl.sogou.com(?:\b|)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; + "~*(?:\b)Crescent(?:\b|)" 3; + "~*(?:\b)CrunchBot(?:\b|)" 3; + "~*(?:\b)CSHttp(?:\b|)" 3; + "~*(?:\b)Curious(?:\b|)" 3; + "~*(?:\b)Custo(?:\b|)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; + "~*(?:\b)DataCha0s(?:\b|)" 3; + "~*(?:\b)DBLBot(?:\b|)" 3; + "~*(?:\b)demandbase-bot(?:\b|)" 3; + "~*(?:\b)Demon(?:\b|)" 3; + "~*(?:\b)Deusu(?:\b|)" 3; + "~*(?:\b)Devil(?:\b|)" 3; + "~*(?:\b)Digincore(?:\b|)" 3; + "~*(?:\b)DigitalPebble(?:\b|)" 3; + "~*(?:\b)DIIbot(?:\b|)" 3; + "~*(?:\b)Dirbuster(?:\b|)" 3; + "~*(?:\b)Disco(?:\b|)" 3; + "~*(?:\b)Discobot(?:\b|)" 3; + "~*(?:\b)Discoverybot(?:\b|)" 3; + "~*(?:\b)Dispatch(?:\b|)" 3; + "~*(?:\b)DittoSpyder(?:\b|)" 3; + "~*(?:\b)DnyzBot(?:\b|)" 3; + "~*(?:\b)DomainAppender(?:\b|)" 3; + "~*(?:\b)DomainCrawler(?:\b|)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; + "~*(?:\b)DomainStatsBot(?:\b|)" 3; + "~*(?:\b)Dotbot(?:\b|)" 3; + "~*(?:\b)Download\ Wonder(?:\b|)" 3; + "~*(?:\b)Dragonfly(?:\b|)" 3; + "~*(?:\b)Drip(?:\b|)" 3; + "~*(?:\b)DSearch(?:\b|)" 3; + "~*(?:\b)DTS\ Agent(?:\b|)" 3; + "~*(?:\b)EasyDL(?:\b|)" 3; + "~*(?:\b)Ebingbong(?:\b|)" 3; + "~*(?:\b)eCatch(?:\b|)" 3; + "~*(?:\b)ECCP/1.0(?:\b|)" 3; + "~*(?:\b)Ecxi(?:\b|)" 3; + "~*(?:\b)EirGrabber(?:\b|)" 3; + "~*(?:\b)EMail\ Siphon(?:\b|)" 3; + "~*(?:\b)EMail\ Wolf(?:\b|)" 3; + "~*(?:\b)EroCrawler(?:\b|)" 3; + "~*(?:\b)evc-batch(?:\b|)" 3; + "~*(?:\b)Evil(?:\b|)" 3; + "~*(?:\b)Exabot(?:\b|)" 3; + "~*(?:\b)Express\ WebPictures(?:\b|)" 3; + "~*(?:\b)ExtLinksBot(?:\b|)" 3; + "~*(?:\b)Extractor(?:\b|)" 3; + "~*(?:\b)ExtractorPro(?:\b|)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; + "~*(?:\b)EyeNetIE(?:\b|)" 3; + "~*(?:\b)Ezooms(?:\b|)" 3; + "~*(?:\b)facebookscraper(?:\b|)" 3; + "~*(?:\b)FDM(?:\b|)" 3; + "~*(?:\b)FemtosearchBot(?:\b|)" 3; + "~*(?:\b)FHscan(?:\b|)" 3; + "~*(?:\b)Fimap(?:\b|)" 3; + "~*(?:\b)Firefox/7.0(?:\b|)" 3; + "~*(?:\b)FlashGet(?:\b|)" 3; + "~*(?:\b)Flunky(?:\b|)" 3; + "~*(?:\b)Foobot(?:\b|)" 3; + "~*(?:\b)Freeuploader(?:\b|)" 3; + "~*(?:\b)FrontPage(?:\b|)" 3; + "~*(?:\b)FyberSpider(?:\b|)" 3; + "~*(?:\b)Fyrebot(?:\b|)" 3; + "~*(?:\b)GalaxyBot(?:\b|)" 3; + "~*(?:\b)Genieo(?:\b|)" 3; + "~*(?:\b)GermCrawler(?:\b|)" 3; + "~*(?:\b)Getintent(?:\b|)" 3; + "~*(?:\b)GetRight(?:\b|)" 3; + "~*(?:\b)GetWeb(?:\b|)" 3; + "~*(?:\b)Gigablast(?:\b|)" 3; + "~*(?:\b)Gigabot(?:\b|)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; + "~*(?:\b)Gotit(?:\b|)" 3; + "~*(?:\b)GoZilla(?:\b|)" 3; + "~*(?:\b)Go!Zilla(?:\b|)" 3; + "~*(?:\b)Grabber(?:\b|)" 3; + "~*(?:\b)GrabNet(?:\b|)" 3; + "~*(?:\b)Grafula(?:\b|)" 3; + "~*(?:\b)GrapeFX(?:\b|)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; + "~*(?:\b)GridBot(?:\b|)" 3; + "~*(?:\b)GT::WWW(?:\b|)" 3; + "~*(?:\b)Haansoft(?:\b|)" 3; + "~*(?:\b)HaosouSpider(?:\b|)" 3; + "~*(?:\b)Harvest(?:\b|)" 3; + "~*(?:\b)Havij(?:\b|)" 3; + "~*(?:\b)HEADMasterSEO(?:\b|)" 3; + "~*(?:\b)Heritrix(?:\b|)" 3; + "~*(?:\b)Hloader(?:\b|)" 3; + "~*(?:\b)HMView(?:\b|)" 3; + "~*(?:\b)HTMLparser(?:\b|)" 3; + "~*(?:\b)HTTP::Lite(?:\b|)" 3; + "~*(?:\b)HTTrack(?:\b|)" 3; + "~*(?:\b)Humanlinks(?:\b|)" 3; + "~*(?:\b)HybridBot(?:\b|)" 3; + "~*(?:\b)Iblog(?:\b|)" 3; + "~*(?:\b)IDBot(?:\b|)" 3; + "~*(?:\b)Id-search(?:\b|)" 3; + "~*(?:\b)IlseBot(?:\b|)" 3; + "~*(?:\b)Image\ Fetch(?:\b|)" 3; + "~*(?:\b)Image\ Sucker(?:\b|)" 3; + "~*(?:\b)IndeedBot(?:\b|)" 3; + "~*(?:\b)Indy\ Library(?:\b|)" 3; + "~*(?:\b)InfoNaviRobot(?:\b|)" 3; + "~*(?:\b)InfoTekies(?:\b|)" 3; + "~*(?:\b)instabid(?:\b|)" 3; + "~*(?:\b)Intelliseek(?:\b|)" 3; + "~*(?:\b)InterGET(?:\b|)" 3; + "~*(?:\b)Internet\ Ninja(?:\b|)" 3; + "~*(?:\b)InternetSeer(?:\b|)" 3; + "~*(?:\b)internetVista\ monitor(?:\b|)" 3; + "~*(?:\b)ips-agent(?:\b|)" 3; + "~*(?:\b)Iria(?:\b|)" 3; + "~*(?:\b)IRLbot(?:\b|)" 3; + "~*(?:\b)Iskanie(?:\b|)" 3; + "~*(?:\b)IstellaBot(?:\b|)" 3; + "~*(?:\b)JamesBOT(?:\b|)" 3; + "~*(?:\b)Jbrofuzz(?:\b|)" 3; + "~*(?:\b)JennyBot(?:\b|)" 3; + "~*(?:\b)JetCar(?:\b|)" 3; + "~*(?:\b)Jetty(?:\b|)" 3; + "~*(?:\b)JikeSpider(?:\b|)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; + "~*(?:\b)Joomla(?:\b|)" 3; + "~*(?:\b)Jorgee(?:\b|)" 3; + "~*(?:\b)JustView(?:\b|)" 3; + "~*(?:\b)Jyxobot(?:\b|)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; + "~*(?:\b)Keyword\ Density(?:\b|)" 3; + "~*(?:\b)Kozmosbot(?:\b|)" 3; + "~*(?:\b)Lanshanbot(?:\b|)" 3; + "~*(?:\b)Larbin(?:\b|)" 3; + "~*(?:\b)LeechFTP(?:\b|)" 3; + "~*(?:\b)LeechGet(?:\b|)" 3; + "~*(?:\b)LexiBot(?:\b|)" 3; + "~*(?:\b)Lftp(?:\b|)" 3; + "~*(?:\b)LibWeb(?:\b|)" 3; + "~*(?:\b)Libwhisker(?:\b|)" 3; + "~*(?:\b)Lightspeedsystems(?:\b|)" 3; + "~*(?:\b)Likse(?:\b|)" 3; + "~*(?:\b)Linkdexbot(?:\b|)" 3; + "~*(?:\b)LinkextractorPro(?:\b|)" 3; + "~*(?:\b)LinkpadBot(?:\b|)" 3; + "~*(?:\b)LinkScan(?:\b|)" 3; + "~*(?:\b)LinksManager(?:\b|)" 3; + "~*(?:\b)LinkWalker(?:\b|)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; + "~*(?:\b)Lipperhey(?:\b|)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; + "~*(?:\b)Litemage_walker(?:\b|)" 3; + "~*(?:\b)Lmspider(?:\b|)" 3; + "~*(?:\b)LNSpiderguy(?:\b|)" 3; + "~*(?:\b)Ltx71(?:\b|)" 3; + "~*(?:\b)lwp-request(?:\b|)" 3; + "~*(?:\b)LWP::Simple(?:\b|)" 3; + "~*(?:\b)lwp-trivial(?:\b|)" 3; + "~*(?:\b)Magnet(?:\b|)" 3; + "~*(?:\b)Mag-Net(?:\b|)" 3; + "~*(?:\b)magpie-crawler(?:\b|)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; + "~*(?:\b)Majestic12(?:\b|)" 3; + "~*(?:\b)Majestic-SEO(?:\b|)" 3; + "~*(?:\b)Majestic\ SEO(?:\b|)" 3; + "~*(?:\b)MarkMonitor(?:\b|)" 3; + "~*(?:\b)MarkWatch(?:\b|)" 3; + "~*(?:\b)Masscan(?:\b|)" 3; + "~*(?:\b)Mass\ Downloader(?:\b|)" 3; + "~*(?:\b)Mata\ Hari(?:\b|)" 3; + "~*(?:\b)MauiBot(?:\b|)" 3; + "~*(?:\b)meanpathbot(?:\b|)" 3; + "~*(?:\b)Meanpathbot(?:\b|)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; + "~*(?:\b)mediawords(?:\b|)" 3; + "~*(?:\b)MegaIndex.ru(?:\b|)" 3; + "~*(?:\b)Metauri(?:\b|)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; + "~*(?:\b)MIDown\ tool(?:\b|)" 3; + "~*(?:\b)MIIxpc(?:\b|)" 3; + "~*(?:\b)Mister\ PiX(?:\b|)" 3; + "~*(?:\b)MJ12bot(?:\b|)" 3; + "~*(?:\b)Mojeek(?:\b|)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; + "~*(?:\b)Mr.4x3(?:\b|)" 3; + "~*(?:\b)MSFrontPage(?:\b|)" 3; + "~*(?:\b)MSIECrawler(?:\b|)" 3; + "~*(?:\b)Msrabot(?:\b|)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; + "~*(?:\b)muhstik-scan(?:\b|)" 3; + "~*(?:\b)Musobot(?:\b|)" 3; + "~*(?:\b)Name\ Intelligence(?:\b|)" 3; + "~*(?:\b)Nameprotect(?:\b|)" 3; + "~*(?:\b)Navroad(?:\b|)" 3; + "~*(?:\b)NearSite(?:\b|)" 3; + "~*(?:\b)Needle(?:\b|)" 3; + "~*(?:\b)Nessus(?:\b|)" 3; + "~*(?:\b)NetAnts(?:\b|)" 3; + "~*(?:\b)Netcraft(?:\b|)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; + "~*(?:\b)NetLyzer(?:\b|)" 3; + "~*(?:\b)NetMechanic(?:\b|)" 3; + "~*(?:\b)NetSpider(?:\b|)" 3; + "~*(?:\b)Nettrack(?:\b|)" 3; + "~*(?:\b)Net\ Vampire(?:\b|)" 3; + "~*(?:\b)Netvibes(?:\b|)" 3; + "~*(?:\b)NetZIP(?:\b|)" 3; + "~*(?:\b)NextGenSearchBot(?:\b|)" 3; + "~*(?:\b)Nibbler(?:\b|)" 3; + "~*(?:\b)NICErsPRO(?:\b|)" 3; + "~*(?:\b)Niki-bot(?:\b|)" 3; + "~*(?:\b)Nikto(?:\b|)" 3; + "~*(?:\b)NimbleCrawler(?:\b|)" 3; + "~*(?:\b)Nimbostratus(?:\b|)" 3; + "~*(?:\b)Ninja(?:\b|)" 3; + "~*(?:\b)Nmap(?:\b|)" 3; + "~*(?:\b)NPbot(?:\b|)" 3; + "~*(?:\b)Nutch(?:\b|)" 3; + "~*(?:\b)oBot(?:\b|)" 3; + "~*(?:\b)Octopus(?:\b|)" 3; + "~*(?:\b)Offline\ Explorer(?:\b|)" 3; + "~*(?:\b)Offline\ Navigator(?:\b|)" 3; + "~*(?:\b)OnCrawl(?:\b|)" 3; + "~*(?:\b)Openfind(?:\b|)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; + "~*(?:\b)Openvas(?:\b|)" 3; + "~*(?:\b)OpenVAS(?:\b|)" 3; + "~*(?:\b)OrangeBot(?:\b|)" 3; + "~*(?:\b)OrangeSpider(?:\b|)" 3; + "~*(?:\b)OutclicksBot(?:\b|)" 3; + "~*(?:\b)OutfoxBot(?:\b|)" 3; + "~*(?:\b)PageAnalyzer(?:\b|)" 3; + "~*(?:\b)Page\ Analyzer(?:\b|)" 3; + "~*(?:\b)PageGrabber(?:\b|)" 3; + "~*(?:\b)page\ scorer(?:\b|)" 3; + "~*(?:\b)PageScorer(?:\b|)" 3; + "~*(?:\b)Pandalytics(?:\b|)" 3; + "~*(?:\b)Panscient(?:\b|)" 3; + "~*(?:\b)Papa\ Foto(?:\b|)" 3; + "~*(?:\b)Pavuk(?:\b|)" 3; + "~*(?:\b)pcBrowser(?:\b|)" 3; + "~*(?:\b)PECL::HTTP(?:\b|)" 3; + "~*(?:\b)PeoplePal(?:\b|)" 3; + "~*(?:\b)PHPCrawl(?:\b|)" 3; + "~*(?:\b)Picscout(?:\b|)" 3; + "~*(?:\b)Picsearch(?:\b|)" 3; + "~*(?:\b)PictureFinder(?:\b|)" 3; + "~*(?:\b)Pimonster(?:\b|)" 3; + "~*(?:\b)Pi-Monster(?:\b|)" 3; + "~*(?:\b)Pixray(?:\b|)" 3; + "~*(?:\b)PleaseCrawl(?:\b|)" 3; + "~*(?:\b)plumanalytics(?:\b|)" 3; + "~*(?:\b)Pockey(?:\b|)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; + "~*(?:\b)Probethenet(?:\b|)" 3; + "~*(?:\b)ProPowerBot(?:\b|)" 3; + "~*(?:\b)ProWebWalker(?:\b|)" 3; + "~*(?:\b)Psbot(?:\b|)" 3; + "~*(?:\b)Pump(?:\b|)" 3; + "~*(?:\b)PxBroker(?:\b|)" 3; + "~*(?:\b)PyCurl(?:\b|)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; + "~*(?:\b)Quick-Crawler(?:\b|)" 3; + "~*(?:\b)RankActive(?:\b|)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; + "~*(?:\b)RankFlex(?:\b|)" 3; + "~*(?:\b)RankingBot(?:\b|)" 3; + "~*(?:\b)RankingBot2(?:\b|)" 3; + "~*(?:\b)Rankivabot(?:\b|)" 3; + "~*(?:\b)RankurBot(?:\b|)" 3; + "~*(?:\b)RealDownload(?:\b|)" 3; + "~*(?:\b)Reaper(?:\b|)" 3; + "~*(?:\b)RebelMouse(?:\b|)" 3; + "~*(?:\b)Recorder(?:\b|)" 3; + "~*(?:\b)RedesScrapy(?:\b|)" 3; + "~*(?:\b)ReGet(?:\b|)" 3; + "~*(?:\b)RepoMonkey(?:\b|)" 3; + "~*(?:\b)Ripper(?:\b|)" 3; + "~*(?:\b)RocketCrawler(?:\b|)" 3; + "~*(?:\b)Rogerbot(?:\b|)" 3; + "~*(?:\b)s1z.ru(?:\b|)" 3; + "~*(?:\b)SalesIntelligent(?:\b|)" 3; + "~*(?:\b)SBIder(?:\b|)" 3; + "~*(?:\b)ScanAlert(?:\b|)" 3; + "~*(?:\b)Scanbot(?:\b|)" 3; + "~*(?:\b)scan.lol(?:\b|)" 3; + "~*(?:\b)ScoutJet(?:\b|)" 3; + "~*(?:\b)Scrapy(?:\b|)" 3; + "~*(?:\b)Screaming(?:\b|)" 3; + "~*(?:\b)ScreenerBot(?:\b|)" 3; + "~*(?:\b)Searchestate(?:\b|)" 3; + "~*(?:\b)SearchmetricsBot(?:\b|)" 3; + "~*(?:\b)Semrush(?:\b|)" 3; + "~*(?:\b)SemrushBot(?:\b|)" 3; + "~*(?:\b)SEOkicks(?:\b|)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; + "~*(?:\b)Seomoz(?:\b|)" 3; + "~*(?:\b)SEOprofiler(?:\b|)" 3; + "~*(?:\b)seoscanners(?:\b|)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; + "~*(?:\b)SEOstats(?:\b|)" 3; + "~*(?:\b)serpstatbot(?:\b|)" 3; + "~*(?:\b)sexsearcher(?:\b|)" 3; + "~*(?:\b)Shodan(?:\b|)" 3; + "~*(?:\b)Siphon(?:\b|)" 3; + "~*(?:\b)SISTRIX(?:\b|)" 3; + "~*(?:\b)Sitebeam(?:\b|)" 3; + "~*(?:\b)SiteExplorer(?:\b|)" 3; + "~*(?:\b)Siteimprove(?:\b|)" 3; + "~*(?:\b)SiteLockSpider(?:\b|)" 3; + "~*(?:\b)SiteSnagger(?:\b|)" 3; + "~*(?:\b)SiteSucker(?:\b|)" 3; + "~*(?:\b)Site\ Sucker(?:\b|)" 3; + "~*(?:\b)Sitevigil(?:\b|)" 3; + "~*(?:\b)SlySearch(?:\b|)" 3; + "~*(?:\b)SmartDownload(?:\b|)" 3; + "~*(?:\b)SMTBot(?:\b|)" 3; + "~*(?:\b)Snake(?:\b|)" 3; + "~*(?:\b)Snapbot(?:\b|)" 3; + "~*(?:\b)Snoopy(?:\b|)" 3; + "~*(?:\b)SocialRankIOBot(?:\b|)" 3; + "~*(?:\b)Sociscraper(?:\b|)" 3; + "~*(?:\b)sogouspider(?:\b|)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; + "~*(?:\b)Sosospider(?:\b|)" 3; + "~*(?:\b)Sottopop(?:\b|)" 3; + "~*(?:\b)SpaceBison(?:\b|)" 3; + "~*(?:\b)Spammen(?:\b|)" 3; + "~*(?:\b)SpankBot(?:\b|)" 3; + "~*(?:\b)Spanner(?:\b|)" 3; + "~*(?:\b)sp_auditbot(?:\b|)" 3; + "~*(?:\b)Spbot(?:\b|)" 3; + "~*(?:\b)Spinn3r(?:\b|)" 3; + "~*(?:\b)SputnikBot(?:\b|)" 3; + "~*(?:\b)spyfu(?:\b|)" 3; + "~*(?:\b)Sqlmap(?:\b|)" 3; + "~*(?:\b)Sqlworm(?:\b|)" 3; + "~*(?:\b)Sqworm(?:\b|)" 3; + "~*(?:\b)Steeler(?:\b|)" 3; + "~*(?:\b)Stripper(?:\b|)" 3; + "~*(?:\b)Sucker(?:\b|)" 3; + "~*(?:\b)Sucuri(?:\b|)" 3; + "~*(?:\b)SuperBot(?:\b|)" 3; + "~*(?:\b)SuperHTTP(?:\b|)" 3; + "~*(?:\b)Surfbot(?:\b|)" 3; + "~*(?:\b)SurveyBot(?:\b|)" 3; + "~*(?:\b)Suzuran(?:\b|)" 3; + "~*(?:\b)Swiftbot(?:\b|)" 3; + "~*(?:\b)sysscan(?:\b|)" 3; + "~*(?:\b)Szukacz(?:\b|)" 3; + "~*(?:\b)T0PHackTeam(?:\b|)" 3; + "~*(?:\b)T8Abot(?:\b|)" 3; + "~*(?:\b)tAkeOut(?:\b|)" 3; + "~*(?:\b)Teleport(?:\b|)" 3; + "~*(?:\b)TeleportPro(?:\b|)" 3; + "~*(?:\b)Telesoft(?:\b|)" 3; + "~*(?:\b)Telesphoreo(?:\b|)" 3; + "~*(?:\b)Telesphorep(?:\b|)" 3; + "~*(?:\b)The\ Intraformant(?:\b|)" 3; + "~*(?:\b)TheNomad(?:\b|)" 3; + "~*(?:\b)TightTwatBot(?:\b|)" 3; + "~*(?:\b)Titan(?:\b|)" 3; + "~*(?:\b)Toata(?:\b|)" 3; + "~*(?:\b)Toweyabot(?:\b|)" 3; + "~*(?:\b)Tracemyfile(?:\b|)" 3; + "~*(?:\b)Trendiction(?:\b|)" 3; + "~*(?:\b)Trendictionbot(?:\b|)" 3; + "~*(?:\b)trendiction.com(?:\b|)" 3; + "~*(?:\b)trendiction.de(?:\b|)" 3; + "~*(?:\b)True_Robot(?:\b|)" 3; + "~*(?:\b)Turingos(?:\b|)" 3; + "~*(?:\b)Turnitin(?:\b|)" 3; + "~*(?:\b)TurnitinBot(?:\b|)" 3; + "~*(?:\b)TwengaBot(?:\b|)" 3; + "~*(?:\b)Twice(?:\b|)" 3; + "~*(?:\b)Typhoeus(?:\b|)" 3; + "~*(?:\b)UnisterBot(?:\b|)" 3; + "~*(?:\b)Upflow(?:\b|)" 3; + "~*(?:\b)URLy.Warning(?:\b|)" 3; + "~*(?:\b)URLy\ Warning(?:\b|)" 3; + "~*(?:\b)Vacuum(?:\b|)" 3; + "~*(?:\b)Vagabondo(?:\b|)" 3; + "~*(?:\b)VB\ Project(?:\b|)" 3; + "~*(?:\b)VCI(?:\b|)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; + "~*(?:\b)VidibleScraper(?:\b|)" 3; + "~*(?:\b)Virusdie(?:\b|)" 3; + "~*(?:\b)VoidEYE(?:\b|)" 3; + "~*(?:\b)Voil(?:\b|)" 3; + "~*(?:\b)Voltron(?:\b|)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; + "~*(?:\b)WallpapersHD(?:\b|)" 3; + "~*(?:\b)WASALive-Bot(?:\b|)" 3; + "~*(?:\b)WBSearchBot(?:\b|)" 3; + "~*(?:\b)Webalta(?:\b|)" 3; + "~*(?:\b)WebAuto(?:\b|)" 3; + "~*(?:\b)Web\ Auto(?:\b|)" 3; + "~*(?:\b)WebBandit(?:\b|)" 3; + "~*(?:\b)WebCollage(?:\b|)" 3; + "~*(?:\b)Web\ Collage(?:\b|)" 3; + "~*(?:\b)WebCopier(?:\b|)" 3; + "~*(?:\b)WEBDAV(?:\b|)" 3; + "~*(?:\b)WebEnhancer(?:\b|)" 3; + "~*(?:\b)Web\ Enhancer(?:\b|)" 3; + "~*(?:\b)WebFetch(?:\b|)" 3; + "~*(?:\b)Web\ Fetch(?:\b|)" 3; + "~*(?:\b)WebFuck(?:\b|)" 3; + "~*(?:\b)Web\ Fuck(?:\b|)" 3; + "~*(?:\b)WebGo\ IS(?:\b|)" 3; + "~*(?:\b)WebImageCollector(?:\b|)" 3; + "~*(?:\b)WebLeacher(?:\b|)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; + "~*(?:\b)webmeup-crawler(?:\b|)" 3; + "~*(?:\b)WebPix(?:\b|)" 3; + "~*(?:\b)Web\ Pix(?:\b|)" 3; + "~*(?:\b)WebReaper(?:\b|)" 3; + "~*(?:\b)WebSauger(?:\b|)" 3; + "~*(?:\b)Web\ Sauger(?:\b|)" 3; + "~*(?:\b)Webshag(?:\b|)" 3; + "~*(?:\b)WebsiteExtractor(?:\b|)" 3; + "~*(?:\b)WebsiteQuester(?:\b|)" 3; + "~*(?:\b)Website\ Quester(?:\b|)" 3; + "~*(?:\b)Webster(?:\b|)" 3; + "~*(?:\b)WebStripper(?:\b|)" 3; + "~*(?:\b)WebSucker(?:\b|)" 3; + "~*(?:\b)Web\ Sucker(?:\b|)" 3; + "~*(?:\b)WebWhacker(?:\b|)" 3; + "~*(?:\b)WebZIP(?:\b|)" 3; + "~*(?:\b)WeSEE(?:\b|)" 3; + "~*(?:\b)Whack(?:\b|)" 3; + "~*(?:\b)Whacker(?:\b|)" 3; + "~*(?:\b)Whatweb(?:\b|)" 3; + "~*(?:\b)Who.is\ Bot(?:\b|)" 3; + "~*(?:\b)Widow(?:\b|)" 3; + "~*(?:\b)WinHTTrack(?:\b|)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; + "~*(?:\b)WISENutbot(?:\b|)" 3; + "~*(?:\b)Wonderbot(?:\b|)" 3; + "~*(?:\b)Woobot(?:\b|)" 3; + "~*(?:\b)Wotbox(?:\b|)" 3; + "~*(?:\b)Wprecon(?:\b|)" 3; + "~*(?:\b)WPScan(?:\b|)" 3; + "~*(?:\b)WWW-Collector-E(?:\b|)" 3; + "~*(?:\b)WWW-Mechanize(?:\b|)" 3; + "~*(?:\b)WWW::Mechanize(?:\b|)" 3; + "~*(?:\b)WWWOFFLE(?:\b|)" 3; + "~*(?:\b)x09Mozilla(?:\b|)" 3; + "~*(?:\b)x22Mozilla(?:\b|)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; + "~*(?:\b)Xenu(?:\b|)" 3; + "~*(?:\b)xpymep1.exe(?:\b|)" 3; + "~*(?:\b)YoudaoBot(?:\b|)" 3; + "~*(?:\b)Zade(?:\b|)" 3; + "~*(?:\b)Zauba(?:\b|)" 3; + "~*(?:\b)zauba.io(?:\b|)" 3; + "~*(?:\b)Zermelo(?:\b|)" 3; + "~*(?:\b)Zeus(?:\b|)" 3; + "~*(?:\b)zgrab(?:\b|)" 3; + "~*(?:\b)Zitebot(?:\b|)" 3; + "~*(?:\b)ZmEu(?:\b|)" 3; + "~*(?:\b)ZumBot(?:\b|)" 3; + "~*(?:\b)ZyBorg(?:\b|)" 3; +# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # *********************************************** # Allow Good User-Agent Strings We Know and Trust @@ -203,571 +765,7 @@ map $http_user_agent $bad_bot { "~*\bYandexBot\b" 2; "~*\bYandexImages\b" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - -# ********************************************* -# Bad User-Agent Strings That We Block Outright -# ********************************************* -# This includes: -# Known Vulnerability Scanners (now merged into one section) - -# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*\b360Spider\b" 3; - "~*\b404checker\b" 3; - "~*\b404enemy\b" 3; - "~*\b80legs\b" 3; - "~*\bAbonti\b" 3; - "~*\bAboundex\b" 3; - "~*\bAboundexbot\b" 3; - "~*\bAcunetix\b" 3; - "~*\bADmantX\b" 3; - "~*\bAfD-Verbotsverfahren\b" 3; - "~*\bAhrefsBot\b" 3; - "~*\bAIBOT\b" 3; - "~*\bAiHitBot\b" 3; - "~*\bAipbot\b" 3; - "~*\bAlexibot\b" 3; - "~*\bAlligator\b" 3; - "~*\bAllSubmitter\b" 3; - "~*\bAlphaBot\b" 3; - "~*\bAnarchie\b" 3; - "~*\bApexoo\b" 3; - "~*\barchive.org_bot\b" 3; - "~*\bASPSeek\b" 3; - "~*\bAsterias\b" 3; - "~*\bAttach\b" 3; - "~*\bautoemailspider\b" 3; - "~*\bBackDoorBot\b" 3; - "~*\bBacklink-Ceck\b" 3; - "~*\bbacklink-check\b" 3; - "~*\bBacklinkCrawler\b" 3; - "~*\bBackStreet\b" 3; - "~*\bBackWeb\b" 3; - "~*\bBadass\b" 3; - "~*\bBandit\b" 3; - "~*\bBarkrowler\b" 3; - "~*\bBatchFTP\b" 3; - "~*\bBattleztar\ Bazinga\b" 3; - "~*\bBBBike\b" 3; - "~*\bBDCbot\b" 3; - "~*\bBDFetch\b" 3; - "~*\bBetaBot\b" 3; - "~*\bBigfoot\b" 3; - "~*\bBitacle\b" 3; - "~*\bBlackboard\b" 3; - "~*\bBlack\ Hole\b" 3; - "~*\bBlackWidow\b" 3; - "~*\bBLEXBot\b" 3; - "~*\bBlow\b" 3; - "~*\bBlowFish\b" 3; - "~*\bBoardreader\b" 3; - "~*\bBolt\b" 3; - "~*\bBotALot\b" 3; - "~*\bBrandprotect\b" 3; - "~*\bBrandwatch\b" 3; - "~*\bBuddy\b" 3; - "~*\bBuiltBotTough\b" 3; - "~*\bBuiltWith\b" 3; - "~*\bBullseye\b" 3; - "~*\bBunnySlippers\b" 3; - "~*\bBuzzSumo\b" 3; - "~*\bCalculon\b" 3; - "~*\bCATExplorador\b" 3; - "~*\bCazoodleBot\b" 3; - "~*\bCCBot\b" 3; - "~*\bCegbfeieh\b" 3; - "~*\bCheeseBot\b" 3; - "~*\bCherryPicker\b" 3; - "~*\bCheTeam\b" 3; - "~*\bChinaClaw\b" 3; - "~*\bChlooe\b" 3; - "~*\bClaritybot\b" 3; - "~*\bCliqzbot\b" 3; - "~*\bCloud\ mapping\b" 3; - "~*\bcoccocbot-web\b" 3; - "~*\bCogentbot\b" 3; - "~*\bcognitiveseo\b" 3; - "~*\bCollector\b" 3; - "~*\bcom.plumanalytics\b" 3; - "~*\bCopier\b" 3; - "~*\bCopyRightCheck\b" 3; - "~*\bCopyscape\b" 3; - "~*\bCosmos\b" 3; - "~*\bCraftbot\b" 3; - "~*\bcrawler4j\b" 3; - "~*\bcrawler.feedback\b" 3; - "~*\bcrawl.sogou.com\b" 3; - "~*\bCrazyWebCrawler\b" 3; - "~*\bCrescent\b" 3; - "~*\bCrunchBot\b" 3; - "~*\bCSHttp\b" 3; - "~*\bCurious\b" 3; - "~*\bCusto\b" 3; - "~*\bDatabaseDriverMysqli\b" 3; - "~*\bDataCha0s\b" 3; - "~*\bDBLBot\b" 3; - "~*\bdemandbase-bot\b" 3; - "~*\bDemon\b" 3; - "~*\bDeusu\b" 3; - "~*\bDevil\b" 3; - "~*\bDigincore\b" 3; - "~*\bDigitalPebble\b" 3; - "~*\bDIIbot\b" 3; - "~*\bDirbuster\b" 3; - "~*\bDisco\b" 3; - "~*\bDiscobot\b" 3; - "~*\bDiscoverybot\b" 3; - "~*\bDispatch\b" 3; - "~*\bDittoSpyder\b" 3; - "~*\bDnyzBot\b" 3; - "~*\bDomainAppender\b" 3; - "~*\bDomainCrawler\b" 3; - "~*\bDomainSigmaCrawler\b" 3; - "~*\bDomainStatsBot\b" 3; - "~*\bDotbot\b" 3; - "~*\bDownload\ Wonder\b" 3; - "~*\bDragonfly\b" 3; - "~*\bDrip\b" 3; - "~*\bDSearch\b" 3; - "~*\bDTS\ Agent\b" 3; - "~*\bEasyDL\b" 3; - "~*\bEbingbong\b" 3; - "~*\beCatch\b" 3; - "~*\bECCP/1.0\b" 3; - "~*\bEcxi\b" 3; - "~*\bEirGrabber\b" 3; - "~*\bEMail\ Siphon\b" 3; - "~*\bEMail\ Wolf\b" 3; - "~*\bEroCrawler\b" 3; - "~*\bevc-batch\b" 3; - "~*\bEvil\b" 3; - "~*\bExabot\b" 3; - "~*\bExpress\ WebPictures\b" 3; - "~*\bExtLinksBot\b" 3; - "~*\bExtractor\b" 3; - "~*\bExtractorPro\b" 3; - "~*\bExtreme\ Picture\ Finder\b" 3; - "~*\bEyeNetIE\b" 3; - "~*\bEzooms\b" 3; - "~*\bfacebookscraper\b" 3; - "~*\bFDM\b" 3; - "~*\bFemtosearchBot\b" 3; - "~*\bFHscan\b" 3; - "~*\bFimap\b" 3; - "~*\bFirefox/7.0\b" 3; - "~*\bFlashGet\b" 3; - "~*\bFlunky\b" 3; - "~*\bFoobot\b" 3; - "~*\bFreeuploader\b" 3; - "~*\bFrontPage\b" 3; - "~*\bFyberSpider\b" 3; - "~*\bFyrebot\b" 3; - "~*\bGalaxyBot\b" 3; - "~*\bGenieo\b" 3; - "~*\bGermCrawler\b" 3; - "~*\bGetintent\b" 3; - "~*\bGetRight\b" 3; - "~*\bGetWeb\b" 3; - "~*\bGigablast\b" 3; - "~*\bGigabot\b" 3; - "~*\bG-i-g-a-b-o-t\b" 3; - "~*\bGo-Ahead-Got-It\b" 3; - "~*\bGotit\b" 3; - "~*\bGoZilla\b" 3; - "~*\bGo!Zilla\b" 3; - "~*\bGrabber\b" 3; - "~*\bGrabNet\b" 3; - "~*\bGrafula\b" 3; - "~*\bGrapeFX\b" 3; - "~*\bGrapeshotCrawler\b" 3; - "~*\bGridBot\b" 3; - "~*\bGT::WWW\b" 3; - "~*\bHaansoft\b" 3; - "~*\bHaosouSpider\b" 3; - "~*\bHarvest\b" 3; - "~*\bHavij\b" 3; - "~*\bHEADMasterSEO\b" 3; - "~*\bHeritrix\b" 3; - "~*\bHloader\b" 3; - "~*\bHMView\b" 3; - "~*\bHTMLparser\b" 3; - "~*\bHTTP::Lite\b" 3; - "~*\bHTTrack\b" 3; - "~*\bHumanlinks\b" 3; - "~*\bHybridBot\b" 3; - "~*\bIblog\b" 3; - "~*\bIDBot\b" 3; - "~*\bId-search\b" 3; - "~*\bIlseBot\b" 3; - "~*\bImage\ Fetch\b" 3; - "~*\bImage\ Sucker\b" 3; - "~*\bIndeedBot\b" 3; - "~*\bIndy\ Library\b" 3; - "~*\bInfoNaviRobot\b" 3; - "~*\bInfoTekies\b" 3; - "~*\binstabid\b" 3; - "~*\bIntelliseek\b" 3; - "~*\bInterGET\b" 3; - "~*\bInternet\ Ninja\b" 3; - "~*\bInternetSeer\b" 3; - "~*\binternetVista\ monitor\b" 3; - "~*\bips-agent\b" 3; - "~*\bIria\b" 3; - "~*\bIRLbot\b" 3; - "~*\bIskanie\b" 3; - "~*\bIstellaBot\b" 3; - "~*\bJamesBOT\b" 3; - "~*\bJbrofuzz\b" 3; - "~*\bJennyBot\b" 3; - "~*\bJetCar\b" 3; - "~*\bJetty\b" 3; - "~*\bJikeSpider\b" 3; - "~*\bJOC\ Web\ Spider\b" 3; - "~*\bJoomla\b" 3; - "~*\bJorgee\b" 3; - "~*\bJustView\b" 3; - "~*\bJyxobot\b" 3; - "~*\bKenjin\ Spider\b" 3; - "~*\bKeyword\ Density\b" 3; - "~*\bKozmosbot\b" 3; - "~*\bLanshanbot\b" 3; - "~*\bLarbin\b" 3; - "~*\bLeechFTP\b" 3; - "~*\bLeechGet\b" 3; - "~*\bLexiBot\b" 3; - "~*\bLftp\b" 3; - "~*\bLibWeb\b" 3; - "~*\bLibwhisker\b" 3; - "~*\bLightspeedsystems\b" 3; - "~*\bLikse\b" 3; - "~*\bLinkdexbot\b" 3; - "~*\bLinkextractorPro\b" 3; - "~*\bLinkpadBot\b" 3; - "~*\bLinkScan\b" 3; - "~*\bLinksManager\b" 3; - "~*\bLinkWalker\b" 3; - "~*\bLinqiaMetadataDownloaderBot\b" 3; - "~*\bLinqiaRSSBot\b" 3; - "~*\bLinqiaScrapeBot\b" 3; - "~*\bLipperhey\b" 3; - "~*\bLipperhey\ Spider\b" 3; - "~*\bLitemage_walker\b" 3; - "~*\bLmspider\b" 3; - "~*\bLNSpiderguy\b" 3; - "~*\bLtx71\b" 3; - "~*\blwp-request\b" 3; - "~*\bLWP::Simple\b" 3; - "~*\blwp-trivial\b" 3; - "~*\bMagnet\b" 3; - "~*\bMag-Net\b" 3; - "~*\bmagpie-crawler\b" 3; - "~*\bMail.RU_Bot\b" 3; - "~*\bMajestic12\b" 3; - "~*\bMajestic-SEO\b" 3; - "~*\bMajestic\ SEO\b" 3; - "~*\bMarkMonitor\b" 3; - "~*\bMarkWatch\b" 3; - "~*\bMasscan\b" 3; - "~*\bMass\ Downloader\b" 3; - "~*\bMata\ Hari\b" 3; - "~*\bMauiBot\b" 3; - "~*\bmeanpathbot\b" 3; - "~*\bMeanpathbot\b" 3; - "~*\bMeanPath\ Bot\b" 3; - "~*\bMediatoolkitbot\b" 3; - "~*\bmediawords\b" 3; - "~*\bMegaIndex.ru\b" 3; - "~*\bMetauri\b" 3; - "~*\bMFC_Tear_Sample\b" 3; - "~*\bMicrosoft\ Data\ Access\b" 3; - "~*\bMicrosoft\ URL\ Control\b" 3; - "~*\bMIDown\ tool\b" 3; - "~*\bMIIxpc\b" 3; - "~*\bMister\ PiX\b" 3; - "~*\bMJ12bot\b" 3; - "~*\bMojeek\b" 3; - "~*\bMorfeus\ Fucking\ Scanner\b" 3; - "~*\bMr.4x3\b" 3; - "~*\bMSFrontPage\b" 3; - "~*\bMSIECrawler\b" 3; - "~*\bMsrabot\b" 3; - "~*\bMS\ Web\ Services\ Client\ Protocol\b" 3; - "~*\bmuhstik-scan\b" 3; - "~*\bMusobot\b" 3; - "~*\bName\ Intelligence\b" 3; - "~*\bNameprotect\b" 3; - "~*\bNavroad\b" 3; - "~*\bNearSite\b" 3; - "~*\bNeedle\b" 3; - "~*\bNessus\b" 3; - "~*\bNetAnts\b" 3; - "~*\bNetcraft\b" 3; - "~*\bnetEstate\ NE\ Crawler\b" 3; - "~*\bNetLyzer\b" 3; - "~*\bNetMechanic\b" 3; - "~*\bNetSpider\b" 3; - "~*\bNettrack\b" 3; - "~*\bNet\ Vampire\b" 3; - "~*\bNetvibes\b" 3; - "~*\bNetZIP\b" 3; - "~*\bNextGenSearchBot\b" 3; - "~*\bNibbler\b" 3; - "~*\bNICErsPRO\b" 3; - "~*\bNiki-bot\b" 3; - "~*\bNikto\b" 3; - "~*\bNimbleCrawler\b" 3; - "~*\bNimbostratus\b" 3; - "~*\bNinja\b" 3; - "~*\bNmap\b" 3; - "~*\bNPbot\b" 3; - "~*\bNutch\b" 3; - "~*\boBot\b" 3; - "~*\bOctopus\b" 3; - "~*\bOffline\ Explorer\b" 3; - "~*\bOffline\ Navigator\b" 3; - "~*\bOnCrawl\b" 3; - "~*\bOpenfind\b" 3; - "~*\bOpenLinkProfiler\b" 3; - "~*\bOpenvas\b" 3; - "~*\bOpenVAS\b" 3; - "~*\bOrangeBot\b" 3; - "~*\bOrangeSpider\b" 3; - "~*\bOutclicksBot\b" 3; - "~*\bOutfoxBot\b" 3; - "~*\bPageAnalyzer\b" 3; - "~*\bPage\ Analyzer\b" 3; - "~*\bPageGrabber\b" 3; - "~*\bpage\ scorer\b" 3; - "~*\bPageScorer\b" 3; - "~*\bPandalytics\b" 3; - "~*\bPanscient\b" 3; - "~*\bPapa\ Foto\b" 3; - "~*\bPavuk\b" 3; - "~*\bpcBrowser\b" 3; - "~*\bPECL::HTTP\b" 3; - "~*\bPeoplePal\b" 3; - "~*\bPHPCrawl\b" 3; - "~*\bPicscout\b" 3; - "~*\bPicsearch\b" 3; - "~*\bPictureFinder\b" 3; - "~*\bPimonster\b" 3; - "~*\bPi-Monster\b" 3; - "~*\bPixray\b" 3; - "~*\bPleaseCrawl\b" 3; - "~*\bplumanalytics\b" 3; - "~*\bPockey\b" 3; - "~*\bPOE-Component-Client-HTTP\b" 3; - "~*\bProbethenet\b" 3; - "~*\bProPowerBot\b" 3; - "~*\bProWebWalker\b" 3; - "~*\bPsbot\b" 3; - "~*\bPump\b" 3; - "~*\bPxBroker\b" 3; - "~*\bPyCurl\b" 3; - "~*\bQueryN\ Metasearch\b" 3; - "~*\bQuick-Crawler\b" 3; - "~*\bRankActive\b" 3; - "~*\bRankActiveLinkBot\b" 3; - "~*\bRankFlex\b" 3; - "~*\bRankingBot\b" 3; - "~*\bRankingBot2\b" 3; - "~*\bRankivabot\b" 3; - "~*\bRankurBot\b" 3; - "~*\bRealDownload\b" 3; - "~*\bReaper\b" 3; - "~*\bRebelMouse\b" 3; - "~*\bRecorder\b" 3; - "~*\bRedesScrapy\b" 3; - "~*\bReGet\b" 3; - "~*\bRepoMonkey\b" 3; - "~*\bRipper\b" 3; - "~*\bRocketCrawler\b" 3; - "~*\bRogerbot\b" 3; - "~*\bs1z.ru\b" 3; - "~*\bSalesIntelligent\b" 3; - "~*\bSBIder\b" 3; - "~*\bScanAlert\b" 3; - "~*\bScanbot\b" 3; - "~*\bscan.lol\b" 3; - "~*\bScoutJet\b" 3; - "~*\bScrapy\b" 3; - "~*\bScreaming\b" 3; - "~*\bScreenerBot\b" 3; - "~*\bSearchestate\b" 3; - "~*\bSearchmetricsBot\b" 3; - "~*\bSemrush\b" 3; - "~*\bSemrushBot\b" 3; - "~*\bSEOkicks\b" 3; - "~*\bSEOkicks-Robot\b" 3; - "~*\bSEOlyticsCrawler\b" 3; - "~*\bSeomoz\b" 3; - "~*\bSEOprofiler\b" 3; - "~*\bseoscanners\b" 3; - "~*\bSeoSiteCheckup\b" 3; - "~*\bSEOstats\b" 3; - "~*\bserpstatbot\b" 3; - "~*\bsexsearcher\b" 3; - "~*\bShodan\b" 3; - "~*\bSiphon\b" 3; - "~*\bSISTRIX\b" 3; - "~*\bSitebeam\b" 3; - "~*\bSiteExplorer\b" 3; - "~*\bSiteimprove\b" 3; - "~*\bSiteLockSpider\b" 3; - "~*\bSiteSnagger\b" 3; - "~*\bSiteSucker\b" 3; - "~*\bSite\ Sucker\b" 3; - "~*\bSitevigil\b" 3; - "~*\bSlySearch\b" 3; - "~*\bSmartDownload\b" 3; - "~*\bSMTBot\b" 3; - "~*\bSnake\b" 3; - "~*\bSnapbot\b" 3; - "~*\bSnoopy\b" 3; - "~*\bSocialRankIOBot\b" 3; - "~*\bSociscraper\b" 3; - "~*\bsogouspider\b" 3; - "~*\bSogou\ web\ spider\b" 3; - "~*\bSosospider\b" 3; - "~*\bSottopop\b" 3; - "~*\bSpaceBison\b" 3; - "~*\bSpammen\b" 3; - "~*\bSpankBot\b" 3; - "~*\bSpanner\b" 3; - "~*\bsp_auditbot\b" 3; - "~*\bSpbot\b" 3; - "~*\bSpinn3r\b" 3; - "~*\bSputnikBot\b" 3; - "~*\bspyfu\b" 3; - "~*\bSqlmap\b" 3; - "~*\bSqlworm\b" 3; - "~*\bSqworm\b" 3; - "~*\bSteeler\b" 3; - "~*\bStripper\b" 3; - "~*\bSucker\b" 3; - "~*\bSucuri\b" 3; - "~*\bSuperBot\b" 3; - "~*\bSuperHTTP\b" 3; - "~*\bSurfbot\b" 3; - "~*\bSurveyBot\b" 3; - "~*\bSuzuran\b" 3; - "~*\bSwiftbot\b" 3; - "~*\bsysscan\b" 3; - "~*\bSzukacz\b" 3; - "~*\bT0PHackTeam\b" 3; - "~*\bT8Abot\b" 3; - "~*\btAkeOut\b" 3; - "~*\bTeleport\b" 3; - "~*\bTeleportPro\b" 3; - "~*\bTelesoft\b" 3; - "~*\bTelesphoreo\b" 3; - "~*\bTelesphorep\b" 3; - "~*\bThe\ Intraformant\b" 3; - "~*\bTheNomad\b" 3; - "~*\bTightTwatBot\b" 3; - "~*\bTitan\b" 3; - "~*\bToata\b" 3; - "~*\bToweyabot\b" 3; - "~*\bTracemyfile\b" 3; - "~*\bTrendiction\b" 3; - "~*\bTrendictionbot\b" 3; - "~*\btrendiction.com\b" 3; - "~*\btrendiction.de\b" 3; - "~*\bTrue_Robot\b" 3; - "~*\bTuringos\b" 3; - "~*\bTurnitin\b" 3; - "~*\bTurnitinBot\b" 3; - "~*\bTwengaBot\b" 3; - "~*\bTwice\b" 3; - "~*\bTyphoeus\b" 3; - "~*\bUnisterBot\b" 3; - "~*\bUpflow\b" 3; - "~*\bURLy.Warning\b" 3; - "~*\bURLy\ Warning\b" 3; - "~*\bVacuum\b" 3; - "~*\bVagabondo\b" 3; - "~*\bVB\ Project\b" 3; - "~*\bVCI\b" 3; - "~*\bVeriCiteCrawler\b" 3; - "~*\bVidibleScraper\b" 3; - "~*\bVirusdie\b" 3; - "~*\bVoidEYE\b" 3; - "~*\bVoil\b" 3; - "~*\bVoltron\b" 3; - "~*\bWallpapers/3.0\b" 3; - "~*\bWallpapersHD\b" 3; - "~*\bWASALive-Bot\b" 3; - "~*\bWBSearchBot\b" 3; - "~*\bWebalta\b" 3; - "~*\bWebAuto\b" 3; - "~*\bWeb\ Auto\b" 3; - "~*\bWebBandit\b" 3; - "~*\bWebCollage\b" 3; - "~*\bWeb\ Collage\b" 3; - "~*\bWebCopier\b" 3; - "~*\bWEBDAV\b" 3; - "~*\bWebEnhancer\b" 3; - "~*\bWeb\ Enhancer\b" 3; - "~*\bWebFetch\b" 3; - "~*\bWeb\ Fetch\b" 3; - "~*\bWebFuck\b" 3; - "~*\bWeb\ Fuck\b" 3; - "~*\bWebGo\ IS\b" 3; - "~*\bWebImageCollector\b" 3; - "~*\bWebLeacher\b" 3; - "~*\bWebmasterWorldForumBot\b" 3; - "~*\bwebmeup-crawler\b" 3; - "~*\bWebPix\b" 3; - "~*\bWeb\ Pix\b" 3; - "~*\bWebReaper\b" 3; - "~*\bWebSauger\b" 3; - "~*\bWeb\ Sauger\b" 3; - "~*\bWebshag\b" 3; - "~*\bWebsiteExtractor\b" 3; - "~*\bWebsiteQuester\b" 3; - "~*\bWebsite\ Quester\b" 3; - "~*\bWebster\b" 3; - "~*\bWebStripper\b" 3; - "~*\bWebSucker\b" 3; - "~*\bWeb\ Sucker\b" 3; - "~*\bWebWhacker\b" 3; - "~*\bWebZIP\b" 3; - "~*\bWeSEE\b" 3; - "~*\bWhack\b" 3; - "~*\bWhacker\b" 3; - "~*\bWhatweb\b" 3; - "~*\bWho.is\ Bot\b" 3; - "~*\bWidow\b" 3; - "~*\bWinHTTrack\b" 3; - "~*\bWiseGuys\ Robot\b" 3; - "~*\bWISENutbot\b" 3; - "~*\bWonderbot\b" 3; - "~*\bWoobot\b" 3; - "~*\bWotbox\b" 3; - "~*\bWprecon\b" 3; - "~*\bWPScan\b" 3; - "~*\bWWW-Collector-E\b" 3; - "~*\bWWW-Mechanize\b" 3; - "~*\bWWW::Mechanize\b" 3; - "~*\bWWWOFFLE\b" 3; - "~*\bx09Mozilla\b" 3; - "~*\bx22Mozilla\b" 3; - "~*\bXaldon_WebSpider\b" 3; - "~*\bXaldon\ WebSpider\b" 3; - "~*\bXenu\b" 3; - "~*\bxpymep1.exe\b" 3; - "~*\bYoudaoBot\b" 3; - "~*\bZade\b" 3; - "~*\bZauba\b" 3; - "~*\bzauba.io\b" 3; - "~*\bZermelo\b" 3; - "~*\bZeus\b" 3; - "~*\bzgrab\b" 3; - "~*\bZitebot\b" 3; - "~*\bZmEu\b" 3; - "~*\bZumBot\b" 3; - "~*\bZyBorg\b" 3; -# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### }