diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 7b00caacd..65f28ba04 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1704 -### Updated: Sat Jun 29 21:12:07 SAST 2019 +### Version: V3.2019.07.1705 +### Updated: Mon Jul 1 08:42:46 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### @@ -147,562 +147,562 @@ map $http_user_agent $bad_bot { # -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b|)" 3; - "~*(?:\b)404checker(?:\b|)" 3; - "~*(?:\b)404enemy(?:\b|)" 3; - "~*(?:\b)80legs(?:\b|)" 3; - "~*(?:\b)Abonti(?:\b|)" 3; - "~*(?:\b)Aboundex(?:\b|)" 3; - "~*(?:\b)Aboundexbot(?:\b|)" 3; - "~*(?:\b)Acunetix(?:\b|)" 3; - "~*(?:\b)ADmantX(?:\b|)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; - "~*(?:\b)AhrefsBot(?:\b|)" 3; - "~*(?:\b)AIBOT(?:\b|)" 3; - "~*(?:\b)AiHitBot(?:\b|)" 3; - "~*(?:\b)Aipbot(?:\b|)" 3; - "~*(?:\b)Alexibot(?:\b|)" 3; - "~*(?:\b)Alligator(?:\b|)" 3; - "~*(?:\b)AllSubmitter(?:\b|)" 3; - "~*(?:\b)AlphaBot(?:\b|)" 3; - "~*(?:\b)Anarchie(?:\b|)" 3; - "~*(?:\b)Apexoo(?:\b|)" 3; - "~*(?:\b)archive.org_bot(?:\b|)" 3; - "~*(?:\b)ASPSeek(?:\b|)" 3; - "~*(?:\b)Asterias(?:\b|)" 3; - "~*(?:\b)Attach(?:\b|)" 3; - "~*(?:\b)autoemailspider(?:\b|)" 3; - "~*(?:\b)BackDoorBot(?:\b|)" 3; - "~*(?:\b)Backlink-Ceck(?:\b|)" 3; - "~*(?:\b)backlink-check(?:\b|)" 3; - "~*(?:\b)BacklinkCrawler(?:\b|)" 3; - "~*(?:\b)BackStreet(?:\b|)" 3; - "~*(?:\b)BackWeb(?:\b|)" 3; - "~*(?:\b)Badass(?:\b|)" 3; - "~*(?:\b)Bandit(?:\b|)" 3; - "~*(?:\b)Barkrowler(?:\b|)" 3; - "~*(?:\b)BatchFTP(?:\b|)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; - "~*(?:\b)BBBike(?:\b|)" 3; - "~*(?:\b)BDCbot(?:\b|)" 3; - "~*(?:\b)BDFetch(?:\b|)" 3; - "~*(?:\b)BetaBot(?:\b|)" 3; - "~*(?:\b)Bigfoot(?:\b|)" 3; - "~*(?:\b)Bitacle(?:\b|)" 3; - "~*(?:\b)Blackboard(?:\b|)" 3; - "~*(?:\b)Black\ Hole(?:\b|)" 3; - "~*(?:\b)BlackWidow(?:\b|)" 3; - "~*(?:\b)BLEXBot(?:\b|)" 3; - "~*(?:\b)Blow(?:\b|)" 3; - "~*(?:\b)BlowFish(?:\b|)" 3; - "~*(?:\b)Boardreader(?:\b|)" 3; - "~*(?:\b)Bolt(?:\b|)" 3; - "~*(?:\b)BotALot(?:\b|)" 3; - "~*(?:\b)Brandprotect(?:\b|)" 3; - "~*(?:\b)Brandwatch(?:\b|)" 3; - "~*(?:\b)Buddy(?:\b|)" 3; - "~*(?:\b)BuiltBotTough(?:\b|)" 3; - "~*(?:\b)BuiltWith(?:\b|)" 3; - "~*(?:\b)Bullseye(?:\b|)" 3; - "~*(?:\b)BunnySlippers(?:\b|)" 3; - "~*(?:\b)BuzzSumo(?:\b|)" 3; - "~*(?:\b)Calculon(?:\b|)" 3; - "~*(?:\b)CATExplorador(?:\b|)" 3; - "~*(?:\b)CazoodleBot(?:\b|)" 3; - "~*(?:\b)CCBot(?:\b|)" 3; - "~*(?:\b)Cegbfeieh(?:\b|)" 3; - "~*(?:\b)CheeseBot(?:\b|)" 3; - "~*(?:\b)CherryPicker(?:\b|)" 3; - "~*(?:\b)CheTeam(?:\b|)" 3; - "~*(?:\b)ChinaClaw(?:\b|)" 3; - "~*(?:\b)Chlooe(?:\b|)" 3; - "~*(?:\b)Claritybot(?:\b|)" 3; - "~*(?:\b)Cliqzbot(?:\b|)" 3; - "~*(?:\b)Cloud\ mapping(?:\b|)" 3; - "~*(?:\b)coccocbot-web(?:\b|)" 3; - "~*(?:\b)Cogentbot(?:\b|)" 3; - "~*(?:\b)cognitiveseo(?:\b|)" 3; - "~*(?:\b)Collector(?:\b|)" 3; - "~*(?:\b)com.plumanalytics(?:\b|)" 3; - "~*(?:\b)Copier(?:\b|)" 3; - "~*(?:\b)CopyRightCheck(?:\b|)" 3; - "~*(?:\b)Copyscape(?:\b|)" 3; - "~*(?:\b)Cosmos(?:\b|)" 3; - "~*(?:\b)Craftbot(?:\b|)" 3; - "~*(?:\b)crawler4j(?:\b|)" 3; - "~*(?:\b)crawler.feedback(?:\b|)" 3; - "~*(?:\b)crawl.sogou.com(?:\b|)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; - "~*(?:\b)Crescent(?:\b|)" 3; - "~*(?:\b)CrunchBot(?:\b|)" 3; - "~*(?:\b)CSHttp(?:\b|)" 3; - "~*(?:\b)Curious(?:\b|)" 3; - "~*(?:\b)Custo(?:\b|)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; - "~*(?:\b)DataCha0s(?:\b|)" 3; - "~*(?:\b)DBLBot(?:\b|)" 3; - "~*(?:\b)demandbase-bot(?:\b|)" 3; - "~*(?:\b)Demon(?:\b|)" 3; - "~*(?:\b)Deusu(?:\b|)" 3; - "~*(?:\b)Devil(?:\b|)" 3; - "~*(?:\b)Digincore(?:\b|)" 3; - "~*(?:\b)DigitalPebble(?:\b|)" 3; - "~*(?:\b)DIIbot(?:\b|)" 3; - "~*(?:\b)Dirbuster(?:\b|)" 3; - "~*(?:\b)Disco(?:\b|)" 3; - "~*(?:\b)Discobot(?:\b|)" 3; - "~*(?:\b)Discoverybot(?:\b|)" 3; - "~*(?:\b)Dispatch(?:\b|)" 3; - "~*(?:\b)DittoSpyder(?:\b|)" 3; - "~*(?:\b)DnyzBot(?:\b|)" 3; - "~*(?:\b)DomainAppender(?:\b|)" 3; - "~*(?:\b)DomainCrawler(?:\b|)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; - "~*(?:\b)DomainStatsBot(?:\b|)" 3; - "~*(?:\b)Dotbot(?:\b|)" 3; - "~*(?:\b)Download\ Wonder(?:\b|)" 3; - "~*(?:\b)Dragonfly(?:\b|)" 3; - "~*(?:\b)Drip(?:\b|)" 3; - "~*(?:\b)DSearch(?:\b|)" 3; - "~*(?:\b)DTS\ Agent(?:\b|)" 3; - "~*(?:\b)EasyDL(?:\b|)" 3; - "~*(?:\b)Ebingbong(?:\b|)" 3; - "~*(?:\b)eCatch(?:\b|)" 3; - "~*(?:\b)ECCP/1.0(?:\b|)" 3; - "~*(?:\b)Ecxi(?:\b|)" 3; - "~*(?:\b)EirGrabber(?:\b|)" 3; - "~*(?:\b)EMail\ Siphon(?:\b|)" 3; - "~*(?:\b)EMail\ Wolf(?:\b|)" 3; - "~*(?:\b)EroCrawler(?:\b|)" 3; - "~*(?:\b)evc-batch(?:\b|)" 3; - "~*(?:\b)Evil(?:\b|)" 3; - "~*(?:\b)Exabot(?:\b|)" 3; - "~*(?:\b)Express\ WebPictures(?:\b|)" 3; - "~*(?:\b)ExtLinksBot(?:\b|)" 3; - "~*(?:\b)Extractor(?:\b|)" 3; - "~*(?:\b)ExtractorPro(?:\b|)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; - "~*(?:\b)EyeNetIE(?:\b|)" 3; - "~*(?:\b)Ezooms(?:\b|)" 3; - "~*(?:\b)facebookscraper(?:\b|)" 3; - "~*(?:\b)FDM(?:\b|)" 3; - "~*(?:\b)FemtosearchBot(?:\b|)" 3; - "~*(?:\b)FHscan(?:\b|)" 3; - "~*(?:\b)Fimap(?:\b|)" 3; - "~*(?:\b)Firefox/7.0(?:\b|)" 3; - "~*(?:\b)FlashGet(?:\b|)" 3; - "~*(?:\b)Flunky(?:\b|)" 3; - "~*(?:\b)Foobot(?:\b|)" 3; - "~*(?:\b)Freeuploader(?:\b|)" 3; - "~*(?:\b)FrontPage(?:\b|)" 3; - "~*(?:\b)FyberSpider(?:\b|)" 3; - "~*(?:\b)Fyrebot(?:\b|)" 3; - "~*(?:\b)GalaxyBot(?:\b|)" 3; - "~*(?:\b)Genieo(?:\b|)" 3; - "~*(?:\b)GermCrawler(?:\b|)" 3; - "~*(?:\b)Getintent(?:\b|)" 3; - "~*(?:\b)GetRight(?:\b|)" 3; - "~*(?:\b)GetWeb(?:\b|)" 3; - "~*(?:\b)Gigablast(?:\b|)" 3; - "~*(?:\b)Gigabot(?:\b|)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; - "~*(?:\b)Gotit(?:\b|)" 3; - "~*(?:\b)GoZilla(?:\b|)" 3; - "~*(?:\b)Go!Zilla(?:\b|)" 3; - "~*(?:\b)Grabber(?:\b|)" 3; - "~*(?:\b)GrabNet(?:\b|)" 3; - "~*(?:\b)Grafula(?:\b|)" 3; - "~*(?:\b)GrapeFX(?:\b|)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; - "~*(?:\b)GridBot(?:\b|)" 3; - "~*(?:\b)GT::WWW(?:\b|)" 3; - "~*(?:\b)Haansoft(?:\b|)" 3; - "~*(?:\b)HaosouSpider(?:\b|)" 3; - "~*(?:\b)Harvest(?:\b|)" 3; - "~*(?:\b)Havij(?:\b|)" 3; - "~*(?:\b)HEADMasterSEO(?:\b|)" 3; - "~*(?:\b)Heritrix(?:\b|)" 3; - "~*(?:\b)Hloader(?:\b|)" 3; - "~*(?:\b)HMView(?:\b|)" 3; - "~*(?:\b)HTMLparser(?:\b|)" 3; - "~*(?:\b)HTTP::Lite(?:\b|)" 3; - "~*(?:\b)HTTrack(?:\b|)" 3; - "~*(?:\b)Humanlinks(?:\b|)" 3; - "~*(?:\b)HybridBot(?:\b|)" 3; - "~*(?:\b)Iblog(?:\b|)" 3; - "~*(?:\b)IDBot(?:\b|)" 3; - "~*(?:\b)Id-search(?:\b|)" 3; - "~*(?:\b)IlseBot(?:\b|)" 3; - "~*(?:\b)Image\ Fetch(?:\b|)" 3; - "~*(?:\b)Image\ Sucker(?:\b|)" 3; - "~*(?:\b)IndeedBot(?:\b|)" 3; - "~*(?:\b)Indy\ Library(?:\b|)" 3; - "~*(?:\b)InfoNaviRobot(?:\b|)" 3; - "~*(?:\b)InfoTekies(?:\b|)" 3; - "~*(?:\b)instabid(?:\b|)" 3; - "~*(?:\b)Intelliseek(?:\b|)" 3; - "~*(?:\b)InterGET(?:\b|)" 3; - "~*(?:\b)Internet\ Ninja(?:\b|)" 3; - "~*(?:\b)InternetSeer(?:\b|)" 3; - "~*(?:\b)internetVista\ monitor(?:\b|)" 3; - "~*(?:\b)ips-agent(?:\b|)" 3; - "~*(?:\b)Iria(?:\b|)" 3; - "~*(?:\b)IRLbot(?:\b|)" 3; - "~*(?:\b)Iskanie(?:\b|)" 3; - "~*(?:\b)IstellaBot(?:\b|)" 3; - "~*(?:\b)JamesBOT(?:\b|)" 3; - "~*(?:\b)Jbrofuzz(?:\b|)" 3; - "~*(?:\b)JennyBot(?:\b|)" 3; - "~*(?:\b)JetCar(?:\b|)" 3; - "~*(?:\b)Jetty(?:\b|)" 3; - "~*(?:\b)JikeSpider(?:\b|)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; - "~*(?:\b)Joomla(?:\b|)" 3; - "~*(?:\b)Jorgee(?:\b|)" 3; - "~*(?:\b)JustView(?:\b|)" 3; - "~*(?:\b)Jyxobot(?:\b|)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; - "~*(?:\b)Keyword\ Density(?:\b|)" 3; - "~*(?:\b)Kozmosbot(?:\b|)" 3; - "~*(?:\b)Lanshanbot(?:\b|)" 3; - "~*(?:\b)Larbin(?:\b|)" 3; - "~*(?:\b)LeechFTP(?:\b|)" 3; - "~*(?:\b)LeechGet(?:\b|)" 3; - "~*(?:\b)LexiBot(?:\b|)" 3; - "~*(?:\b)Lftp(?:\b|)" 3; - "~*(?:\b)LibWeb(?:\b|)" 3; - "~*(?:\b)Libwhisker(?:\b|)" 3; - "~*(?:\b)Lightspeedsystems(?:\b|)" 3; - "~*(?:\b)Likse(?:\b|)" 3; - "~*(?:\b)Linkdexbot(?:\b|)" 3; - "~*(?:\b)LinkextractorPro(?:\b|)" 3; - "~*(?:\b)LinkpadBot(?:\b|)" 3; - "~*(?:\b)LinkScan(?:\b|)" 3; - "~*(?:\b)LinksManager(?:\b|)" 3; - "~*(?:\b)LinkWalker(?:\b|)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; - "~*(?:\b)Lipperhey(?:\b|)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; - "~*(?:\b)Litemage_walker(?:\b|)" 3; - "~*(?:\b)Lmspider(?:\b|)" 3; - "~*(?:\b)LNSpiderguy(?:\b|)" 3; - "~*(?:\b)Ltx71(?:\b|)" 3; - "~*(?:\b)lwp-request(?:\b|)" 3; - "~*(?:\b)LWP::Simple(?:\b|)" 3; - "~*(?:\b)lwp-trivial(?:\b|)" 3; - "~*(?:\b)Magnet(?:\b|)" 3; - "~*(?:\b)Mag-Net(?:\b|)" 3; - "~*(?:\b)magpie-crawler(?:\b|)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; - "~*(?:\b)Majestic12(?:\b|)" 3; - "~*(?:\b)Majestic-SEO(?:\b|)" 3; - "~*(?:\b)Majestic\ SEO(?:\b|)" 3; - "~*(?:\b)MarkMonitor(?:\b|)" 3; - "~*(?:\b)MarkWatch(?:\b|)" 3; - "~*(?:\b)Masscan(?:\b|)" 3; - "~*(?:\b)Mass\ Downloader(?:\b|)" 3; - "~*(?:\b)Mata\ Hari(?:\b|)" 3; - "~*(?:\b)MauiBot(?:\b|)" 3; - "~*(?:\b)meanpathbot(?:\b|)" 3; - "~*(?:\b)Meanpathbot(?:\b|)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; - "~*(?:\b)mediawords(?:\b|)" 3; - "~*(?:\b)MegaIndex.ru(?:\b|)" 3; - "~*(?:\b)Metauri(?:\b|)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; - "~*(?:\b)MIDown\ tool(?:\b|)" 3; - "~*(?:\b)MIIxpc(?:\b|)" 3; - "~*(?:\b)Mister\ PiX(?:\b|)" 3; - "~*(?:\b)MJ12bot(?:\b|)" 3; - "~*(?:\b)Mojeek(?:\b|)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; - "~*(?:\b)Mr.4x3(?:\b|)" 3; - "~*(?:\b)MSFrontPage(?:\b|)" 3; - "~*(?:\b)MSIECrawler(?:\b|)" 3; - "~*(?:\b)Msrabot(?:\b|)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; - "~*(?:\b)muhstik-scan(?:\b|)" 3; - "~*(?:\b)Musobot(?:\b|)" 3; - "~*(?:\b)Name\ Intelligence(?:\b|)" 3; - "~*(?:\b)Nameprotect(?:\b|)" 3; - "~*(?:\b)Navroad(?:\b|)" 3; - "~*(?:\b)NearSite(?:\b|)" 3; - "~*(?:\b)Needle(?:\b|)" 3; - "~*(?:\b)Nessus(?:\b|)" 3; - "~*(?:\b)NetAnts(?:\b|)" 3; - "~*(?:\b)Netcraft(?:\b|)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; - "~*(?:\b)NetLyzer(?:\b|)" 3; - "~*(?:\b)NetMechanic(?:\b|)" 3; - "~*(?:\b)NetSpider(?:\b|)" 3; - "~*(?:\b)Nettrack(?:\b|)" 3; - "~*(?:\b)Net\ Vampire(?:\b|)" 3; - "~*(?:\b)Netvibes(?:\b|)" 3; - "~*(?:\b)NetZIP(?:\b|)" 3; - "~*(?:\b)NextGenSearchBot(?:\b|)" 3; - "~*(?:\b)Nibbler(?:\b|)" 3; - "~*(?:\b)NICErsPRO(?:\b|)" 3; - "~*(?:\b)Niki-bot(?:\b|)" 3; - "~*(?:\b)Nikto(?:\b|)" 3; - "~*(?:\b)NimbleCrawler(?:\b|)" 3; - "~*(?:\b)Nimbostratus(?:\b|)" 3; - "~*(?:\b)Ninja(?:\b|)" 3; - "~*(?:\b)Nmap(?:\b|)" 3; - "~*(?:\b)NPbot(?:\b|)" 3; - "~*(?:\b)Nutch(?:\b|)" 3; - "~*(?:\b)oBot(?:\b|)" 3; - "~*(?:\b)Octopus(?:\b|)" 3; - "~*(?:\b)Offline\ Explorer(?:\b|)" 3; - "~*(?:\b)Offline\ Navigator(?:\b|)" 3; - "~*(?:\b)OnCrawl(?:\b|)" 3; - "~*(?:\b)Openfind(?:\b|)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; - "~*(?:\b)Openvas(?:\b|)" 3; - "~*(?:\b)OpenVAS(?:\b|)" 3; - "~*(?:\b)OrangeBot(?:\b|)" 3; - "~*(?:\b)OrangeSpider(?:\b|)" 3; - "~*(?:\b)OutclicksBot(?:\b|)" 3; - "~*(?:\b)OutfoxBot(?:\b|)" 3; - "~*(?:\b)PageAnalyzer(?:\b|)" 3; - "~*(?:\b)Page\ Analyzer(?:\b|)" 3; - "~*(?:\b)PageGrabber(?:\b|)" 3; - "~*(?:\b)page\ scorer(?:\b|)" 3; - "~*(?:\b)PageScorer(?:\b|)" 3; - "~*(?:\b)Pandalytics(?:\b|)" 3; - "~*(?:\b)Panscient(?:\b|)" 3; - "~*(?:\b)Papa\ Foto(?:\b|)" 3; - "~*(?:\b)Pavuk(?:\b|)" 3; - "~*(?:\b)pcBrowser(?:\b|)" 3; - "~*(?:\b)PECL::HTTP(?:\b|)" 3; - "~*(?:\b)PeoplePal(?:\b|)" 3; - "~*(?:\b)PHPCrawl(?:\b|)" 3; - "~*(?:\b)Picscout(?:\b|)" 3; - "~*(?:\b)Picsearch(?:\b|)" 3; - "~*(?:\b)PictureFinder(?:\b|)" 3; - "~*(?:\b)Pimonster(?:\b|)" 3; - "~*(?:\b)Pi-Monster(?:\b|)" 3; - "~*(?:\b)Pixray(?:\b|)" 3; - "~*(?:\b)PleaseCrawl(?:\b|)" 3; - "~*(?:\b)plumanalytics(?:\b|)" 3; - "~*(?:\b)Pockey(?:\b|)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; - "~*(?:\b)Probethenet(?:\b|)" 3; - "~*(?:\b)ProPowerBot(?:\b|)" 3; - "~*(?:\b)ProWebWalker(?:\b|)" 3; - "~*(?:\b)Psbot(?:\b|)" 3; - "~*(?:\b)Pump(?:\b|)" 3; - "~*(?:\b)PxBroker(?:\b|)" 3; - "~*(?:\b)PyCurl(?:\b|)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; - "~*(?:\b)Quick-Crawler(?:\b|)" 3; - "~*(?:\b)RankActive(?:\b|)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; - "~*(?:\b)RankFlex(?:\b|)" 3; - "~*(?:\b)RankingBot(?:\b|)" 3; - "~*(?:\b)RankingBot2(?:\b|)" 3; - "~*(?:\b)Rankivabot(?:\b|)" 3; - "~*(?:\b)RankurBot(?:\b|)" 3; - "~*(?:\b)RealDownload(?:\b|)" 3; - "~*(?:\b)Reaper(?:\b|)" 3; - "~*(?:\b)RebelMouse(?:\b|)" 3; - "~*(?:\b)Recorder(?:\b|)" 3; - "~*(?:\b)RedesScrapy(?:\b|)" 3; - "~*(?:\b)ReGet(?:\b|)" 3; - "~*(?:\b)RepoMonkey(?:\b|)" 3; - "~*(?:\b)Ripper(?:\b|)" 3; - "~*(?:\b)RocketCrawler(?:\b|)" 3; - "~*(?:\b)Rogerbot(?:\b|)" 3; - "~*(?:\b)s1z.ru(?:\b|)" 3; - "~*(?:\b)SalesIntelligent(?:\b|)" 3; - "~*(?:\b)SBIder(?:\b|)" 3; - "~*(?:\b)ScanAlert(?:\b|)" 3; - "~*(?:\b)Scanbot(?:\b|)" 3; - "~*(?:\b)scan.lol(?:\b|)" 3; - "~*(?:\b)ScoutJet(?:\b|)" 3; - "~*(?:\b)Scrapy(?:\b|)" 3; - "~*(?:\b)Screaming(?:\b|)" 3; - "~*(?:\b)ScreenerBot(?:\b|)" 3; - "~*(?:\b)Searchestate(?:\b|)" 3; - "~*(?:\b)SearchmetricsBot(?:\b|)" 3; - "~*(?:\b)Semrush(?:\b|)" 3; - "~*(?:\b)SemrushBot(?:\b|)" 3; - "~*(?:\b)SEOkicks(?:\b|)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; - "~*(?:\b)Seomoz(?:\b|)" 3; - "~*(?:\b)SEOprofiler(?:\b|)" 3; - "~*(?:\b)seoscanners(?:\b|)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; - "~*(?:\b)SEOstats(?:\b|)" 3; - "~*(?:\b)serpstatbot(?:\b|)" 3; - "~*(?:\b)sexsearcher(?:\b|)" 3; - "~*(?:\b)Shodan(?:\b|)" 3; - "~*(?:\b)Siphon(?:\b|)" 3; - "~*(?:\b)SISTRIX(?:\b|)" 3; - "~*(?:\b)Sitebeam(?:\b|)" 3; - "~*(?:\b)SiteExplorer(?:\b|)" 3; - "~*(?:\b)Siteimprove(?:\b|)" 3; - "~*(?:\b)SiteLockSpider(?:\b|)" 3; - "~*(?:\b)SiteSnagger(?:\b|)" 3; - "~*(?:\b)SiteSucker(?:\b|)" 3; - "~*(?:\b)Site\ Sucker(?:\b|)" 3; - "~*(?:\b)Sitevigil(?:\b|)" 3; - "~*(?:\b)SlySearch(?:\b|)" 3; - "~*(?:\b)SmartDownload(?:\b|)" 3; - "~*(?:\b)SMTBot(?:\b|)" 3; - "~*(?:\b)Snake(?:\b|)" 3; - "~*(?:\b)Snapbot(?:\b|)" 3; - "~*(?:\b)Snoopy(?:\b|)" 3; - "~*(?:\b)SocialRankIOBot(?:\b|)" 3; - "~*(?:\b)Sociscraper(?:\b|)" 3; - "~*(?:\b)sogouspider(?:\b|)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; - "~*(?:\b)Sosospider(?:\b|)" 3; - "~*(?:\b)Sottopop(?:\b|)" 3; - "~*(?:\b)SpaceBison(?:\b|)" 3; - "~*(?:\b)Spammen(?:\b|)" 3; - "~*(?:\b)SpankBot(?:\b|)" 3; - "~*(?:\b)Spanner(?:\b|)" 3; - "~*(?:\b)sp_auditbot(?:\b|)" 3; - "~*(?:\b)Spbot(?:\b|)" 3; - "~*(?:\b)Spinn3r(?:\b|)" 3; - "~*(?:\b)SputnikBot(?:\b|)" 3; - "~*(?:\b)spyfu(?:\b|)" 3; - "~*(?:\b)Sqlmap(?:\b|)" 3; - "~*(?:\b)Sqlworm(?:\b|)" 3; - "~*(?:\b)Sqworm(?:\b|)" 3; - "~*(?:\b)Steeler(?:\b|)" 3; - "~*(?:\b)Stripper(?:\b|)" 3; - "~*(?:\b)Sucker(?:\b|)" 3; - "~*(?:\b)Sucuri(?:\b|)" 3; - "~*(?:\b)SuperBot(?:\b|)" 3; - "~*(?:\b)SuperHTTP(?:\b|)" 3; - "~*(?:\b)Surfbot(?:\b|)" 3; - "~*(?:\b)SurveyBot(?:\b|)" 3; - "~*(?:\b)Suzuran(?:\b|)" 3; - "~*(?:\b)Swiftbot(?:\b|)" 3; - "~*(?:\b)sysscan(?:\b|)" 3; - "~*(?:\b)Szukacz(?:\b|)" 3; - "~*(?:\b)T0PHackTeam(?:\b|)" 3; - "~*(?:\b)T8Abot(?:\b|)" 3; - "~*(?:\b)tAkeOut(?:\b|)" 3; - "~*(?:\b)Teleport(?:\b|)" 3; - "~*(?:\b)TeleportPro(?:\b|)" 3; - "~*(?:\b)Telesoft(?:\b|)" 3; - "~*(?:\b)Telesphoreo(?:\b|)" 3; - "~*(?:\b)Telesphorep(?:\b|)" 3; - "~*(?:\b)The\ Intraformant(?:\b|)" 3; - "~*(?:\b)TheNomad(?:\b|)" 3; - "~*(?:\b)Thumbor(?:\b|)" 3; - "~*(?:\b)TightTwatBot(?:\b|)" 3; - "~*(?:\b)Titan(?:\b|)" 3; - "~*(?:\b)Toata(?:\b|)" 3; - "~*(?:\b)Toweyabot(?:\b|)" 3; - "~*(?:\b)Tracemyfile(?:\b|)" 3; - "~*(?:\b)Trendiction(?:\b|)" 3; - "~*(?:\b)Trendictionbot(?:\b|)" 3; - "~*(?:\b)trendiction.com(?:\b|)" 3; - "~*(?:\b)trendiction.de(?:\b|)" 3; - "~*(?:\b)True_Robot(?:\b|)" 3; - "~*(?:\b)Turingos(?:\b|)" 3; - "~*(?:\b)Turnitin(?:\b|)" 3; - "~*(?:\b)TurnitinBot(?:\b|)" 3; - "~*(?:\b)TwengaBot(?:\b|)" 3; - "~*(?:\b)Twice(?:\b|)" 3; - "~*(?:\b)Typhoeus(?:\b|)" 3; - "~*(?:\b)UnisterBot(?:\b|)" 3; - "~*(?:\b)Upflow(?:\b|)" 3; - "~*(?:\b)URLy.Warning(?:\b|)" 3; - "~*(?:\b)URLy\ Warning(?:\b|)" 3; - "~*(?:\b)Vacuum(?:\b|)" 3; - "~*(?:\b)Vagabondo(?:\b|)" 3; - "~*(?:\b)VB\ Project(?:\b|)" 3; - "~*(?:\b)VCI(?:\b|)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; - "~*(?:\b)VidibleScraper(?:\b|)" 3; - "~*(?:\b)Virusdie(?:\b|)" 3; - "~*(?:\b)VoidEYE(?:\b|)" 3; - "~*(?:\b)Voil(?:\b|)" 3; - "~*(?:\b)Voltron(?:\b|)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; - "~*(?:\b)WallpapersHD(?:\b|)" 3; - "~*(?:\b)WASALive-Bot(?:\b|)" 3; - "~*(?:\b)WBSearchBot(?:\b|)" 3; - "~*(?:\b)Webalta(?:\b|)" 3; - "~*(?:\b)WebAuto(?:\b|)" 3; - "~*(?:\b)Web\ Auto(?:\b|)" 3; - "~*(?:\b)WebBandit(?:\b|)" 3; - "~*(?:\b)WebCollage(?:\b|)" 3; - "~*(?:\b)Web\ Collage(?:\b|)" 3; - "~*(?:\b)WebCopier(?:\b|)" 3; - "~*(?:\b)WEBDAV(?:\b|)" 3; - "~*(?:\b)WebEnhancer(?:\b|)" 3; - "~*(?:\b)Web\ Enhancer(?:\b|)" 3; - "~*(?:\b)WebFetch(?:\b|)" 3; - "~*(?:\b)Web\ Fetch(?:\b|)" 3; - "~*(?:\b)WebFuck(?:\b|)" 3; - "~*(?:\b)Web\ Fuck(?:\b|)" 3; - "~*(?:\b)WebGo\ IS(?:\b|)" 3; - "~*(?:\b)WebImageCollector(?:\b|)" 3; - "~*(?:\b)WebLeacher(?:\b|)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; - "~*(?:\b)webmeup-crawler(?:\b|)" 3; - "~*(?:\b)WebPix(?:\b|)" 3; - "~*(?:\b)Web\ Pix(?:\b|)" 3; - "~*(?:\b)WebReaper(?:\b|)" 3; - "~*(?:\b)WebSauger(?:\b|)" 3; - "~*(?:\b)Web\ Sauger(?:\b|)" 3; - "~*(?:\b)Webshag(?:\b|)" 3; - "~*(?:\b)WebsiteExtractor(?:\b|)" 3; - "~*(?:\b)WebsiteQuester(?:\b|)" 3; - "~*(?:\b)Website\ Quester(?:\b|)" 3; - "~*(?:\b)Webster(?:\b|)" 3; - "~*(?:\b)WebStripper(?:\b|)" 3; - "~*(?:\b)WebSucker(?:\b|)" 3; - "~*(?:\b)Web\ Sucker(?:\b|)" 3; - "~*(?:\b)WebWhacker(?:\b|)" 3; - "~*(?:\b)WebZIP(?:\b|)" 3; - "~*(?:\b)WeSEE(?:\b|)" 3; - "~*(?:\b)Whack(?:\b|)" 3; - "~*(?:\b)Whacker(?:\b|)" 3; - "~*(?:\b)Whatweb(?:\b|)" 3; - "~*(?:\b)Who.is\ Bot(?:\b|)" 3; - "~*(?:\b)Widow(?:\b|)" 3; - "~*(?:\b)WinHTTrack(?:\b|)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; - "~*(?:\b)WISENutbot(?:\b|)" 3; - "~*(?:\b)Wonderbot(?:\b|)" 3; - "~*(?:\b)Woobot(?:\b|)" 3; - "~*(?:\b)Wotbox(?:\b|)" 3; - "~*(?:\b)Wprecon(?:\b|)" 3; - "~*(?:\b)WPScan(?:\b|)" 3; - "~*(?:\b)WWW-Collector-E(?:\b|)" 3; - "~*(?:\b)WWW-Mechanize(?:\b|)" 3; - "~*(?:\b)WWW::Mechanize(?:\b|)" 3; - "~*(?:\b)WWWOFFLE(?:\b|)" 3; - "~*(?:\b)x09Mozilla(?:\b|)" 3; - "~*(?:\b)x22Mozilla(?:\b|)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; - "~*(?:\b)Xenu(?:\b|)" 3; - "~*(?:\b)xpymep1.exe(?:\b|)" 3; - "~*(?:\b)YoudaoBot(?:\b|)" 3; - "~*(?:\b)Zade(?:\b|)" 3; - "~*(?:\b)Zauba(?:\b|)" 3; - "~*(?:\b)zauba.io(?:\b|)" 3; - "~*(?:\b)Zermelo(?:\b|)" 3; - "~*(?:\b)Zeus(?:\b|)" 3; - "~*(?:\b)zgrab(?:\b|)" 3; - "~*(?:\b)Zitebot(?:\b|)" 3; - "~*(?:\b)ZmEu(?:\b|)" 3; - "~*(?:\b)ZumBot(?:\b|)" 3; - "~*(?:\b)ZyBorg(?:\b|)" 3; + "~*(?:\b)360Spider(?:\b)" 3; + "~*(?:\b)404checker(?:\b)" 3; + "~*(?:\b)404enemy(?:\b)" 3; + "~*(?:\b)80legs(?:\b)" 3; + "~*(?:\b)Abonti(?:\b)" 3; + "~*(?:\b)Aboundex(?:\b)" 3; + "~*(?:\b)Aboundexbot(?:\b)" 3; + "~*(?:\b)Acunetix(?:\b)" 3; + "~*(?:\b)ADmantX(?:\b)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; + "~*(?:\b)AhrefsBot(?:\b)" 3; + "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)AiHitBot(?:\b)" 3; + "~*(?:\b)Aipbot(?:\b)" 3; + "~*(?:\b)Alexibot(?:\b)" 3; + "~*(?:\b)Alligator(?:\b)" 3; + "~*(?:\b)AllSubmitter(?:\b)" 3; + "~*(?:\b)AlphaBot(?:\b)" 3; + "~*(?:\b)Anarchie(?:\b)" 3; + "~*(?:\b)Apexoo(?:\b)" 3; + "~*(?:\b)archive.org_bot(?:\b)" 3; + "~*(?:\b)ASPSeek(?:\b)" 3; + "~*(?:\b)Asterias(?:\b)" 3; + "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)BackDoorBot(?:\b)" 3; + "~*(?:\b)Backlink-Ceck(?:\b)" 3; + "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)BacklinkCrawler(?:\b)" 3; + "~*(?:\b)BackStreet(?:\b)" 3; + "~*(?:\b)BackWeb(?:\b)" 3; + "~*(?:\b)Badass(?:\b)" 3; + "~*(?:\b)Bandit(?:\b)" 3; + "~*(?:\b)Barkrowler(?:\b)" 3; + "~*(?:\b)BatchFTP(?:\b)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; + "~*(?:\b)BBBike(?:\b)" 3; + "~*(?:\b)BDCbot(?:\b)" 3; + "~*(?:\b)BDFetch(?:\b)" 3; + "~*(?:\b)BetaBot(?:\b)" 3; + "~*(?:\b)Bigfoot(?:\b)" 3; + "~*(?:\b)Bitacle(?:\b)" 3; + "~*(?:\b)Blackboard(?:\b)" 3; + "~*(?:\b)Black\ Hole(?:\b)" 3; + "~*(?:\b)BlackWidow(?:\b)" 3; + "~*(?:\b)BLEXBot(?:\b)" 3; + "~*(?:\b)Blow(?:\b)" 3; + "~*(?:\b)BlowFish(?:\b)" 3; + "~*(?:\b)Boardreader(?:\b)" 3; + "~*(?:\b)Bolt(?:\b)" 3; + "~*(?:\b)BotALot(?:\b)" 3; + "~*(?:\b)Brandprotect(?:\b)" 3; + "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buddy(?:\b)" 3; + "~*(?:\b)BuiltBotTough(?:\b)" 3; + "~*(?:\b)BuiltWith(?:\b)" 3; + "~*(?:\b)Bullseye(?:\b)" 3; + "~*(?:\b)BunnySlippers(?:\b)" 3; + "~*(?:\b)BuzzSumo(?:\b)" 3; + "~*(?:\b)Calculon(?:\b)" 3; + "~*(?:\b)CATExplorador(?:\b)" 3; + "~*(?:\b)CazoodleBot(?:\b)" 3; + "~*(?:\b)CCBot(?:\b)" 3; + "~*(?:\b)Cegbfeieh(?:\b)" 3; + "~*(?:\b)CheeseBot(?:\b)" 3; + "~*(?:\b)CherryPicker(?:\b)" 3; + "~*(?:\b)CheTeam(?:\b)" 3; + "~*(?:\b)ChinaClaw(?:\b)" 3; + "~*(?:\b)Chlooe(?:\b)" 3; + "~*(?:\b)Claritybot(?:\b)" 3; + "~*(?:\b)Cliqzbot(?:\b)" 3; + "~*(?:\b)Cloud\ mapping(?:\b)" 3; + "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cogentbot(?:\b)" 3; + "~*(?:\b)cognitiveseo(?:\b)" 3; + "~*(?:\b)Collector(?:\b)" 3; + "~*(?:\b)com.plumanalytics(?:\b)" 3; + "~*(?:\b)Copier(?:\b)" 3; + "~*(?:\b)CopyRightCheck(?:\b)" 3; + "~*(?:\b)Copyscape(?:\b)" 3; + "~*(?:\b)Cosmos(?:\b)" 3; + "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)crawler4j(?:\b)" 3; + "~*(?:\b)crawler.feedback(?:\b)" 3; + "~*(?:\b)crawl.sogou.com(?:\b)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b)" 3; + "~*(?:\b)Crescent(?:\b)" 3; + "~*(?:\b)CrunchBot(?:\b)" 3; + "~*(?:\b)CSHttp(?:\b)" 3; + "~*(?:\b)Curious(?:\b)" 3; + "~*(?:\b)Custo(?:\b)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; + "~*(?:\b)DataCha0s(?:\b)" 3; + "~*(?:\b)DBLBot(?:\b)" 3; + "~*(?:\b)demandbase-bot(?:\b)" 3; + "~*(?:\b)Demon(?:\b)" 3; + "~*(?:\b)Deusu(?:\b)" 3; + "~*(?:\b)Devil(?:\b)" 3; + "~*(?:\b)Digincore(?:\b)" 3; + "~*(?:\b)DigitalPebble(?:\b)" 3; + "~*(?:\b)DIIbot(?:\b)" 3; + "~*(?:\b)Dirbuster(?:\b)" 3; + "~*(?:\b)Disco(?:\b)" 3; + "~*(?:\b)Discobot(?:\b)" 3; + "~*(?:\b)Discoverybot(?:\b)" 3; + "~*(?:\b)Dispatch(?:\b)" 3; + "~*(?:\b)DittoSpyder(?:\b)" 3; + "~*(?:\b)DnyzBot(?:\b)" 3; + "~*(?:\b)DomainAppender(?:\b)" 3; + "~*(?:\b)DomainCrawler(?:\b)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)DomainStatsBot(?:\b)" 3; + "~*(?:\b)Dotbot(?:\b)" 3; + "~*(?:\b)Download\ Wonder(?:\b)" 3; + "~*(?:\b)Dragonfly(?:\b)" 3; + "~*(?:\b)Drip(?:\b)" 3; + "~*(?:\b)DSearch(?:\b)" 3; + "~*(?:\b)DTS\ Agent(?:\b)" 3; + "~*(?:\b)EasyDL(?:\b)" 3; + "~*(?:\b)Ebingbong(?:\b)" 3; + "~*(?:\b)eCatch(?:\b)" 3; + "~*(?:\b)ECCP/1.0(?:\b)" 3; + "~*(?:\b)Ecxi(?:\b)" 3; + "~*(?:\b)EirGrabber(?:\b)" 3; + "~*(?:\b)EMail\ Siphon(?:\b)" 3; + "~*(?:\b)EMail\ Wolf(?:\b)" 3; + "~*(?:\b)EroCrawler(?:\b)" 3; + "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)Evil(?:\b)" 3; + "~*(?:\b)Exabot(?:\b)" 3; + "~*(?:\b)Express\ WebPictures(?:\b)" 3; + "~*(?:\b)ExtLinksBot(?:\b)" 3; + "~*(?:\b)Extractor(?:\b)" 3; + "~*(?:\b)ExtractorPro(?:\b)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; + "~*(?:\b)EyeNetIE(?:\b)" 3; + "~*(?:\b)Ezooms(?:\b)" 3; + "~*(?:\b)facebookscraper(?:\b)" 3; + "~*(?:\b)FDM(?:\b)" 3; + "~*(?:\b)FemtosearchBot(?:\b)" 3; + "~*(?:\b)FHscan(?:\b)" 3; + "~*(?:\b)Fimap(?:\b)" 3; + "~*(?:\b)Firefox/7.0(?:\b)" 3; + "~*(?:\b)FlashGet(?:\b)" 3; + "~*(?:\b)Flunky(?:\b)" 3; + "~*(?:\b)Foobot(?:\b)" 3; + "~*(?:\b)Freeuploader(?:\b)" 3; + "~*(?:\b)FrontPage(?:\b)" 3; + "~*(?:\b)FyberSpider(?:\b)" 3; + "~*(?:\b)Fyrebot(?:\b)" 3; + "~*(?:\b)GalaxyBot(?:\b)" 3; + "~*(?:\b)Genieo(?:\b)" 3; + "~*(?:\b)GermCrawler(?:\b)" 3; + "~*(?:\b)Getintent(?:\b)" 3; + "~*(?:\b)GetRight(?:\b)" 3; + "~*(?:\b)GetWeb(?:\b)" 3; + "~*(?:\b)Gigablast(?:\b)" 3; + "~*(?:\b)Gigabot(?:\b)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; + "~*(?:\b)Gotit(?:\b)" 3; + "~*(?:\b)GoZilla(?:\b)" 3; + "~*(?:\b)Go!Zilla(?:\b)" 3; + "~*(?:\b)Grabber(?:\b)" 3; + "~*(?:\b)GrabNet(?:\b)" 3; + "~*(?:\b)Grafula(?:\b)" 3; + "~*(?:\b)GrapeFX(?:\b)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b)" 3; + "~*(?:\b)GridBot(?:\b)" 3; + "~*(?:\b)GT::WWW(?:\b)" 3; + "~*(?:\b)Haansoft(?:\b)" 3; + "~*(?:\b)HaosouSpider(?:\b)" 3; + "~*(?:\b)Harvest(?:\b)" 3; + "~*(?:\b)Havij(?:\b)" 3; + "~*(?:\b)HEADMasterSEO(?:\b)" 3; + "~*(?:\b)Heritrix(?:\b)" 3; + "~*(?:\b)Hloader(?:\b)" 3; + "~*(?:\b)HMView(?:\b)" 3; + "~*(?:\b)HTMLparser(?:\b)" 3; + "~*(?:\b)HTTP::Lite(?:\b)" 3; + "~*(?:\b)HTTrack(?:\b)" 3; + "~*(?:\b)Humanlinks(?:\b)" 3; + "~*(?:\b)HybridBot(?:\b)" 3; + "~*(?:\b)Iblog(?:\b)" 3; + "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)Id-search(?:\b)" 3; + "~*(?:\b)IlseBot(?:\b)" 3; + "~*(?:\b)Image\ Fetch(?:\b)" 3; + "~*(?:\b)Image\ Sucker(?:\b)" 3; + "~*(?:\b)IndeedBot(?:\b)" 3; + "~*(?:\b)Indy\ Library(?:\b)" 3; + "~*(?:\b)InfoNaviRobot(?:\b)" 3; + "~*(?:\b)InfoTekies(?:\b)" 3; + "~*(?:\b)instabid(?:\b)" 3; + "~*(?:\b)Intelliseek(?:\b)" 3; + "~*(?:\b)InterGET(?:\b)" 3; + "~*(?:\b)Internet\ Ninja(?:\b)" 3; + "~*(?:\b)InternetSeer(?:\b)" 3; + "~*(?:\b)internetVista\ monitor(?:\b)" 3; + "~*(?:\b)ips-agent(?:\b)" 3; + "~*(?:\b)Iria(?:\b)" 3; + "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)Iskanie(?:\b)" 3; + "~*(?:\b)IstellaBot(?:\b)" 3; + "~*(?:\b)JamesBOT(?:\b)" 3; + "~*(?:\b)Jbrofuzz(?:\b)" 3; + "~*(?:\b)JennyBot(?:\b)" 3; + "~*(?:\b)JetCar(?:\b)" 3; + "~*(?:\b)Jetty(?:\b)" 3; + "~*(?:\b)JikeSpider(?:\b)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; + "~*(?:\b)Joomla(?:\b)" 3; + "~*(?:\b)Jorgee(?:\b)" 3; + "~*(?:\b)JustView(?:\b)" 3; + "~*(?:\b)Jyxobot(?:\b)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b)" 3; + "~*(?:\b)Keyword\ Density(?:\b)" 3; + "~*(?:\b)Kozmosbot(?:\b)" 3; + "~*(?:\b)Lanshanbot(?:\b)" 3; + "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)LeechFTP(?:\b)" 3; + "~*(?:\b)LeechGet(?:\b)" 3; + "~*(?:\b)LexiBot(?:\b)" 3; + "~*(?:\b)Lftp(?:\b)" 3; + "~*(?:\b)LibWeb(?:\b)" 3; + "~*(?:\b)Libwhisker(?:\b)" 3; + "~*(?:\b)Lightspeedsystems(?:\b)" 3; + "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkdexbot(?:\b)" 3; + "~*(?:\b)LinkextractorPro(?:\b)" 3; + "~*(?:\b)LinkpadBot(?:\b)" 3; + "~*(?:\b)LinkScan(?:\b)" 3; + "~*(?:\b)LinksManager(?:\b)" 3; + "~*(?:\b)LinkWalker(?:\b)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; + "~*(?:\b)Lipperhey(?:\b)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; + "~*(?:\b)Litemage_walker(?:\b)" 3; + "~*(?:\b)Lmspider(?:\b)" 3; + "~*(?:\b)LNSpiderguy(?:\b)" 3; + "~*(?:\b)Ltx71(?:\b)" 3; + "~*(?:\b)lwp-request(?:\b)" 3; + "~*(?:\b)LWP::Simple(?:\b)" 3; + "~*(?:\b)lwp-trivial(?:\b)" 3; + "~*(?:\b)Magnet(?:\b)" 3; + "~*(?:\b)Mag-Net(?:\b)" 3; + "~*(?:\b)magpie-crawler(?:\b)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b)" 3; + "~*(?:\b)Majestic12(?:\b)" 3; + "~*(?:\b)Majestic-SEO(?:\b)" 3; + "~*(?:\b)Majestic\ SEO(?:\b)" 3; + "~*(?:\b)MarkMonitor(?:\b)" 3; + "~*(?:\b)MarkWatch(?:\b)" 3; + "~*(?:\b)Masscan(?:\b)" 3; + "~*(?:\b)Mass\ Downloader(?:\b)" 3; + "~*(?:\b)Mata\ Hari(?:\b)" 3; + "~*(?:\b)MauiBot(?:\b)" 3; + "~*(?:\b)meanpathbot(?:\b)" 3; + "~*(?:\b)Meanpathbot(?:\b)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b)" 3; + "~*(?:\b)mediawords(?:\b)" 3; + "~*(?:\b)MegaIndex.ru(?:\b)" 3; + "~*(?:\b)Metauri(?:\b)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; + "~*(?:\b)MIDown\ tool(?:\b)" 3; + "~*(?:\b)MIIxpc(?:\b)" 3; + "~*(?:\b)Mister\ PiX(?:\b)" 3; + "~*(?:\b)MJ12bot(?:\b)" 3; + "~*(?:\b)Mojeek(?:\b)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; + "~*(?:\b)Mr.4x3(?:\b)" 3; + "~*(?:\b)MSFrontPage(?:\b)" 3; + "~*(?:\b)MSIECrawler(?:\b)" 3; + "~*(?:\b)Msrabot(?:\b)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; + "~*(?:\b)muhstik-scan(?:\b)" 3; + "~*(?:\b)Musobot(?:\b)" 3; + "~*(?:\b)Name\ Intelligence(?:\b)" 3; + "~*(?:\b)Nameprotect(?:\b)" 3; + "~*(?:\b)Navroad(?:\b)" 3; + "~*(?:\b)NearSite(?:\b)" 3; + "~*(?:\b)Needle(?:\b)" 3; + "~*(?:\b)Nessus(?:\b)" 3; + "~*(?:\b)NetAnts(?:\b)" 3; + "~*(?:\b)Netcraft(?:\b)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; + "~*(?:\b)NetLyzer(?:\b)" 3; + "~*(?:\b)NetMechanic(?:\b)" 3; + "~*(?:\b)NetSpider(?:\b)" 3; + "~*(?:\b)Nettrack(?:\b)" 3; + "~*(?:\b)Net\ Vampire(?:\b)" 3; + "~*(?:\b)Netvibes(?:\b)" 3; + "~*(?:\b)NetZIP(?:\b)" 3; + "~*(?:\b)NextGenSearchBot(?:\b)" 3; + "~*(?:\b)Nibbler(?:\b)" 3; + "~*(?:\b)NICErsPRO(?:\b)" 3; + "~*(?:\b)Niki-bot(?:\b)" 3; + "~*(?:\b)Nikto(?:\b)" 3; + "~*(?:\b)NimbleCrawler(?:\b)" 3; + "~*(?:\b)Nimbostratus(?:\b)" 3; + "~*(?:\b)Ninja(?:\b)" 3; + "~*(?:\b)Nmap(?:\b)" 3; + "~*(?:\b)NPbot(?:\b)" 3; + "~*(?:\b)Nutch(?:\b)" 3; + "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)Octopus(?:\b)" 3; + "~*(?:\b)Offline\ Explorer(?:\b)" 3; + "~*(?:\b)Offline\ Navigator(?:\b)" 3; + "~*(?:\b)OnCrawl(?:\b)" 3; + "~*(?:\b)Openfind(?:\b)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b)" 3; + "~*(?:\b)Openvas(?:\b)" 3; + "~*(?:\b)OpenVAS(?:\b)" 3; + "~*(?:\b)OrangeBot(?:\b)" 3; + "~*(?:\b)OrangeSpider(?:\b)" 3; + "~*(?:\b)OutclicksBot(?:\b)" 3; + "~*(?:\b)OutfoxBot(?:\b)" 3; + "~*(?:\b)PageAnalyzer(?:\b)" 3; + "~*(?:\b)Page\ Analyzer(?:\b)" 3; + "~*(?:\b)PageGrabber(?:\b)" 3; + "~*(?:\b)page\ scorer(?:\b)" 3; + "~*(?:\b)PageScorer(?:\b)" 3; + "~*(?:\b)Pandalytics(?:\b)" 3; + "~*(?:\b)Panscient(?:\b)" 3; + "~*(?:\b)Papa\ Foto(?:\b)" 3; + "~*(?:\b)Pavuk(?:\b)" 3; + "~*(?:\b)pcBrowser(?:\b)" 3; + "~*(?:\b)PECL::HTTP(?:\b)" 3; + "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)PHPCrawl(?:\b)" 3; + "~*(?:\b)Picscout(?:\b)" 3; + "~*(?:\b)Picsearch(?:\b)" 3; + "~*(?:\b)PictureFinder(?:\b)" 3; + "~*(?:\b)Pimonster(?:\b)" 3; + "~*(?:\b)Pi-Monster(?:\b)" 3; + "~*(?:\b)Pixray(?:\b)" 3; + "~*(?:\b)PleaseCrawl(?:\b)" 3; + "~*(?:\b)plumanalytics(?:\b)" 3; + "~*(?:\b)Pockey(?:\b)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; + "~*(?:\b)Probethenet(?:\b)" 3; + "~*(?:\b)ProPowerBot(?:\b)" 3; + "~*(?:\b)ProWebWalker(?:\b)" 3; + "~*(?:\b)Psbot(?:\b)" 3; + "~*(?:\b)Pump(?:\b)" 3; + "~*(?:\b)PxBroker(?:\b)" 3; + "~*(?:\b)PyCurl(?:\b)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; + "~*(?:\b)Quick-Crawler(?:\b)" 3; + "~*(?:\b)RankActive(?:\b)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b)" 3; + "~*(?:\b)RankFlex(?:\b)" 3; + "~*(?:\b)RankingBot(?:\b)" 3; + "~*(?:\b)RankingBot2(?:\b)" 3; + "~*(?:\b)Rankivabot(?:\b)" 3; + "~*(?:\b)RankurBot(?:\b)" 3; + "~*(?:\b)RealDownload(?:\b)" 3; + "~*(?:\b)Reaper(?:\b)" 3; + "~*(?:\b)RebelMouse(?:\b)" 3; + "~*(?:\b)Recorder(?:\b)" 3; + "~*(?:\b)RedesScrapy(?:\b)" 3; + "~*(?:\b)ReGet(?:\b)" 3; + "~*(?:\b)RepoMonkey(?:\b)" 3; + "~*(?:\b)Ripper(?:\b)" 3; + "~*(?:\b)RocketCrawler(?:\b)" 3; + "~*(?:\b)Rogerbot(?:\b)" 3; + "~*(?:\b)s1z.ru(?:\b)" 3; + "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)SBIder(?:\b)" 3; + "~*(?:\b)ScanAlert(?:\b)" 3; + "~*(?:\b)Scanbot(?:\b)" 3; + "~*(?:\b)scan.lol(?:\b)" 3; + "~*(?:\b)ScoutJet(?:\b)" 3; + "~*(?:\b)Scrapy(?:\b)" 3; + "~*(?:\b)Screaming(?:\b)" 3; + "~*(?:\b)ScreenerBot(?:\b)" 3; + "~*(?:\b)Searchestate(?:\b)" 3; + "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Semrush(?:\b)" 3; + "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SEOkicks(?:\b)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; + "~*(?:\b)Seomoz(?:\b)" 3; + "~*(?:\b)SEOprofiler(?:\b)" 3; + "~*(?:\b)seoscanners(?:\b)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b)" 3; + "~*(?:\b)SEOstats(?:\b)" 3; + "~*(?:\b)serpstatbot(?:\b)" 3; + "~*(?:\b)sexsearcher(?:\b)" 3; + "~*(?:\b)Shodan(?:\b)" 3; + "~*(?:\b)Siphon(?:\b)" 3; + "~*(?:\b)SISTRIX(?:\b)" 3; + "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteExplorer(?:\b)" 3; + "~*(?:\b)Siteimprove(?:\b)" 3; + "~*(?:\b)SiteLockSpider(?:\b)" 3; + "~*(?:\b)SiteSnagger(?:\b)" 3; + "~*(?:\b)SiteSucker(?:\b)" 3; + "~*(?:\b)Site\ Sucker(?:\b)" 3; + "~*(?:\b)Sitevigil(?:\b)" 3; + "~*(?:\b)SlySearch(?:\b)" 3; + "~*(?:\b)SmartDownload(?:\b)" 3; + "~*(?:\b)SMTBot(?:\b)" 3; + "~*(?:\b)Snake(?:\b)" 3; + "~*(?:\b)Snapbot(?:\b)" 3; + "~*(?:\b)Snoopy(?:\b)" 3; + "~*(?:\b)SocialRankIOBot(?:\b)" 3; + "~*(?:\b)Sociscraper(?:\b)" 3; + "~*(?:\b)sogouspider(?:\b)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; + "~*(?:\b)Sosospider(?:\b)" 3; + "~*(?:\b)Sottopop(?:\b)" 3; + "~*(?:\b)SpaceBison(?:\b)" 3; + "~*(?:\b)Spammen(?:\b)" 3; + "~*(?:\b)SpankBot(?:\b)" 3; + "~*(?:\b)Spanner(?:\b)" 3; + "~*(?:\b)sp_auditbot(?:\b)" 3; + "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spinn3r(?:\b)" 3; + "~*(?:\b)SputnikBot(?:\b)" 3; + "~*(?:\b)spyfu(?:\b)" 3; + "~*(?:\b)Sqlmap(?:\b)" 3; + "~*(?:\b)Sqlworm(?:\b)" 3; + "~*(?:\b)Sqworm(?:\b)" 3; + "~*(?:\b)Steeler(?:\b)" 3; + "~*(?:\b)Stripper(?:\b)" 3; + "~*(?:\b)Sucker(?:\b)" 3; + "~*(?:\b)Sucuri(?:\b)" 3; + "~*(?:\b)SuperBot(?:\b)" 3; + "~*(?:\b)SuperHTTP(?:\b)" 3; + "~*(?:\b)Surfbot(?:\b)" 3; + "~*(?:\b)SurveyBot(?:\b)" 3; + "~*(?:\b)Suzuran(?:\b)" 3; + "~*(?:\b)Swiftbot(?:\b)" 3; + "~*(?:\b)sysscan(?:\b)" 3; + "~*(?:\b)Szukacz(?:\b)" 3; + "~*(?:\b)T0PHackTeam(?:\b)" 3; + "~*(?:\b)T8Abot(?:\b)" 3; + "~*(?:\b)tAkeOut(?:\b)" 3; + "~*(?:\b)Teleport(?:\b)" 3; + "~*(?:\b)TeleportPro(?:\b)" 3; + "~*(?:\b)Telesoft(?:\b)" 3; + "~*(?:\b)Telesphoreo(?:\b)" 3; + "~*(?:\b)Telesphorep(?:\b)" 3; + "~*(?:\b)The\ Intraformant(?:\b)" 3; + "~*(?:\b)TheNomad(?:\b)" 3; + "~*(?:\b)Thumbor(?:\b)" 3; + "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)Titan(?:\b)" 3; + "~*(?:\b)Toata(?:\b)" 3; + "~*(?:\b)Toweyabot(?:\b)" 3; + "~*(?:\b)Tracemyfile(?:\b)" 3; + "~*(?:\b)Trendiction(?:\b)" 3; + "~*(?:\b)Trendictionbot(?:\b)" 3; + "~*(?:\b)trendiction.com(?:\b)" 3; + "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)True_Robot(?:\b)" 3; + "~*(?:\b)Turingos(?:\b)" 3; + "~*(?:\b)Turnitin(?:\b)" 3; + "~*(?:\b)TurnitinBot(?:\b)" 3; + "~*(?:\b)TwengaBot(?:\b)" 3; + "~*(?:\b)Twice(?:\b)" 3; + "~*(?:\b)Typhoeus(?:\b)" 3; + "~*(?:\b)UnisterBot(?:\b)" 3; + "~*(?:\b)Upflow(?:\b)" 3; + "~*(?:\b)URLy.Warning(?:\b)" 3; + "~*(?:\b)URLy\ Warning(?:\b)" 3; + "~*(?:\b)Vacuum(?:\b)" 3; + "~*(?:\b)Vagabondo(?:\b)" 3; + "~*(?:\b)VB\ Project(?:\b)" 3; + "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b)" 3; + "~*(?:\b)VidibleScraper(?:\b)" 3; + "~*(?:\b)Virusdie(?:\b)" 3; + "~*(?:\b)VoidEYE(?:\b)" 3; + "~*(?:\b)Voil(?:\b)" 3; + "~*(?:\b)Voltron(?:\b)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b)" 3; + "~*(?:\b)WallpapersHD(?:\b)" 3; + "~*(?:\b)WASALive-Bot(?:\b)" 3; + "~*(?:\b)WBSearchBot(?:\b)" 3; + "~*(?:\b)Webalta(?:\b)" 3; + "~*(?:\b)WebAuto(?:\b)" 3; + "~*(?:\b)Web\ Auto(?:\b)" 3; + "~*(?:\b)WebBandit(?:\b)" 3; + "~*(?:\b)WebCollage(?:\b)" 3; + "~*(?:\b)Web\ Collage(?:\b)" 3; + "~*(?:\b)WebCopier(?:\b)" 3; + "~*(?:\b)WEBDAV(?:\b)" 3; + "~*(?:\b)WebEnhancer(?:\b)" 3; + "~*(?:\b)Web\ Enhancer(?:\b)" 3; + "~*(?:\b)WebFetch(?:\b)" 3; + "~*(?:\b)Web\ Fetch(?:\b)" 3; + "~*(?:\b)WebFuck(?:\b)" 3; + "~*(?:\b)Web\ Fuck(?:\b)" 3; + "~*(?:\b)WebGo\ IS(?:\b)" 3; + "~*(?:\b)WebImageCollector(?:\b)" 3; + "~*(?:\b)WebLeacher(?:\b)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; + "~*(?:\b)webmeup-crawler(?:\b)" 3; + "~*(?:\b)WebPix(?:\b)" 3; + "~*(?:\b)Web\ Pix(?:\b)" 3; + "~*(?:\b)WebReaper(?:\b)" 3; + "~*(?:\b)WebSauger(?:\b)" 3; + "~*(?:\b)Web\ Sauger(?:\b)" 3; + "~*(?:\b)Webshag(?:\b)" 3; + "~*(?:\b)WebsiteExtractor(?:\b)" 3; + "~*(?:\b)WebsiteQuester(?:\b)" 3; + "~*(?:\b)Website\ Quester(?:\b)" 3; + "~*(?:\b)Webster(?:\b)" 3; + "~*(?:\b)WebStripper(?:\b)" 3; + "~*(?:\b)WebSucker(?:\b)" 3; + "~*(?:\b)Web\ Sucker(?:\b)" 3; + "~*(?:\b)WebWhacker(?:\b)" 3; + "~*(?:\b)WebZIP(?:\b)" 3; + "~*(?:\b)WeSEE(?:\b)" 3; + "~*(?:\b)Whack(?:\b)" 3; + "~*(?:\b)Whacker(?:\b)" 3; + "~*(?:\b)Whatweb(?:\b)" 3; + "~*(?:\b)Who.is\ Bot(?:\b)" 3; + "~*(?:\b)Widow(?:\b)" 3; + "~*(?:\b)WinHTTrack(?:\b)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; + "~*(?:\b)WISENutbot(?:\b)" 3; + "~*(?:\b)Wonderbot(?:\b)" 3; + "~*(?:\b)Woobot(?:\b)" 3; + "~*(?:\b)Wotbox(?:\b)" 3; + "~*(?:\b)Wprecon(?:\b)" 3; + "~*(?:\b)WPScan(?:\b)" 3; + "~*(?:\b)WWW-Collector-E(?:\b)" 3; + "~*(?:\b)WWW-Mechanize(?:\b)" 3; + "~*(?:\b)WWW::Mechanize(?:\b)" 3; + "~*(?:\b)WWWOFFLE(?:\b)" 3; + "~*(?:\b)x09Mozilla(?:\b)" 3; + "~*(?:\b)x22Mozilla(?:\b)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; + "~*(?:\b)Xenu(?:\b)" 3; + "~*(?:\b)xpymep1.exe(?:\b)" 3; + "~*(?:\b)YoudaoBot(?:\b)" 3; + "~*(?:\b)Zade(?:\b)" 3; + "~*(?:\b)Zauba(?:\b)" 3; + "~*(?:\b)zauba.io(?:\b)" 3; + "~*(?:\b)Zermelo(?:\b)" 3; + "~*(?:\b)Zeus(?:\b)" 3; + "~*(?:\b)zgrab(?:\b)" 3; + "~*(?:\b)Zitebot(?:\b)" 3; + "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZumBot(?:\b)" 3; + "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------- @@ -717,38 +717,38 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)adidxbot(?:\b|)" 0; - "~*(?:\b)AdsBot-Google(?:\b|)" 0; - "~*(?:\b)aolbuild(?:\b|)" 0; - "~*(?:\b)bingbot(?:\b|)" 0; - "~*(?:\b)bingpreview(?:\b|)" 0; - "~*(?:\b)DoCoMo(?:\b|)" 0; - "~*(?:\b)duckduckgo(?:\b|)" 0; - "~*(?:\b)facebookexternalhit(?:\b|)" 0; - "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; - "~*(?:\b)Googlebot(?:\b|)" 0; - "~*(?:\b)Googlebot-Image(?:\b|)" 0; - "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; - "~*(?:\b)Googlebot-News(?:\b|)" 0; - "~*(?:\b)Googlebot/Test(?:\b|)" 0; - "~*(?:\b)Googlebot-Video(?:\b|)" 0; - "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; - "~*(?:\b)Gravityscan(?:\b|)" 0; - "~*(?:\b)gsa-crawler(?:\b|)" 0; - "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; - "~*(?:\b)Kraken/0.1(?:\b|)" 0; - "~*(?:\b)LinkedInBot(?:\b|)" 0; - "~*(?:\b)Mediapartners-Google(?:\b|)" 0; - "~*(?:\b)msnbot(?:\b|)" 0; - "~*(?:\b)msnbot-media(?:\b|)" 0; - "~*(?:\b)SAMSUNG(?:\b|)" 0; - "~*(?:\b)Slackbot(?:\b|)" 0; - "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; - "~*(?:\b)slurp(?:\b|)" 0; - "~*(?:\b)teoma(?:\b|)" 0; - "~*(?:\b)TwitterBot(?:\b|)" 0; - "~*(?:\b)Wordpress(?:\b|)" 0; - "~*(?:\b)yahoo(?:\b|)" 0; + "~*(?:\b)adidxbot(?:\b)" 0; + "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bingbot(?:\b)" 0; + "~*(?:\b)bingpreview(?:\b)" 0; + "~*(?:\b)DoCoMo(?:\b)" 0; + "~*(?:\b)duckduckgo(?:\b)" 0; + "~*(?:\b)facebookexternalhit(?:\b)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b)" 0; + "~*(?:\b)Googlebot(?:\b)" 0; + "~*(?:\b)Googlebot-Image(?:\b)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b)" 0; + "~*(?:\b)Googlebot-News(?:\b)" 0; + "~*(?:\b)Googlebot/Test(?:\b)" 0; + "~*(?:\b)Googlebot-Video(?:\b)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; + "~*(?:\b)Gravityscan(?:\b)" 0; + "~*(?:\b)gsa-crawler(?:\b)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b)" 0; + "~*(?:\b)Kraken/0.1(?:\b)" 0; + "~*(?:\b)LinkedInBot(?:\b)" 0; + "~*(?:\b)Mediapartners-Google(?:\b)" 0; + "~*(?:\b)msnbot(?:\b)" 0; + "~*(?:\b)msnbot-media(?:\b)" 0; + "~*(?:\b)SAMSUNG(?:\b)" 0; + "~*(?:\b)Slackbot(?:\b)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 0; + "~*(?:\b)slurp(?:\b)" 0; + "~*(?:\b)teoma(?:\b)" 0; + "~*(?:\b)TwitterBot(?:\b)" 0; + "~*(?:\b)Wordpress(?:\b)" 0; + "~*(?:\b)yahoo(?:\b)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------------------- @@ -760,12 +760,12 @@ map $http_user_agent $bad_bot { #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)jetmon(?:\b|)" 1; - "~*(?:\b)libwww-perl(?:\b|)" 1; - "~*(?:\b)Lynx(?:\b|)" 1; - "~*(?:\b)munin(?:\b|)" 1; - "~*(?:\b)Presto(?:\b|)" 1; - "~*(?:\b)Wget/1.15(?:\b|)" 1; + "~*(?:\b)jetmon(?:\b)" 1; + "~*(?:\b)libwww-perl(?:\b)" 1; + "~*(?:\b)Lynx(?:\b)" 1; + "~*(?:\b)munin(?:\b)" 1; + "~*(?:\b)Presto(?:\b)" 1; + "~*(?:\b)Wget/1.15(?:\b)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ------------------------------------------------------- @@ -778,23 +778,23 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)Alexa(?:\b|)" 2; - "~*(?:\b)archive.org(?:\b|)" 2; - "~*(?:\b)Baidu(?:\b|)" 2; - "~*(?:\b)BUbiNG(?:\b|)" 2; - "~*(?:\b)FlipboardProxy(?:\b|)" 2; - "~*(?:\b)ia_archiver(?:\b|)" 2; - "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; - "~*(?:\b)Proximic(?:\b|)" 2; - "~*(?:\b)R6_CommentReader(?:\b|)" 2; - "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; - "~*(?:\b)RED/1(?:\b|)" 2; - "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; - "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; - "~*(?:\b)Spaidu(?:\b|)" 2; - "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; - "~*(?:\b)YandexBot(?:\b|)" 2; - "~*(?:\b)YandexImages(?:\b|)" 2; + "~*(?:\b)Alexa(?:\b)" 2; + "~*(?:\b)archive.org(?:\b)" 2; + "~*(?:\b)Baidu(?:\b)" 2; + "~*(?:\b)BUbiNG(?:\b)" 2; + "~*(?:\b)FlipboardProxy(?:\b)" 2; + "~*(?:\b)ia_archiver(?:\b)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b)" 2; + "~*(?:\b)Proximic(?:\b)" 2; + "~*(?:\b)R6_CommentReader(?:\b)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b)" 2; + "~*(?:\b)RED/1(?:\b)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b)" 2; + "~*(?:\b)Spaidu(?:\b)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b)" 2; + "~*(?:\b)YandexBot(?:\b)" 2; + "~*(?:\b)YandexImages(?:\b)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 2e2965ffc..12f18b91b 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.188.73.34 1; +35.202.145.110 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index d3d331ed4..c3c284efc 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.188.73.34 0; +35.202.145.110 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 7b00caacd..65f28ba04 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1704 -### Updated: Sat Jun 29 21:12:07 SAST 2019 +### Version: V3.2019.07.1705 +### Updated: Mon Jul 1 08:42:46 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### @@ -147,562 +147,562 @@ map $http_user_agent $bad_bot { # -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b|)" 3; - "~*(?:\b)404checker(?:\b|)" 3; - "~*(?:\b)404enemy(?:\b|)" 3; - "~*(?:\b)80legs(?:\b|)" 3; - "~*(?:\b)Abonti(?:\b|)" 3; - "~*(?:\b)Aboundex(?:\b|)" 3; - "~*(?:\b)Aboundexbot(?:\b|)" 3; - "~*(?:\b)Acunetix(?:\b|)" 3; - "~*(?:\b)ADmantX(?:\b|)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; - "~*(?:\b)AhrefsBot(?:\b|)" 3; - "~*(?:\b)AIBOT(?:\b|)" 3; - "~*(?:\b)AiHitBot(?:\b|)" 3; - "~*(?:\b)Aipbot(?:\b|)" 3; - "~*(?:\b)Alexibot(?:\b|)" 3; - "~*(?:\b)Alligator(?:\b|)" 3; - "~*(?:\b)AllSubmitter(?:\b|)" 3; - "~*(?:\b)AlphaBot(?:\b|)" 3; - "~*(?:\b)Anarchie(?:\b|)" 3; - "~*(?:\b)Apexoo(?:\b|)" 3; - "~*(?:\b)archive.org_bot(?:\b|)" 3; - "~*(?:\b)ASPSeek(?:\b|)" 3; - "~*(?:\b)Asterias(?:\b|)" 3; - "~*(?:\b)Attach(?:\b|)" 3; - "~*(?:\b)autoemailspider(?:\b|)" 3; - "~*(?:\b)BackDoorBot(?:\b|)" 3; - "~*(?:\b)Backlink-Ceck(?:\b|)" 3; - "~*(?:\b)backlink-check(?:\b|)" 3; - "~*(?:\b)BacklinkCrawler(?:\b|)" 3; - "~*(?:\b)BackStreet(?:\b|)" 3; - "~*(?:\b)BackWeb(?:\b|)" 3; - "~*(?:\b)Badass(?:\b|)" 3; - "~*(?:\b)Bandit(?:\b|)" 3; - "~*(?:\b)Barkrowler(?:\b|)" 3; - "~*(?:\b)BatchFTP(?:\b|)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; - "~*(?:\b)BBBike(?:\b|)" 3; - "~*(?:\b)BDCbot(?:\b|)" 3; - "~*(?:\b)BDFetch(?:\b|)" 3; - "~*(?:\b)BetaBot(?:\b|)" 3; - "~*(?:\b)Bigfoot(?:\b|)" 3; - "~*(?:\b)Bitacle(?:\b|)" 3; - "~*(?:\b)Blackboard(?:\b|)" 3; - "~*(?:\b)Black\ Hole(?:\b|)" 3; - "~*(?:\b)BlackWidow(?:\b|)" 3; - "~*(?:\b)BLEXBot(?:\b|)" 3; - "~*(?:\b)Blow(?:\b|)" 3; - "~*(?:\b)BlowFish(?:\b|)" 3; - "~*(?:\b)Boardreader(?:\b|)" 3; - "~*(?:\b)Bolt(?:\b|)" 3; - "~*(?:\b)BotALot(?:\b|)" 3; - "~*(?:\b)Brandprotect(?:\b|)" 3; - "~*(?:\b)Brandwatch(?:\b|)" 3; - "~*(?:\b)Buddy(?:\b|)" 3; - "~*(?:\b)BuiltBotTough(?:\b|)" 3; - "~*(?:\b)BuiltWith(?:\b|)" 3; - "~*(?:\b)Bullseye(?:\b|)" 3; - "~*(?:\b)BunnySlippers(?:\b|)" 3; - "~*(?:\b)BuzzSumo(?:\b|)" 3; - "~*(?:\b)Calculon(?:\b|)" 3; - "~*(?:\b)CATExplorador(?:\b|)" 3; - "~*(?:\b)CazoodleBot(?:\b|)" 3; - "~*(?:\b)CCBot(?:\b|)" 3; - "~*(?:\b)Cegbfeieh(?:\b|)" 3; - "~*(?:\b)CheeseBot(?:\b|)" 3; - "~*(?:\b)CherryPicker(?:\b|)" 3; - "~*(?:\b)CheTeam(?:\b|)" 3; - "~*(?:\b)ChinaClaw(?:\b|)" 3; - "~*(?:\b)Chlooe(?:\b|)" 3; - "~*(?:\b)Claritybot(?:\b|)" 3; - "~*(?:\b)Cliqzbot(?:\b|)" 3; - "~*(?:\b)Cloud\ mapping(?:\b|)" 3; - "~*(?:\b)coccocbot-web(?:\b|)" 3; - "~*(?:\b)Cogentbot(?:\b|)" 3; - "~*(?:\b)cognitiveseo(?:\b|)" 3; - "~*(?:\b)Collector(?:\b|)" 3; - "~*(?:\b)com.plumanalytics(?:\b|)" 3; - "~*(?:\b)Copier(?:\b|)" 3; - "~*(?:\b)CopyRightCheck(?:\b|)" 3; - "~*(?:\b)Copyscape(?:\b|)" 3; - "~*(?:\b)Cosmos(?:\b|)" 3; - "~*(?:\b)Craftbot(?:\b|)" 3; - "~*(?:\b)crawler4j(?:\b|)" 3; - "~*(?:\b)crawler.feedback(?:\b|)" 3; - "~*(?:\b)crawl.sogou.com(?:\b|)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; - "~*(?:\b)Crescent(?:\b|)" 3; - "~*(?:\b)CrunchBot(?:\b|)" 3; - "~*(?:\b)CSHttp(?:\b|)" 3; - "~*(?:\b)Curious(?:\b|)" 3; - "~*(?:\b)Custo(?:\b|)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; - "~*(?:\b)DataCha0s(?:\b|)" 3; - "~*(?:\b)DBLBot(?:\b|)" 3; - "~*(?:\b)demandbase-bot(?:\b|)" 3; - "~*(?:\b)Demon(?:\b|)" 3; - "~*(?:\b)Deusu(?:\b|)" 3; - "~*(?:\b)Devil(?:\b|)" 3; - "~*(?:\b)Digincore(?:\b|)" 3; - "~*(?:\b)DigitalPebble(?:\b|)" 3; - "~*(?:\b)DIIbot(?:\b|)" 3; - "~*(?:\b)Dirbuster(?:\b|)" 3; - "~*(?:\b)Disco(?:\b|)" 3; - "~*(?:\b)Discobot(?:\b|)" 3; - "~*(?:\b)Discoverybot(?:\b|)" 3; - "~*(?:\b)Dispatch(?:\b|)" 3; - "~*(?:\b)DittoSpyder(?:\b|)" 3; - "~*(?:\b)DnyzBot(?:\b|)" 3; - "~*(?:\b)DomainAppender(?:\b|)" 3; - "~*(?:\b)DomainCrawler(?:\b|)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; - "~*(?:\b)DomainStatsBot(?:\b|)" 3; - "~*(?:\b)Dotbot(?:\b|)" 3; - "~*(?:\b)Download\ Wonder(?:\b|)" 3; - "~*(?:\b)Dragonfly(?:\b|)" 3; - "~*(?:\b)Drip(?:\b|)" 3; - "~*(?:\b)DSearch(?:\b|)" 3; - "~*(?:\b)DTS\ Agent(?:\b|)" 3; - "~*(?:\b)EasyDL(?:\b|)" 3; - "~*(?:\b)Ebingbong(?:\b|)" 3; - "~*(?:\b)eCatch(?:\b|)" 3; - "~*(?:\b)ECCP/1.0(?:\b|)" 3; - "~*(?:\b)Ecxi(?:\b|)" 3; - "~*(?:\b)EirGrabber(?:\b|)" 3; - "~*(?:\b)EMail\ Siphon(?:\b|)" 3; - "~*(?:\b)EMail\ Wolf(?:\b|)" 3; - "~*(?:\b)EroCrawler(?:\b|)" 3; - "~*(?:\b)evc-batch(?:\b|)" 3; - "~*(?:\b)Evil(?:\b|)" 3; - "~*(?:\b)Exabot(?:\b|)" 3; - "~*(?:\b)Express\ WebPictures(?:\b|)" 3; - "~*(?:\b)ExtLinksBot(?:\b|)" 3; - "~*(?:\b)Extractor(?:\b|)" 3; - "~*(?:\b)ExtractorPro(?:\b|)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; - "~*(?:\b)EyeNetIE(?:\b|)" 3; - "~*(?:\b)Ezooms(?:\b|)" 3; - "~*(?:\b)facebookscraper(?:\b|)" 3; - "~*(?:\b)FDM(?:\b|)" 3; - "~*(?:\b)FemtosearchBot(?:\b|)" 3; - "~*(?:\b)FHscan(?:\b|)" 3; - "~*(?:\b)Fimap(?:\b|)" 3; - "~*(?:\b)Firefox/7.0(?:\b|)" 3; - "~*(?:\b)FlashGet(?:\b|)" 3; - "~*(?:\b)Flunky(?:\b|)" 3; - "~*(?:\b)Foobot(?:\b|)" 3; - "~*(?:\b)Freeuploader(?:\b|)" 3; - "~*(?:\b)FrontPage(?:\b|)" 3; - "~*(?:\b)FyberSpider(?:\b|)" 3; - "~*(?:\b)Fyrebot(?:\b|)" 3; - "~*(?:\b)GalaxyBot(?:\b|)" 3; - "~*(?:\b)Genieo(?:\b|)" 3; - "~*(?:\b)GermCrawler(?:\b|)" 3; - "~*(?:\b)Getintent(?:\b|)" 3; - "~*(?:\b)GetRight(?:\b|)" 3; - "~*(?:\b)GetWeb(?:\b|)" 3; - "~*(?:\b)Gigablast(?:\b|)" 3; - "~*(?:\b)Gigabot(?:\b|)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; - "~*(?:\b)Gotit(?:\b|)" 3; - "~*(?:\b)GoZilla(?:\b|)" 3; - "~*(?:\b)Go!Zilla(?:\b|)" 3; - "~*(?:\b)Grabber(?:\b|)" 3; - "~*(?:\b)GrabNet(?:\b|)" 3; - "~*(?:\b)Grafula(?:\b|)" 3; - "~*(?:\b)GrapeFX(?:\b|)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; - "~*(?:\b)GridBot(?:\b|)" 3; - "~*(?:\b)GT::WWW(?:\b|)" 3; - "~*(?:\b)Haansoft(?:\b|)" 3; - "~*(?:\b)HaosouSpider(?:\b|)" 3; - "~*(?:\b)Harvest(?:\b|)" 3; - "~*(?:\b)Havij(?:\b|)" 3; - "~*(?:\b)HEADMasterSEO(?:\b|)" 3; - "~*(?:\b)Heritrix(?:\b|)" 3; - "~*(?:\b)Hloader(?:\b|)" 3; - "~*(?:\b)HMView(?:\b|)" 3; - "~*(?:\b)HTMLparser(?:\b|)" 3; - "~*(?:\b)HTTP::Lite(?:\b|)" 3; - "~*(?:\b)HTTrack(?:\b|)" 3; - "~*(?:\b)Humanlinks(?:\b|)" 3; - "~*(?:\b)HybridBot(?:\b|)" 3; - "~*(?:\b)Iblog(?:\b|)" 3; - "~*(?:\b)IDBot(?:\b|)" 3; - "~*(?:\b)Id-search(?:\b|)" 3; - "~*(?:\b)IlseBot(?:\b|)" 3; - "~*(?:\b)Image\ Fetch(?:\b|)" 3; - "~*(?:\b)Image\ Sucker(?:\b|)" 3; - "~*(?:\b)IndeedBot(?:\b|)" 3; - "~*(?:\b)Indy\ Library(?:\b|)" 3; - "~*(?:\b)InfoNaviRobot(?:\b|)" 3; - "~*(?:\b)InfoTekies(?:\b|)" 3; - "~*(?:\b)instabid(?:\b|)" 3; - "~*(?:\b)Intelliseek(?:\b|)" 3; - "~*(?:\b)InterGET(?:\b|)" 3; - "~*(?:\b)Internet\ Ninja(?:\b|)" 3; - "~*(?:\b)InternetSeer(?:\b|)" 3; - "~*(?:\b)internetVista\ monitor(?:\b|)" 3; - "~*(?:\b)ips-agent(?:\b|)" 3; - "~*(?:\b)Iria(?:\b|)" 3; - "~*(?:\b)IRLbot(?:\b|)" 3; - "~*(?:\b)Iskanie(?:\b|)" 3; - "~*(?:\b)IstellaBot(?:\b|)" 3; - "~*(?:\b)JamesBOT(?:\b|)" 3; - "~*(?:\b)Jbrofuzz(?:\b|)" 3; - "~*(?:\b)JennyBot(?:\b|)" 3; - "~*(?:\b)JetCar(?:\b|)" 3; - "~*(?:\b)Jetty(?:\b|)" 3; - "~*(?:\b)JikeSpider(?:\b|)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; - "~*(?:\b)Joomla(?:\b|)" 3; - "~*(?:\b)Jorgee(?:\b|)" 3; - "~*(?:\b)JustView(?:\b|)" 3; - "~*(?:\b)Jyxobot(?:\b|)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; - "~*(?:\b)Keyword\ Density(?:\b|)" 3; - "~*(?:\b)Kozmosbot(?:\b|)" 3; - "~*(?:\b)Lanshanbot(?:\b|)" 3; - "~*(?:\b)Larbin(?:\b|)" 3; - "~*(?:\b)LeechFTP(?:\b|)" 3; - "~*(?:\b)LeechGet(?:\b|)" 3; - "~*(?:\b)LexiBot(?:\b|)" 3; - "~*(?:\b)Lftp(?:\b|)" 3; - "~*(?:\b)LibWeb(?:\b|)" 3; - "~*(?:\b)Libwhisker(?:\b|)" 3; - "~*(?:\b)Lightspeedsystems(?:\b|)" 3; - "~*(?:\b)Likse(?:\b|)" 3; - "~*(?:\b)Linkdexbot(?:\b|)" 3; - "~*(?:\b)LinkextractorPro(?:\b|)" 3; - "~*(?:\b)LinkpadBot(?:\b|)" 3; - "~*(?:\b)LinkScan(?:\b|)" 3; - "~*(?:\b)LinksManager(?:\b|)" 3; - "~*(?:\b)LinkWalker(?:\b|)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; - "~*(?:\b)Lipperhey(?:\b|)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; - "~*(?:\b)Litemage_walker(?:\b|)" 3; - "~*(?:\b)Lmspider(?:\b|)" 3; - "~*(?:\b)LNSpiderguy(?:\b|)" 3; - "~*(?:\b)Ltx71(?:\b|)" 3; - "~*(?:\b)lwp-request(?:\b|)" 3; - "~*(?:\b)LWP::Simple(?:\b|)" 3; - "~*(?:\b)lwp-trivial(?:\b|)" 3; - "~*(?:\b)Magnet(?:\b|)" 3; - "~*(?:\b)Mag-Net(?:\b|)" 3; - "~*(?:\b)magpie-crawler(?:\b|)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; - "~*(?:\b)Majestic12(?:\b|)" 3; - "~*(?:\b)Majestic-SEO(?:\b|)" 3; - "~*(?:\b)Majestic\ SEO(?:\b|)" 3; - "~*(?:\b)MarkMonitor(?:\b|)" 3; - "~*(?:\b)MarkWatch(?:\b|)" 3; - "~*(?:\b)Masscan(?:\b|)" 3; - "~*(?:\b)Mass\ Downloader(?:\b|)" 3; - "~*(?:\b)Mata\ Hari(?:\b|)" 3; - "~*(?:\b)MauiBot(?:\b|)" 3; - "~*(?:\b)meanpathbot(?:\b|)" 3; - "~*(?:\b)Meanpathbot(?:\b|)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; - "~*(?:\b)mediawords(?:\b|)" 3; - "~*(?:\b)MegaIndex.ru(?:\b|)" 3; - "~*(?:\b)Metauri(?:\b|)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; - "~*(?:\b)MIDown\ tool(?:\b|)" 3; - "~*(?:\b)MIIxpc(?:\b|)" 3; - "~*(?:\b)Mister\ PiX(?:\b|)" 3; - "~*(?:\b)MJ12bot(?:\b|)" 3; - "~*(?:\b)Mojeek(?:\b|)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; - "~*(?:\b)Mr.4x3(?:\b|)" 3; - "~*(?:\b)MSFrontPage(?:\b|)" 3; - "~*(?:\b)MSIECrawler(?:\b|)" 3; - "~*(?:\b)Msrabot(?:\b|)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; - "~*(?:\b)muhstik-scan(?:\b|)" 3; - "~*(?:\b)Musobot(?:\b|)" 3; - "~*(?:\b)Name\ Intelligence(?:\b|)" 3; - "~*(?:\b)Nameprotect(?:\b|)" 3; - "~*(?:\b)Navroad(?:\b|)" 3; - "~*(?:\b)NearSite(?:\b|)" 3; - "~*(?:\b)Needle(?:\b|)" 3; - "~*(?:\b)Nessus(?:\b|)" 3; - "~*(?:\b)NetAnts(?:\b|)" 3; - "~*(?:\b)Netcraft(?:\b|)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; - "~*(?:\b)NetLyzer(?:\b|)" 3; - "~*(?:\b)NetMechanic(?:\b|)" 3; - "~*(?:\b)NetSpider(?:\b|)" 3; - "~*(?:\b)Nettrack(?:\b|)" 3; - "~*(?:\b)Net\ Vampire(?:\b|)" 3; - "~*(?:\b)Netvibes(?:\b|)" 3; - "~*(?:\b)NetZIP(?:\b|)" 3; - "~*(?:\b)NextGenSearchBot(?:\b|)" 3; - "~*(?:\b)Nibbler(?:\b|)" 3; - "~*(?:\b)NICErsPRO(?:\b|)" 3; - "~*(?:\b)Niki-bot(?:\b|)" 3; - "~*(?:\b)Nikto(?:\b|)" 3; - "~*(?:\b)NimbleCrawler(?:\b|)" 3; - "~*(?:\b)Nimbostratus(?:\b|)" 3; - "~*(?:\b)Ninja(?:\b|)" 3; - "~*(?:\b)Nmap(?:\b|)" 3; - "~*(?:\b)NPbot(?:\b|)" 3; - "~*(?:\b)Nutch(?:\b|)" 3; - "~*(?:\b)oBot(?:\b|)" 3; - "~*(?:\b)Octopus(?:\b|)" 3; - "~*(?:\b)Offline\ Explorer(?:\b|)" 3; - "~*(?:\b)Offline\ Navigator(?:\b|)" 3; - "~*(?:\b)OnCrawl(?:\b|)" 3; - "~*(?:\b)Openfind(?:\b|)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; - "~*(?:\b)Openvas(?:\b|)" 3; - "~*(?:\b)OpenVAS(?:\b|)" 3; - "~*(?:\b)OrangeBot(?:\b|)" 3; - "~*(?:\b)OrangeSpider(?:\b|)" 3; - "~*(?:\b)OutclicksBot(?:\b|)" 3; - "~*(?:\b)OutfoxBot(?:\b|)" 3; - "~*(?:\b)PageAnalyzer(?:\b|)" 3; - "~*(?:\b)Page\ Analyzer(?:\b|)" 3; - "~*(?:\b)PageGrabber(?:\b|)" 3; - "~*(?:\b)page\ scorer(?:\b|)" 3; - "~*(?:\b)PageScorer(?:\b|)" 3; - "~*(?:\b)Pandalytics(?:\b|)" 3; - "~*(?:\b)Panscient(?:\b|)" 3; - "~*(?:\b)Papa\ Foto(?:\b|)" 3; - "~*(?:\b)Pavuk(?:\b|)" 3; - "~*(?:\b)pcBrowser(?:\b|)" 3; - "~*(?:\b)PECL::HTTP(?:\b|)" 3; - "~*(?:\b)PeoplePal(?:\b|)" 3; - "~*(?:\b)PHPCrawl(?:\b|)" 3; - "~*(?:\b)Picscout(?:\b|)" 3; - "~*(?:\b)Picsearch(?:\b|)" 3; - "~*(?:\b)PictureFinder(?:\b|)" 3; - "~*(?:\b)Pimonster(?:\b|)" 3; - "~*(?:\b)Pi-Monster(?:\b|)" 3; - "~*(?:\b)Pixray(?:\b|)" 3; - "~*(?:\b)PleaseCrawl(?:\b|)" 3; - "~*(?:\b)plumanalytics(?:\b|)" 3; - "~*(?:\b)Pockey(?:\b|)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; - "~*(?:\b)Probethenet(?:\b|)" 3; - "~*(?:\b)ProPowerBot(?:\b|)" 3; - "~*(?:\b)ProWebWalker(?:\b|)" 3; - "~*(?:\b)Psbot(?:\b|)" 3; - "~*(?:\b)Pump(?:\b|)" 3; - "~*(?:\b)PxBroker(?:\b|)" 3; - "~*(?:\b)PyCurl(?:\b|)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; - "~*(?:\b)Quick-Crawler(?:\b|)" 3; - "~*(?:\b)RankActive(?:\b|)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; - "~*(?:\b)RankFlex(?:\b|)" 3; - "~*(?:\b)RankingBot(?:\b|)" 3; - "~*(?:\b)RankingBot2(?:\b|)" 3; - "~*(?:\b)Rankivabot(?:\b|)" 3; - "~*(?:\b)RankurBot(?:\b|)" 3; - "~*(?:\b)RealDownload(?:\b|)" 3; - "~*(?:\b)Reaper(?:\b|)" 3; - "~*(?:\b)RebelMouse(?:\b|)" 3; - "~*(?:\b)Recorder(?:\b|)" 3; - "~*(?:\b)RedesScrapy(?:\b|)" 3; - "~*(?:\b)ReGet(?:\b|)" 3; - "~*(?:\b)RepoMonkey(?:\b|)" 3; - "~*(?:\b)Ripper(?:\b|)" 3; - "~*(?:\b)RocketCrawler(?:\b|)" 3; - "~*(?:\b)Rogerbot(?:\b|)" 3; - "~*(?:\b)s1z.ru(?:\b|)" 3; - "~*(?:\b)SalesIntelligent(?:\b|)" 3; - "~*(?:\b)SBIder(?:\b|)" 3; - "~*(?:\b)ScanAlert(?:\b|)" 3; - "~*(?:\b)Scanbot(?:\b|)" 3; - "~*(?:\b)scan.lol(?:\b|)" 3; - "~*(?:\b)ScoutJet(?:\b|)" 3; - "~*(?:\b)Scrapy(?:\b|)" 3; - "~*(?:\b)Screaming(?:\b|)" 3; - "~*(?:\b)ScreenerBot(?:\b|)" 3; - "~*(?:\b)Searchestate(?:\b|)" 3; - "~*(?:\b)SearchmetricsBot(?:\b|)" 3; - "~*(?:\b)Semrush(?:\b|)" 3; - "~*(?:\b)SemrushBot(?:\b|)" 3; - "~*(?:\b)SEOkicks(?:\b|)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; - "~*(?:\b)Seomoz(?:\b|)" 3; - "~*(?:\b)SEOprofiler(?:\b|)" 3; - "~*(?:\b)seoscanners(?:\b|)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; - "~*(?:\b)SEOstats(?:\b|)" 3; - "~*(?:\b)serpstatbot(?:\b|)" 3; - "~*(?:\b)sexsearcher(?:\b|)" 3; - "~*(?:\b)Shodan(?:\b|)" 3; - "~*(?:\b)Siphon(?:\b|)" 3; - "~*(?:\b)SISTRIX(?:\b|)" 3; - "~*(?:\b)Sitebeam(?:\b|)" 3; - "~*(?:\b)SiteExplorer(?:\b|)" 3; - "~*(?:\b)Siteimprove(?:\b|)" 3; - "~*(?:\b)SiteLockSpider(?:\b|)" 3; - "~*(?:\b)SiteSnagger(?:\b|)" 3; - "~*(?:\b)SiteSucker(?:\b|)" 3; - "~*(?:\b)Site\ Sucker(?:\b|)" 3; - "~*(?:\b)Sitevigil(?:\b|)" 3; - "~*(?:\b)SlySearch(?:\b|)" 3; - "~*(?:\b)SmartDownload(?:\b|)" 3; - "~*(?:\b)SMTBot(?:\b|)" 3; - "~*(?:\b)Snake(?:\b|)" 3; - "~*(?:\b)Snapbot(?:\b|)" 3; - "~*(?:\b)Snoopy(?:\b|)" 3; - "~*(?:\b)SocialRankIOBot(?:\b|)" 3; - "~*(?:\b)Sociscraper(?:\b|)" 3; - "~*(?:\b)sogouspider(?:\b|)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; - "~*(?:\b)Sosospider(?:\b|)" 3; - "~*(?:\b)Sottopop(?:\b|)" 3; - "~*(?:\b)SpaceBison(?:\b|)" 3; - "~*(?:\b)Spammen(?:\b|)" 3; - "~*(?:\b)SpankBot(?:\b|)" 3; - "~*(?:\b)Spanner(?:\b|)" 3; - "~*(?:\b)sp_auditbot(?:\b|)" 3; - "~*(?:\b)Spbot(?:\b|)" 3; - "~*(?:\b)Spinn3r(?:\b|)" 3; - "~*(?:\b)SputnikBot(?:\b|)" 3; - "~*(?:\b)spyfu(?:\b|)" 3; - "~*(?:\b)Sqlmap(?:\b|)" 3; - "~*(?:\b)Sqlworm(?:\b|)" 3; - "~*(?:\b)Sqworm(?:\b|)" 3; - "~*(?:\b)Steeler(?:\b|)" 3; - "~*(?:\b)Stripper(?:\b|)" 3; - "~*(?:\b)Sucker(?:\b|)" 3; - "~*(?:\b)Sucuri(?:\b|)" 3; - "~*(?:\b)SuperBot(?:\b|)" 3; - "~*(?:\b)SuperHTTP(?:\b|)" 3; - "~*(?:\b)Surfbot(?:\b|)" 3; - "~*(?:\b)SurveyBot(?:\b|)" 3; - "~*(?:\b)Suzuran(?:\b|)" 3; - "~*(?:\b)Swiftbot(?:\b|)" 3; - "~*(?:\b)sysscan(?:\b|)" 3; - "~*(?:\b)Szukacz(?:\b|)" 3; - "~*(?:\b)T0PHackTeam(?:\b|)" 3; - "~*(?:\b)T8Abot(?:\b|)" 3; - "~*(?:\b)tAkeOut(?:\b|)" 3; - "~*(?:\b)Teleport(?:\b|)" 3; - "~*(?:\b)TeleportPro(?:\b|)" 3; - "~*(?:\b)Telesoft(?:\b|)" 3; - "~*(?:\b)Telesphoreo(?:\b|)" 3; - "~*(?:\b)Telesphorep(?:\b|)" 3; - "~*(?:\b)The\ Intraformant(?:\b|)" 3; - "~*(?:\b)TheNomad(?:\b|)" 3; - "~*(?:\b)Thumbor(?:\b|)" 3; - "~*(?:\b)TightTwatBot(?:\b|)" 3; - "~*(?:\b)Titan(?:\b|)" 3; - "~*(?:\b)Toata(?:\b|)" 3; - "~*(?:\b)Toweyabot(?:\b|)" 3; - "~*(?:\b)Tracemyfile(?:\b|)" 3; - "~*(?:\b)Trendiction(?:\b|)" 3; - "~*(?:\b)Trendictionbot(?:\b|)" 3; - "~*(?:\b)trendiction.com(?:\b|)" 3; - "~*(?:\b)trendiction.de(?:\b|)" 3; - "~*(?:\b)True_Robot(?:\b|)" 3; - "~*(?:\b)Turingos(?:\b|)" 3; - "~*(?:\b)Turnitin(?:\b|)" 3; - "~*(?:\b)TurnitinBot(?:\b|)" 3; - "~*(?:\b)TwengaBot(?:\b|)" 3; - "~*(?:\b)Twice(?:\b|)" 3; - "~*(?:\b)Typhoeus(?:\b|)" 3; - "~*(?:\b)UnisterBot(?:\b|)" 3; - "~*(?:\b)Upflow(?:\b|)" 3; - "~*(?:\b)URLy.Warning(?:\b|)" 3; - "~*(?:\b)URLy\ Warning(?:\b|)" 3; - "~*(?:\b)Vacuum(?:\b|)" 3; - "~*(?:\b)Vagabondo(?:\b|)" 3; - "~*(?:\b)VB\ Project(?:\b|)" 3; - "~*(?:\b)VCI(?:\b|)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; - "~*(?:\b)VidibleScraper(?:\b|)" 3; - "~*(?:\b)Virusdie(?:\b|)" 3; - "~*(?:\b)VoidEYE(?:\b|)" 3; - "~*(?:\b)Voil(?:\b|)" 3; - "~*(?:\b)Voltron(?:\b|)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; - "~*(?:\b)WallpapersHD(?:\b|)" 3; - "~*(?:\b)WASALive-Bot(?:\b|)" 3; - "~*(?:\b)WBSearchBot(?:\b|)" 3; - "~*(?:\b)Webalta(?:\b|)" 3; - "~*(?:\b)WebAuto(?:\b|)" 3; - "~*(?:\b)Web\ Auto(?:\b|)" 3; - "~*(?:\b)WebBandit(?:\b|)" 3; - "~*(?:\b)WebCollage(?:\b|)" 3; - "~*(?:\b)Web\ Collage(?:\b|)" 3; - "~*(?:\b)WebCopier(?:\b|)" 3; - "~*(?:\b)WEBDAV(?:\b|)" 3; - "~*(?:\b)WebEnhancer(?:\b|)" 3; - "~*(?:\b)Web\ Enhancer(?:\b|)" 3; - "~*(?:\b)WebFetch(?:\b|)" 3; - "~*(?:\b)Web\ Fetch(?:\b|)" 3; - "~*(?:\b)WebFuck(?:\b|)" 3; - "~*(?:\b)Web\ Fuck(?:\b|)" 3; - "~*(?:\b)WebGo\ IS(?:\b|)" 3; - "~*(?:\b)WebImageCollector(?:\b|)" 3; - "~*(?:\b)WebLeacher(?:\b|)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; - "~*(?:\b)webmeup-crawler(?:\b|)" 3; - "~*(?:\b)WebPix(?:\b|)" 3; - "~*(?:\b)Web\ Pix(?:\b|)" 3; - "~*(?:\b)WebReaper(?:\b|)" 3; - "~*(?:\b)WebSauger(?:\b|)" 3; - "~*(?:\b)Web\ Sauger(?:\b|)" 3; - "~*(?:\b)Webshag(?:\b|)" 3; - "~*(?:\b)WebsiteExtractor(?:\b|)" 3; - "~*(?:\b)WebsiteQuester(?:\b|)" 3; - "~*(?:\b)Website\ Quester(?:\b|)" 3; - "~*(?:\b)Webster(?:\b|)" 3; - "~*(?:\b)WebStripper(?:\b|)" 3; - "~*(?:\b)WebSucker(?:\b|)" 3; - "~*(?:\b)Web\ Sucker(?:\b|)" 3; - "~*(?:\b)WebWhacker(?:\b|)" 3; - "~*(?:\b)WebZIP(?:\b|)" 3; - "~*(?:\b)WeSEE(?:\b|)" 3; - "~*(?:\b)Whack(?:\b|)" 3; - "~*(?:\b)Whacker(?:\b|)" 3; - "~*(?:\b)Whatweb(?:\b|)" 3; - "~*(?:\b)Who.is\ Bot(?:\b|)" 3; - "~*(?:\b)Widow(?:\b|)" 3; - "~*(?:\b)WinHTTrack(?:\b|)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; - "~*(?:\b)WISENutbot(?:\b|)" 3; - "~*(?:\b)Wonderbot(?:\b|)" 3; - "~*(?:\b)Woobot(?:\b|)" 3; - "~*(?:\b)Wotbox(?:\b|)" 3; - "~*(?:\b)Wprecon(?:\b|)" 3; - "~*(?:\b)WPScan(?:\b|)" 3; - "~*(?:\b)WWW-Collector-E(?:\b|)" 3; - "~*(?:\b)WWW-Mechanize(?:\b|)" 3; - "~*(?:\b)WWW::Mechanize(?:\b|)" 3; - "~*(?:\b)WWWOFFLE(?:\b|)" 3; - "~*(?:\b)x09Mozilla(?:\b|)" 3; - "~*(?:\b)x22Mozilla(?:\b|)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; - "~*(?:\b)Xenu(?:\b|)" 3; - "~*(?:\b)xpymep1.exe(?:\b|)" 3; - "~*(?:\b)YoudaoBot(?:\b|)" 3; - "~*(?:\b)Zade(?:\b|)" 3; - "~*(?:\b)Zauba(?:\b|)" 3; - "~*(?:\b)zauba.io(?:\b|)" 3; - "~*(?:\b)Zermelo(?:\b|)" 3; - "~*(?:\b)Zeus(?:\b|)" 3; - "~*(?:\b)zgrab(?:\b|)" 3; - "~*(?:\b)Zitebot(?:\b|)" 3; - "~*(?:\b)ZmEu(?:\b|)" 3; - "~*(?:\b)ZumBot(?:\b|)" 3; - "~*(?:\b)ZyBorg(?:\b|)" 3; + "~*(?:\b)360Spider(?:\b)" 3; + "~*(?:\b)404checker(?:\b)" 3; + "~*(?:\b)404enemy(?:\b)" 3; + "~*(?:\b)80legs(?:\b)" 3; + "~*(?:\b)Abonti(?:\b)" 3; + "~*(?:\b)Aboundex(?:\b)" 3; + "~*(?:\b)Aboundexbot(?:\b)" 3; + "~*(?:\b)Acunetix(?:\b)" 3; + "~*(?:\b)ADmantX(?:\b)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; + "~*(?:\b)AhrefsBot(?:\b)" 3; + "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)AiHitBot(?:\b)" 3; + "~*(?:\b)Aipbot(?:\b)" 3; + "~*(?:\b)Alexibot(?:\b)" 3; + "~*(?:\b)Alligator(?:\b)" 3; + "~*(?:\b)AllSubmitter(?:\b)" 3; + "~*(?:\b)AlphaBot(?:\b)" 3; + "~*(?:\b)Anarchie(?:\b)" 3; + "~*(?:\b)Apexoo(?:\b)" 3; + "~*(?:\b)archive.org_bot(?:\b)" 3; + "~*(?:\b)ASPSeek(?:\b)" 3; + "~*(?:\b)Asterias(?:\b)" 3; + "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)BackDoorBot(?:\b)" 3; + "~*(?:\b)Backlink-Ceck(?:\b)" 3; + "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)BacklinkCrawler(?:\b)" 3; + "~*(?:\b)BackStreet(?:\b)" 3; + "~*(?:\b)BackWeb(?:\b)" 3; + "~*(?:\b)Badass(?:\b)" 3; + "~*(?:\b)Bandit(?:\b)" 3; + "~*(?:\b)Barkrowler(?:\b)" 3; + "~*(?:\b)BatchFTP(?:\b)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; + "~*(?:\b)BBBike(?:\b)" 3; + "~*(?:\b)BDCbot(?:\b)" 3; + "~*(?:\b)BDFetch(?:\b)" 3; + "~*(?:\b)BetaBot(?:\b)" 3; + "~*(?:\b)Bigfoot(?:\b)" 3; + "~*(?:\b)Bitacle(?:\b)" 3; + "~*(?:\b)Blackboard(?:\b)" 3; + "~*(?:\b)Black\ Hole(?:\b)" 3; + "~*(?:\b)BlackWidow(?:\b)" 3; + "~*(?:\b)BLEXBot(?:\b)" 3; + "~*(?:\b)Blow(?:\b)" 3; + "~*(?:\b)BlowFish(?:\b)" 3; + "~*(?:\b)Boardreader(?:\b)" 3; + "~*(?:\b)Bolt(?:\b)" 3; + "~*(?:\b)BotALot(?:\b)" 3; + "~*(?:\b)Brandprotect(?:\b)" 3; + "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buddy(?:\b)" 3; + "~*(?:\b)BuiltBotTough(?:\b)" 3; + "~*(?:\b)BuiltWith(?:\b)" 3; + "~*(?:\b)Bullseye(?:\b)" 3; + "~*(?:\b)BunnySlippers(?:\b)" 3; + "~*(?:\b)BuzzSumo(?:\b)" 3; + "~*(?:\b)Calculon(?:\b)" 3; + "~*(?:\b)CATExplorador(?:\b)" 3; + "~*(?:\b)CazoodleBot(?:\b)" 3; + "~*(?:\b)CCBot(?:\b)" 3; + "~*(?:\b)Cegbfeieh(?:\b)" 3; + "~*(?:\b)CheeseBot(?:\b)" 3; + "~*(?:\b)CherryPicker(?:\b)" 3; + "~*(?:\b)CheTeam(?:\b)" 3; + "~*(?:\b)ChinaClaw(?:\b)" 3; + "~*(?:\b)Chlooe(?:\b)" 3; + "~*(?:\b)Claritybot(?:\b)" 3; + "~*(?:\b)Cliqzbot(?:\b)" 3; + "~*(?:\b)Cloud\ mapping(?:\b)" 3; + "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cogentbot(?:\b)" 3; + "~*(?:\b)cognitiveseo(?:\b)" 3; + "~*(?:\b)Collector(?:\b)" 3; + "~*(?:\b)com.plumanalytics(?:\b)" 3; + "~*(?:\b)Copier(?:\b)" 3; + "~*(?:\b)CopyRightCheck(?:\b)" 3; + "~*(?:\b)Copyscape(?:\b)" 3; + "~*(?:\b)Cosmos(?:\b)" 3; + "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)crawler4j(?:\b)" 3; + "~*(?:\b)crawler.feedback(?:\b)" 3; + "~*(?:\b)crawl.sogou.com(?:\b)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b)" 3; + "~*(?:\b)Crescent(?:\b)" 3; + "~*(?:\b)CrunchBot(?:\b)" 3; + "~*(?:\b)CSHttp(?:\b)" 3; + "~*(?:\b)Curious(?:\b)" 3; + "~*(?:\b)Custo(?:\b)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; + "~*(?:\b)DataCha0s(?:\b)" 3; + "~*(?:\b)DBLBot(?:\b)" 3; + "~*(?:\b)demandbase-bot(?:\b)" 3; + "~*(?:\b)Demon(?:\b)" 3; + "~*(?:\b)Deusu(?:\b)" 3; + "~*(?:\b)Devil(?:\b)" 3; + "~*(?:\b)Digincore(?:\b)" 3; + "~*(?:\b)DigitalPebble(?:\b)" 3; + "~*(?:\b)DIIbot(?:\b)" 3; + "~*(?:\b)Dirbuster(?:\b)" 3; + "~*(?:\b)Disco(?:\b)" 3; + "~*(?:\b)Discobot(?:\b)" 3; + "~*(?:\b)Discoverybot(?:\b)" 3; + "~*(?:\b)Dispatch(?:\b)" 3; + "~*(?:\b)DittoSpyder(?:\b)" 3; + "~*(?:\b)DnyzBot(?:\b)" 3; + "~*(?:\b)DomainAppender(?:\b)" 3; + "~*(?:\b)DomainCrawler(?:\b)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)DomainStatsBot(?:\b)" 3; + "~*(?:\b)Dotbot(?:\b)" 3; + "~*(?:\b)Download\ Wonder(?:\b)" 3; + "~*(?:\b)Dragonfly(?:\b)" 3; + "~*(?:\b)Drip(?:\b)" 3; + "~*(?:\b)DSearch(?:\b)" 3; + "~*(?:\b)DTS\ Agent(?:\b)" 3; + "~*(?:\b)EasyDL(?:\b)" 3; + "~*(?:\b)Ebingbong(?:\b)" 3; + "~*(?:\b)eCatch(?:\b)" 3; + "~*(?:\b)ECCP/1.0(?:\b)" 3; + "~*(?:\b)Ecxi(?:\b)" 3; + "~*(?:\b)EirGrabber(?:\b)" 3; + "~*(?:\b)EMail\ Siphon(?:\b)" 3; + "~*(?:\b)EMail\ Wolf(?:\b)" 3; + "~*(?:\b)EroCrawler(?:\b)" 3; + "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)Evil(?:\b)" 3; + "~*(?:\b)Exabot(?:\b)" 3; + "~*(?:\b)Express\ WebPictures(?:\b)" 3; + "~*(?:\b)ExtLinksBot(?:\b)" 3; + "~*(?:\b)Extractor(?:\b)" 3; + "~*(?:\b)ExtractorPro(?:\b)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; + "~*(?:\b)EyeNetIE(?:\b)" 3; + "~*(?:\b)Ezooms(?:\b)" 3; + "~*(?:\b)facebookscraper(?:\b)" 3; + "~*(?:\b)FDM(?:\b)" 3; + "~*(?:\b)FemtosearchBot(?:\b)" 3; + "~*(?:\b)FHscan(?:\b)" 3; + "~*(?:\b)Fimap(?:\b)" 3; + "~*(?:\b)Firefox/7.0(?:\b)" 3; + "~*(?:\b)FlashGet(?:\b)" 3; + "~*(?:\b)Flunky(?:\b)" 3; + "~*(?:\b)Foobot(?:\b)" 3; + "~*(?:\b)Freeuploader(?:\b)" 3; + "~*(?:\b)FrontPage(?:\b)" 3; + "~*(?:\b)FyberSpider(?:\b)" 3; + "~*(?:\b)Fyrebot(?:\b)" 3; + "~*(?:\b)GalaxyBot(?:\b)" 3; + "~*(?:\b)Genieo(?:\b)" 3; + "~*(?:\b)GermCrawler(?:\b)" 3; + "~*(?:\b)Getintent(?:\b)" 3; + "~*(?:\b)GetRight(?:\b)" 3; + "~*(?:\b)GetWeb(?:\b)" 3; + "~*(?:\b)Gigablast(?:\b)" 3; + "~*(?:\b)Gigabot(?:\b)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; + "~*(?:\b)Gotit(?:\b)" 3; + "~*(?:\b)GoZilla(?:\b)" 3; + "~*(?:\b)Go!Zilla(?:\b)" 3; + "~*(?:\b)Grabber(?:\b)" 3; + "~*(?:\b)GrabNet(?:\b)" 3; + "~*(?:\b)Grafula(?:\b)" 3; + "~*(?:\b)GrapeFX(?:\b)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b)" 3; + "~*(?:\b)GridBot(?:\b)" 3; + "~*(?:\b)GT::WWW(?:\b)" 3; + "~*(?:\b)Haansoft(?:\b)" 3; + "~*(?:\b)HaosouSpider(?:\b)" 3; + "~*(?:\b)Harvest(?:\b)" 3; + "~*(?:\b)Havij(?:\b)" 3; + "~*(?:\b)HEADMasterSEO(?:\b)" 3; + "~*(?:\b)Heritrix(?:\b)" 3; + "~*(?:\b)Hloader(?:\b)" 3; + "~*(?:\b)HMView(?:\b)" 3; + "~*(?:\b)HTMLparser(?:\b)" 3; + "~*(?:\b)HTTP::Lite(?:\b)" 3; + "~*(?:\b)HTTrack(?:\b)" 3; + "~*(?:\b)Humanlinks(?:\b)" 3; + "~*(?:\b)HybridBot(?:\b)" 3; + "~*(?:\b)Iblog(?:\b)" 3; + "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)Id-search(?:\b)" 3; + "~*(?:\b)IlseBot(?:\b)" 3; + "~*(?:\b)Image\ Fetch(?:\b)" 3; + "~*(?:\b)Image\ Sucker(?:\b)" 3; + "~*(?:\b)IndeedBot(?:\b)" 3; + "~*(?:\b)Indy\ Library(?:\b)" 3; + "~*(?:\b)InfoNaviRobot(?:\b)" 3; + "~*(?:\b)InfoTekies(?:\b)" 3; + "~*(?:\b)instabid(?:\b)" 3; + "~*(?:\b)Intelliseek(?:\b)" 3; + "~*(?:\b)InterGET(?:\b)" 3; + "~*(?:\b)Internet\ Ninja(?:\b)" 3; + "~*(?:\b)InternetSeer(?:\b)" 3; + "~*(?:\b)internetVista\ monitor(?:\b)" 3; + "~*(?:\b)ips-agent(?:\b)" 3; + "~*(?:\b)Iria(?:\b)" 3; + "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)Iskanie(?:\b)" 3; + "~*(?:\b)IstellaBot(?:\b)" 3; + "~*(?:\b)JamesBOT(?:\b)" 3; + "~*(?:\b)Jbrofuzz(?:\b)" 3; + "~*(?:\b)JennyBot(?:\b)" 3; + "~*(?:\b)JetCar(?:\b)" 3; + "~*(?:\b)Jetty(?:\b)" 3; + "~*(?:\b)JikeSpider(?:\b)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; + "~*(?:\b)Joomla(?:\b)" 3; + "~*(?:\b)Jorgee(?:\b)" 3; + "~*(?:\b)JustView(?:\b)" 3; + "~*(?:\b)Jyxobot(?:\b)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b)" 3; + "~*(?:\b)Keyword\ Density(?:\b)" 3; + "~*(?:\b)Kozmosbot(?:\b)" 3; + "~*(?:\b)Lanshanbot(?:\b)" 3; + "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)LeechFTP(?:\b)" 3; + "~*(?:\b)LeechGet(?:\b)" 3; + "~*(?:\b)LexiBot(?:\b)" 3; + "~*(?:\b)Lftp(?:\b)" 3; + "~*(?:\b)LibWeb(?:\b)" 3; + "~*(?:\b)Libwhisker(?:\b)" 3; + "~*(?:\b)Lightspeedsystems(?:\b)" 3; + "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkdexbot(?:\b)" 3; + "~*(?:\b)LinkextractorPro(?:\b)" 3; + "~*(?:\b)LinkpadBot(?:\b)" 3; + "~*(?:\b)LinkScan(?:\b)" 3; + "~*(?:\b)LinksManager(?:\b)" 3; + "~*(?:\b)LinkWalker(?:\b)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; + "~*(?:\b)Lipperhey(?:\b)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; + "~*(?:\b)Litemage_walker(?:\b)" 3; + "~*(?:\b)Lmspider(?:\b)" 3; + "~*(?:\b)LNSpiderguy(?:\b)" 3; + "~*(?:\b)Ltx71(?:\b)" 3; + "~*(?:\b)lwp-request(?:\b)" 3; + "~*(?:\b)LWP::Simple(?:\b)" 3; + "~*(?:\b)lwp-trivial(?:\b)" 3; + "~*(?:\b)Magnet(?:\b)" 3; + "~*(?:\b)Mag-Net(?:\b)" 3; + "~*(?:\b)magpie-crawler(?:\b)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b)" 3; + "~*(?:\b)Majestic12(?:\b)" 3; + "~*(?:\b)Majestic-SEO(?:\b)" 3; + "~*(?:\b)Majestic\ SEO(?:\b)" 3; + "~*(?:\b)MarkMonitor(?:\b)" 3; + "~*(?:\b)MarkWatch(?:\b)" 3; + "~*(?:\b)Masscan(?:\b)" 3; + "~*(?:\b)Mass\ Downloader(?:\b)" 3; + "~*(?:\b)Mata\ Hari(?:\b)" 3; + "~*(?:\b)MauiBot(?:\b)" 3; + "~*(?:\b)meanpathbot(?:\b)" 3; + "~*(?:\b)Meanpathbot(?:\b)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b)" 3; + "~*(?:\b)mediawords(?:\b)" 3; + "~*(?:\b)MegaIndex.ru(?:\b)" 3; + "~*(?:\b)Metauri(?:\b)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; + "~*(?:\b)MIDown\ tool(?:\b)" 3; + "~*(?:\b)MIIxpc(?:\b)" 3; + "~*(?:\b)Mister\ PiX(?:\b)" 3; + "~*(?:\b)MJ12bot(?:\b)" 3; + "~*(?:\b)Mojeek(?:\b)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; + "~*(?:\b)Mr.4x3(?:\b)" 3; + "~*(?:\b)MSFrontPage(?:\b)" 3; + "~*(?:\b)MSIECrawler(?:\b)" 3; + "~*(?:\b)Msrabot(?:\b)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; + "~*(?:\b)muhstik-scan(?:\b)" 3; + "~*(?:\b)Musobot(?:\b)" 3; + "~*(?:\b)Name\ Intelligence(?:\b)" 3; + "~*(?:\b)Nameprotect(?:\b)" 3; + "~*(?:\b)Navroad(?:\b)" 3; + "~*(?:\b)NearSite(?:\b)" 3; + "~*(?:\b)Needle(?:\b)" 3; + "~*(?:\b)Nessus(?:\b)" 3; + "~*(?:\b)NetAnts(?:\b)" 3; + "~*(?:\b)Netcraft(?:\b)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; + "~*(?:\b)NetLyzer(?:\b)" 3; + "~*(?:\b)NetMechanic(?:\b)" 3; + "~*(?:\b)NetSpider(?:\b)" 3; + "~*(?:\b)Nettrack(?:\b)" 3; + "~*(?:\b)Net\ Vampire(?:\b)" 3; + "~*(?:\b)Netvibes(?:\b)" 3; + "~*(?:\b)NetZIP(?:\b)" 3; + "~*(?:\b)NextGenSearchBot(?:\b)" 3; + "~*(?:\b)Nibbler(?:\b)" 3; + "~*(?:\b)NICErsPRO(?:\b)" 3; + "~*(?:\b)Niki-bot(?:\b)" 3; + "~*(?:\b)Nikto(?:\b)" 3; + "~*(?:\b)NimbleCrawler(?:\b)" 3; + "~*(?:\b)Nimbostratus(?:\b)" 3; + "~*(?:\b)Ninja(?:\b)" 3; + "~*(?:\b)Nmap(?:\b)" 3; + "~*(?:\b)NPbot(?:\b)" 3; + "~*(?:\b)Nutch(?:\b)" 3; + "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)Octopus(?:\b)" 3; + "~*(?:\b)Offline\ Explorer(?:\b)" 3; + "~*(?:\b)Offline\ Navigator(?:\b)" 3; + "~*(?:\b)OnCrawl(?:\b)" 3; + "~*(?:\b)Openfind(?:\b)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b)" 3; + "~*(?:\b)Openvas(?:\b)" 3; + "~*(?:\b)OpenVAS(?:\b)" 3; + "~*(?:\b)OrangeBot(?:\b)" 3; + "~*(?:\b)OrangeSpider(?:\b)" 3; + "~*(?:\b)OutclicksBot(?:\b)" 3; + "~*(?:\b)OutfoxBot(?:\b)" 3; + "~*(?:\b)PageAnalyzer(?:\b)" 3; + "~*(?:\b)Page\ Analyzer(?:\b)" 3; + "~*(?:\b)PageGrabber(?:\b)" 3; + "~*(?:\b)page\ scorer(?:\b)" 3; + "~*(?:\b)PageScorer(?:\b)" 3; + "~*(?:\b)Pandalytics(?:\b)" 3; + "~*(?:\b)Panscient(?:\b)" 3; + "~*(?:\b)Papa\ Foto(?:\b)" 3; + "~*(?:\b)Pavuk(?:\b)" 3; + "~*(?:\b)pcBrowser(?:\b)" 3; + "~*(?:\b)PECL::HTTP(?:\b)" 3; + "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)PHPCrawl(?:\b)" 3; + "~*(?:\b)Picscout(?:\b)" 3; + "~*(?:\b)Picsearch(?:\b)" 3; + "~*(?:\b)PictureFinder(?:\b)" 3; + "~*(?:\b)Pimonster(?:\b)" 3; + "~*(?:\b)Pi-Monster(?:\b)" 3; + "~*(?:\b)Pixray(?:\b)" 3; + "~*(?:\b)PleaseCrawl(?:\b)" 3; + "~*(?:\b)plumanalytics(?:\b)" 3; + "~*(?:\b)Pockey(?:\b)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; + "~*(?:\b)Probethenet(?:\b)" 3; + "~*(?:\b)ProPowerBot(?:\b)" 3; + "~*(?:\b)ProWebWalker(?:\b)" 3; + "~*(?:\b)Psbot(?:\b)" 3; + "~*(?:\b)Pump(?:\b)" 3; + "~*(?:\b)PxBroker(?:\b)" 3; + "~*(?:\b)PyCurl(?:\b)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; + "~*(?:\b)Quick-Crawler(?:\b)" 3; + "~*(?:\b)RankActive(?:\b)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b)" 3; + "~*(?:\b)RankFlex(?:\b)" 3; + "~*(?:\b)RankingBot(?:\b)" 3; + "~*(?:\b)RankingBot2(?:\b)" 3; + "~*(?:\b)Rankivabot(?:\b)" 3; + "~*(?:\b)RankurBot(?:\b)" 3; + "~*(?:\b)RealDownload(?:\b)" 3; + "~*(?:\b)Reaper(?:\b)" 3; + "~*(?:\b)RebelMouse(?:\b)" 3; + "~*(?:\b)Recorder(?:\b)" 3; + "~*(?:\b)RedesScrapy(?:\b)" 3; + "~*(?:\b)ReGet(?:\b)" 3; + "~*(?:\b)RepoMonkey(?:\b)" 3; + "~*(?:\b)Ripper(?:\b)" 3; + "~*(?:\b)RocketCrawler(?:\b)" 3; + "~*(?:\b)Rogerbot(?:\b)" 3; + "~*(?:\b)s1z.ru(?:\b)" 3; + "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)SBIder(?:\b)" 3; + "~*(?:\b)ScanAlert(?:\b)" 3; + "~*(?:\b)Scanbot(?:\b)" 3; + "~*(?:\b)scan.lol(?:\b)" 3; + "~*(?:\b)ScoutJet(?:\b)" 3; + "~*(?:\b)Scrapy(?:\b)" 3; + "~*(?:\b)Screaming(?:\b)" 3; + "~*(?:\b)ScreenerBot(?:\b)" 3; + "~*(?:\b)Searchestate(?:\b)" 3; + "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Semrush(?:\b)" 3; + "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SEOkicks(?:\b)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; + "~*(?:\b)Seomoz(?:\b)" 3; + "~*(?:\b)SEOprofiler(?:\b)" 3; + "~*(?:\b)seoscanners(?:\b)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b)" 3; + "~*(?:\b)SEOstats(?:\b)" 3; + "~*(?:\b)serpstatbot(?:\b)" 3; + "~*(?:\b)sexsearcher(?:\b)" 3; + "~*(?:\b)Shodan(?:\b)" 3; + "~*(?:\b)Siphon(?:\b)" 3; + "~*(?:\b)SISTRIX(?:\b)" 3; + "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteExplorer(?:\b)" 3; + "~*(?:\b)Siteimprove(?:\b)" 3; + "~*(?:\b)SiteLockSpider(?:\b)" 3; + "~*(?:\b)SiteSnagger(?:\b)" 3; + "~*(?:\b)SiteSucker(?:\b)" 3; + "~*(?:\b)Site\ Sucker(?:\b)" 3; + "~*(?:\b)Sitevigil(?:\b)" 3; + "~*(?:\b)SlySearch(?:\b)" 3; + "~*(?:\b)SmartDownload(?:\b)" 3; + "~*(?:\b)SMTBot(?:\b)" 3; + "~*(?:\b)Snake(?:\b)" 3; + "~*(?:\b)Snapbot(?:\b)" 3; + "~*(?:\b)Snoopy(?:\b)" 3; + "~*(?:\b)SocialRankIOBot(?:\b)" 3; + "~*(?:\b)Sociscraper(?:\b)" 3; + "~*(?:\b)sogouspider(?:\b)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; + "~*(?:\b)Sosospider(?:\b)" 3; + "~*(?:\b)Sottopop(?:\b)" 3; + "~*(?:\b)SpaceBison(?:\b)" 3; + "~*(?:\b)Spammen(?:\b)" 3; + "~*(?:\b)SpankBot(?:\b)" 3; + "~*(?:\b)Spanner(?:\b)" 3; + "~*(?:\b)sp_auditbot(?:\b)" 3; + "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spinn3r(?:\b)" 3; + "~*(?:\b)SputnikBot(?:\b)" 3; + "~*(?:\b)spyfu(?:\b)" 3; + "~*(?:\b)Sqlmap(?:\b)" 3; + "~*(?:\b)Sqlworm(?:\b)" 3; + "~*(?:\b)Sqworm(?:\b)" 3; + "~*(?:\b)Steeler(?:\b)" 3; + "~*(?:\b)Stripper(?:\b)" 3; + "~*(?:\b)Sucker(?:\b)" 3; + "~*(?:\b)Sucuri(?:\b)" 3; + "~*(?:\b)SuperBot(?:\b)" 3; + "~*(?:\b)SuperHTTP(?:\b)" 3; + "~*(?:\b)Surfbot(?:\b)" 3; + "~*(?:\b)SurveyBot(?:\b)" 3; + "~*(?:\b)Suzuran(?:\b)" 3; + "~*(?:\b)Swiftbot(?:\b)" 3; + "~*(?:\b)sysscan(?:\b)" 3; + "~*(?:\b)Szukacz(?:\b)" 3; + "~*(?:\b)T0PHackTeam(?:\b)" 3; + "~*(?:\b)T8Abot(?:\b)" 3; + "~*(?:\b)tAkeOut(?:\b)" 3; + "~*(?:\b)Teleport(?:\b)" 3; + "~*(?:\b)TeleportPro(?:\b)" 3; + "~*(?:\b)Telesoft(?:\b)" 3; + "~*(?:\b)Telesphoreo(?:\b)" 3; + "~*(?:\b)Telesphorep(?:\b)" 3; + "~*(?:\b)The\ Intraformant(?:\b)" 3; + "~*(?:\b)TheNomad(?:\b)" 3; + "~*(?:\b)Thumbor(?:\b)" 3; + "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)Titan(?:\b)" 3; + "~*(?:\b)Toata(?:\b)" 3; + "~*(?:\b)Toweyabot(?:\b)" 3; + "~*(?:\b)Tracemyfile(?:\b)" 3; + "~*(?:\b)Trendiction(?:\b)" 3; + "~*(?:\b)Trendictionbot(?:\b)" 3; + "~*(?:\b)trendiction.com(?:\b)" 3; + "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)True_Robot(?:\b)" 3; + "~*(?:\b)Turingos(?:\b)" 3; + "~*(?:\b)Turnitin(?:\b)" 3; + "~*(?:\b)TurnitinBot(?:\b)" 3; + "~*(?:\b)TwengaBot(?:\b)" 3; + "~*(?:\b)Twice(?:\b)" 3; + "~*(?:\b)Typhoeus(?:\b)" 3; + "~*(?:\b)UnisterBot(?:\b)" 3; + "~*(?:\b)Upflow(?:\b)" 3; + "~*(?:\b)URLy.Warning(?:\b)" 3; + "~*(?:\b)URLy\ Warning(?:\b)" 3; + "~*(?:\b)Vacuum(?:\b)" 3; + "~*(?:\b)Vagabondo(?:\b)" 3; + "~*(?:\b)VB\ Project(?:\b)" 3; + "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b)" 3; + "~*(?:\b)VidibleScraper(?:\b)" 3; + "~*(?:\b)Virusdie(?:\b)" 3; + "~*(?:\b)VoidEYE(?:\b)" 3; + "~*(?:\b)Voil(?:\b)" 3; + "~*(?:\b)Voltron(?:\b)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b)" 3; + "~*(?:\b)WallpapersHD(?:\b)" 3; + "~*(?:\b)WASALive-Bot(?:\b)" 3; + "~*(?:\b)WBSearchBot(?:\b)" 3; + "~*(?:\b)Webalta(?:\b)" 3; + "~*(?:\b)WebAuto(?:\b)" 3; + "~*(?:\b)Web\ Auto(?:\b)" 3; + "~*(?:\b)WebBandit(?:\b)" 3; + "~*(?:\b)WebCollage(?:\b)" 3; + "~*(?:\b)Web\ Collage(?:\b)" 3; + "~*(?:\b)WebCopier(?:\b)" 3; + "~*(?:\b)WEBDAV(?:\b)" 3; + "~*(?:\b)WebEnhancer(?:\b)" 3; + "~*(?:\b)Web\ Enhancer(?:\b)" 3; + "~*(?:\b)WebFetch(?:\b)" 3; + "~*(?:\b)Web\ Fetch(?:\b)" 3; + "~*(?:\b)WebFuck(?:\b)" 3; + "~*(?:\b)Web\ Fuck(?:\b)" 3; + "~*(?:\b)WebGo\ IS(?:\b)" 3; + "~*(?:\b)WebImageCollector(?:\b)" 3; + "~*(?:\b)WebLeacher(?:\b)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; + "~*(?:\b)webmeup-crawler(?:\b)" 3; + "~*(?:\b)WebPix(?:\b)" 3; + "~*(?:\b)Web\ Pix(?:\b)" 3; + "~*(?:\b)WebReaper(?:\b)" 3; + "~*(?:\b)WebSauger(?:\b)" 3; + "~*(?:\b)Web\ Sauger(?:\b)" 3; + "~*(?:\b)Webshag(?:\b)" 3; + "~*(?:\b)WebsiteExtractor(?:\b)" 3; + "~*(?:\b)WebsiteQuester(?:\b)" 3; + "~*(?:\b)Website\ Quester(?:\b)" 3; + "~*(?:\b)Webster(?:\b)" 3; + "~*(?:\b)WebStripper(?:\b)" 3; + "~*(?:\b)WebSucker(?:\b)" 3; + "~*(?:\b)Web\ Sucker(?:\b)" 3; + "~*(?:\b)WebWhacker(?:\b)" 3; + "~*(?:\b)WebZIP(?:\b)" 3; + "~*(?:\b)WeSEE(?:\b)" 3; + "~*(?:\b)Whack(?:\b)" 3; + "~*(?:\b)Whacker(?:\b)" 3; + "~*(?:\b)Whatweb(?:\b)" 3; + "~*(?:\b)Who.is\ Bot(?:\b)" 3; + "~*(?:\b)Widow(?:\b)" 3; + "~*(?:\b)WinHTTrack(?:\b)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; + "~*(?:\b)WISENutbot(?:\b)" 3; + "~*(?:\b)Wonderbot(?:\b)" 3; + "~*(?:\b)Woobot(?:\b)" 3; + "~*(?:\b)Wotbox(?:\b)" 3; + "~*(?:\b)Wprecon(?:\b)" 3; + "~*(?:\b)WPScan(?:\b)" 3; + "~*(?:\b)WWW-Collector-E(?:\b)" 3; + "~*(?:\b)WWW-Mechanize(?:\b)" 3; + "~*(?:\b)WWW::Mechanize(?:\b)" 3; + "~*(?:\b)WWWOFFLE(?:\b)" 3; + "~*(?:\b)x09Mozilla(?:\b)" 3; + "~*(?:\b)x22Mozilla(?:\b)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; + "~*(?:\b)Xenu(?:\b)" 3; + "~*(?:\b)xpymep1.exe(?:\b)" 3; + "~*(?:\b)YoudaoBot(?:\b)" 3; + "~*(?:\b)Zade(?:\b)" 3; + "~*(?:\b)Zauba(?:\b)" 3; + "~*(?:\b)zauba.io(?:\b)" 3; + "~*(?:\b)Zermelo(?:\b)" 3; + "~*(?:\b)Zeus(?:\b)" 3; + "~*(?:\b)zgrab(?:\b)" 3; + "~*(?:\b)Zitebot(?:\b)" 3; + "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZumBot(?:\b)" 3; + "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------- @@ -717,38 +717,38 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)adidxbot(?:\b|)" 0; - "~*(?:\b)AdsBot-Google(?:\b|)" 0; - "~*(?:\b)aolbuild(?:\b|)" 0; - "~*(?:\b)bingbot(?:\b|)" 0; - "~*(?:\b)bingpreview(?:\b|)" 0; - "~*(?:\b)DoCoMo(?:\b|)" 0; - "~*(?:\b)duckduckgo(?:\b|)" 0; - "~*(?:\b)facebookexternalhit(?:\b|)" 0; - "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; - "~*(?:\b)Googlebot(?:\b|)" 0; - "~*(?:\b)Googlebot-Image(?:\b|)" 0; - "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; - "~*(?:\b)Googlebot-News(?:\b|)" 0; - "~*(?:\b)Googlebot/Test(?:\b|)" 0; - "~*(?:\b)Googlebot-Video(?:\b|)" 0; - "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; - "~*(?:\b)Gravityscan(?:\b|)" 0; - "~*(?:\b)gsa-crawler(?:\b|)" 0; - "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; - "~*(?:\b)Kraken/0.1(?:\b|)" 0; - "~*(?:\b)LinkedInBot(?:\b|)" 0; - "~*(?:\b)Mediapartners-Google(?:\b|)" 0; - "~*(?:\b)msnbot(?:\b|)" 0; - "~*(?:\b)msnbot-media(?:\b|)" 0; - "~*(?:\b)SAMSUNG(?:\b|)" 0; - "~*(?:\b)Slackbot(?:\b|)" 0; - "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; - "~*(?:\b)slurp(?:\b|)" 0; - "~*(?:\b)teoma(?:\b|)" 0; - "~*(?:\b)TwitterBot(?:\b|)" 0; - "~*(?:\b)Wordpress(?:\b|)" 0; - "~*(?:\b)yahoo(?:\b|)" 0; + "~*(?:\b)adidxbot(?:\b)" 0; + "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bingbot(?:\b)" 0; + "~*(?:\b)bingpreview(?:\b)" 0; + "~*(?:\b)DoCoMo(?:\b)" 0; + "~*(?:\b)duckduckgo(?:\b)" 0; + "~*(?:\b)facebookexternalhit(?:\b)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b)" 0; + "~*(?:\b)Googlebot(?:\b)" 0; + "~*(?:\b)Googlebot-Image(?:\b)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b)" 0; + "~*(?:\b)Googlebot-News(?:\b)" 0; + "~*(?:\b)Googlebot/Test(?:\b)" 0; + "~*(?:\b)Googlebot-Video(?:\b)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; + "~*(?:\b)Gravityscan(?:\b)" 0; + "~*(?:\b)gsa-crawler(?:\b)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b)" 0; + "~*(?:\b)Kraken/0.1(?:\b)" 0; + "~*(?:\b)LinkedInBot(?:\b)" 0; + "~*(?:\b)Mediapartners-Google(?:\b)" 0; + "~*(?:\b)msnbot(?:\b)" 0; + "~*(?:\b)msnbot-media(?:\b)" 0; + "~*(?:\b)SAMSUNG(?:\b)" 0; + "~*(?:\b)Slackbot(?:\b)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 0; + "~*(?:\b)slurp(?:\b)" 0; + "~*(?:\b)teoma(?:\b)" 0; + "~*(?:\b)TwitterBot(?:\b)" 0; + "~*(?:\b)Wordpress(?:\b)" 0; + "~*(?:\b)yahoo(?:\b)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------------------- @@ -760,12 +760,12 @@ map $http_user_agent $bad_bot { #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)jetmon(?:\b|)" 1; - "~*(?:\b)libwww-perl(?:\b|)" 1; - "~*(?:\b)Lynx(?:\b|)" 1; - "~*(?:\b)munin(?:\b|)" 1; - "~*(?:\b)Presto(?:\b|)" 1; - "~*(?:\b)Wget/1.15(?:\b|)" 1; + "~*(?:\b)jetmon(?:\b)" 1; + "~*(?:\b)libwww-perl(?:\b)" 1; + "~*(?:\b)Lynx(?:\b)" 1; + "~*(?:\b)munin(?:\b)" 1; + "~*(?:\b)Presto(?:\b)" 1; + "~*(?:\b)Wget/1.15(?:\b)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ------------------------------------------------------- @@ -778,23 +778,23 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)Alexa(?:\b|)" 2; - "~*(?:\b)archive.org(?:\b|)" 2; - "~*(?:\b)Baidu(?:\b|)" 2; - "~*(?:\b)BUbiNG(?:\b|)" 2; - "~*(?:\b)FlipboardProxy(?:\b|)" 2; - "~*(?:\b)ia_archiver(?:\b|)" 2; - "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; - "~*(?:\b)Proximic(?:\b|)" 2; - "~*(?:\b)R6_CommentReader(?:\b|)" 2; - "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; - "~*(?:\b)RED/1(?:\b|)" 2; - "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; - "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; - "~*(?:\b)Spaidu(?:\b|)" 2; - "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; - "~*(?:\b)YandexBot(?:\b|)" 2; - "~*(?:\b)YandexImages(?:\b|)" 2; + "~*(?:\b)Alexa(?:\b)" 2; + "~*(?:\b)archive.org(?:\b)" 2; + "~*(?:\b)Baidu(?:\b)" 2; + "~*(?:\b)BUbiNG(?:\b)" 2; + "~*(?:\b)FlipboardProxy(?:\b)" 2; + "~*(?:\b)ia_archiver(?:\b)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b)" 2; + "~*(?:\b)Proximic(?:\b)" 2; + "~*(?:\b)R6_CommentReader(?:\b)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b)" 2; + "~*(?:\b)RED/1(?:\b)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b)" 2; + "~*(?:\b)Spaidu(?:\b)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b)" 2; + "~*(?:\b)YandexBot(?:\b)" 2; + "~*(?:\b)YandexImages(?:\b)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 2e2965ffc..12f18b91b 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.188.73.34 1; +35.202.145.110 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index d3d331ed4..c3c284efc 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.188.73.34 0; +35.202.145.110 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 7b00caacd..65f28ba04 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1704 -### Updated: Sat Jun 29 21:12:07 SAST 2019 +### Version: V3.2019.07.1705 +### Updated: Mon Jul 1 08:42:46 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### @@ -147,562 +147,562 @@ map $http_user_agent $bad_bot { # -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b|)" 3; - "~*(?:\b)404checker(?:\b|)" 3; - "~*(?:\b)404enemy(?:\b|)" 3; - "~*(?:\b)80legs(?:\b|)" 3; - "~*(?:\b)Abonti(?:\b|)" 3; - "~*(?:\b)Aboundex(?:\b|)" 3; - "~*(?:\b)Aboundexbot(?:\b|)" 3; - "~*(?:\b)Acunetix(?:\b|)" 3; - "~*(?:\b)ADmantX(?:\b|)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; - "~*(?:\b)AhrefsBot(?:\b|)" 3; - "~*(?:\b)AIBOT(?:\b|)" 3; - "~*(?:\b)AiHitBot(?:\b|)" 3; - "~*(?:\b)Aipbot(?:\b|)" 3; - "~*(?:\b)Alexibot(?:\b|)" 3; - "~*(?:\b)Alligator(?:\b|)" 3; - "~*(?:\b)AllSubmitter(?:\b|)" 3; - "~*(?:\b)AlphaBot(?:\b|)" 3; - "~*(?:\b)Anarchie(?:\b|)" 3; - "~*(?:\b)Apexoo(?:\b|)" 3; - "~*(?:\b)archive.org_bot(?:\b|)" 3; - "~*(?:\b)ASPSeek(?:\b|)" 3; - "~*(?:\b)Asterias(?:\b|)" 3; - "~*(?:\b)Attach(?:\b|)" 3; - "~*(?:\b)autoemailspider(?:\b|)" 3; - "~*(?:\b)BackDoorBot(?:\b|)" 3; - "~*(?:\b)Backlink-Ceck(?:\b|)" 3; - "~*(?:\b)backlink-check(?:\b|)" 3; - "~*(?:\b)BacklinkCrawler(?:\b|)" 3; - "~*(?:\b)BackStreet(?:\b|)" 3; - "~*(?:\b)BackWeb(?:\b|)" 3; - "~*(?:\b)Badass(?:\b|)" 3; - "~*(?:\b)Bandit(?:\b|)" 3; - "~*(?:\b)Barkrowler(?:\b|)" 3; - "~*(?:\b)BatchFTP(?:\b|)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; - "~*(?:\b)BBBike(?:\b|)" 3; - "~*(?:\b)BDCbot(?:\b|)" 3; - "~*(?:\b)BDFetch(?:\b|)" 3; - "~*(?:\b)BetaBot(?:\b|)" 3; - "~*(?:\b)Bigfoot(?:\b|)" 3; - "~*(?:\b)Bitacle(?:\b|)" 3; - "~*(?:\b)Blackboard(?:\b|)" 3; - "~*(?:\b)Black\ Hole(?:\b|)" 3; - "~*(?:\b)BlackWidow(?:\b|)" 3; - "~*(?:\b)BLEXBot(?:\b|)" 3; - "~*(?:\b)Blow(?:\b|)" 3; - "~*(?:\b)BlowFish(?:\b|)" 3; - "~*(?:\b)Boardreader(?:\b|)" 3; - "~*(?:\b)Bolt(?:\b|)" 3; - "~*(?:\b)BotALot(?:\b|)" 3; - "~*(?:\b)Brandprotect(?:\b|)" 3; - "~*(?:\b)Brandwatch(?:\b|)" 3; - "~*(?:\b)Buddy(?:\b|)" 3; - "~*(?:\b)BuiltBotTough(?:\b|)" 3; - "~*(?:\b)BuiltWith(?:\b|)" 3; - "~*(?:\b)Bullseye(?:\b|)" 3; - "~*(?:\b)BunnySlippers(?:\b|)" 3; - "~*(?:\b)BuzzSumo(?:\b|)" 3; - "~*(?:\b)Calculon(?:\b|)" 3; - "~*(?:\b)CATExplorador(?:\b|)" 3; - "~*(?:\b)CazoodleBot(?:\b|)" 3; - "~*(?:\b)CCBot(?:\b|)" 3; - "~*(?:\b)Cegbfeieh(?:\b|)" 3; - "~*(?:\b)CheeseBot(?:\b|)" 3; - "~*(?:\b)CherryPicker(?:\b|)" 3; - "~*(?:\b)CheTeam(?:\b|)" 3; - "~*(?:\b)ChinaClaw(?:\b|)" 3; - "~*(?:\b)Chlooe(?:\b|)" 3; - "~*(?:\b)Claritybot(?:\b|)" 3; - "~*(?:\b)Cliqzbot(?:\b|)" 3; - "~*(?:\b)Cloud\ mapping(?:\b|)" 3; - "~*(?:\b)coccocbot-web(?:\b|)" 3; - "~*(?:\b)Cogentbot(?:\b|)" 3; - "~*(?:\b)cognitiveseo(?:\b|)" 3; - "~*(?:\b)Collector(?:\b|)" 3; - "~*(?:\b)com.plumanalytics(?:\b|)" 3; - "~*(?:\b)Copier(?:\b|)" 3; - "~*(?:\b)CopyRightCheck(?:\b|)" 3; - "~*(?:\b)Copyscape(?:\b|)" 3; - "~*(?:\b)Cosmos(?:\b|)" 3; - "~*(?:\b)Craftbot(?:\b|)" 3; - "~*(?:\b)crawler4j(?:\b|)" 3; - "~*(?:\b)crawler.feedback(?:\b|)" 3; - "~*(?:\b)crawl.sogou.com(?:\b|)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; - "~*(?:\b)Crescent(?:\b|)" 3; - "~*(?:\b)CrunchBot(?:\b|)" 3; - "~*(?:\b)CSHttp(?:\b|)" 3; - "~*(?:\b)Curious(?:\b|)" 3; - "~*(?:\b)Custo(?:\b|)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; - "~*(?:\b)DataCha0s(?:\b|)" 3; - "~*(?:\b)DBLBot(?:\b|)" 3; - "~*(?:\b)demandbase-bot(?:\b|)" 3; - "~*(?:\b)Demon(?:\b|)" 3; - "~*(?:\b)Deusu(?:\b|)" 3; - "~*(?:\b)Devil(?:\b|)" 3; - "~*(?:\b)Digincore(?:\b|)" 3; - "~*(?:\b)DigitalPebble(?:\b|)" 3; - "~*(?:\b)DIIbot(?:\b|)" 3; - "~*(?:\b)Dirbuster(?:\b|)" 3; - "~*(?:\b)Disco(?:\b|)" 3; - "~*(?:\b)Discobot(?:\b|)" 3; - "~*(?:\b)Discoverybot(?:\b|)" 3; - "~*(?:\b)Dispatch(?:\b|)" 3; - "~*(?:\b)DittoSpyder(?:\b|)" 3; - "~*(?:\b)DnyzBot(?:\b|)" 3; - "~*(?:\b)DomainAppender(?:\b|)" 3; - "~*(?:\b)DomainCrawler(?:\b|)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; - "~*(?:\b)DomainStatsBot(?:\b|)" 3; - "~*(?:\b)Dotbot(?:\b|)" 3; - "~*(?:\b)Download\ Wonder(?:\b|)" 3; - "~*(?:\b)Dragonfly(?:\b|)" 3; - "~*(?:\b)Drip(?:\b|)" 3; - "~*(?:\b)DSearch(?:\b|)" 3; - "~*(?:\b)DTS\ Agent(?:\b|)" 3; - "~*(?:\b)EasyDL(?:\b|)" 3; - "~*(?:\b)Ebingbong(?:\b|)" 3; - "~*(?:\b)eCatch(?:\b|)" 3; - "~*(?:\b)ECCP/1.0(?:\b|)" 3; - "~*(?:\b)Ecxi(?:\b|)" 3; - "~*(?:\b)EirGrabber(?:\b|)" 3; - "~*(?:\b)EMail\ Siphon(?:\b|)" 3; - "~*(?:\b)EMail\ Wolf(?:\b|)" 3; - "~*(?:\b)EroCrawler(?:\b|)" 3; - "~*(?:\b)evc-batch(?:\b|)" 3; - "~*(?:\b)Evil(?:\b|)" 3; - "~*(?:\b)Exabot(?:\b|)" 3; - "~*(?:\b)Express\ WebPictures(?:\b|)" 3; - "~*(?:\b)ExtLinksBot(?:\b|)" 3; - "~*(?:\b)Extractor(?:\b|)" 3; - "~*(?:\b)ExtractorPro(?:\b|)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; - "~*(?:\b)EyeNetIE(?:\b|)" 3; - "~*(?:\b)Ezooms(?:\b|)" 3; - "~*(?:\b)facebookscraper(?:\b|)" 3; - "~*(?:\b)FDM(?:\b|)" 3; - "~*(?:\b)FemtosearchBot(?:\b|)" 3; - "~*(?:\b)FHscan(?:\b|)" 3; - "~*(?:\b)Fimap(?:\b|)" 3; - "~*(?:\b)Firefox/7.0(?:\b|)" 3; - "~*(?:\b)FlashGet(?:\b|)" 3; - "~*(?:\b)Flunky(?:\b|)" 3; - "~*(?:\b)Foobot(?:\b|)" 3; - "~*(?:\b)Freeuploader(?:\b|)" 3; - "~*(?:\b)FrontPage(?:\b|)" 3; - "~*(?:\b)FyberSpider(?:\b|)" 3; - "~*(?:\b)Fyrebot(?:\b|)" 3; - "~*(?:\b)GalaxyBot(?:\b|)" 3; - "~*(?:\b)Genieo(?:\b|)" 3; - "~*(?:\b)GermCrawler(?:\b|)" 3; - "~*(?:\b)Getintent(?:\b|)" 3; - "~*(?:\b)GetRight(?:\b|)" 3; - "~*(?:\b)GetWeb(?:\b|)" 3; - "~*(?:\b)Gigablast(?:\b|)" 3; - "~*(?:\b)Gigabot(?:\b|)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; - "~*(?:\b)Gotit(?:\b|)" 3; - "~*(?:\b)GoZilla(?:\b|)" 3; - "~*(?:\b)Go!Zilla(?:\b|)" 3; - "~*(?:\b)Grabber(?:\b|)" 3; - "~*(?:\b)GrabNet(?:\b|)" 3; - "~*(?:\b)Grafula(?:\b|)" 3; - "~*(?:\b)GrapeFX(?:\b|)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; - "~*(?:\b)GridBot(?:\b|)" 3; - "~*(?:\b)GT::WWW(?:\b|)" 3; - "~*(?:\b)Haansoft(?:\b|)" 3; - "~*(?:\b)HaosouSpider(?:\b|)" 3; - "~*(?:\b)Harvest(?:\b|)" 3; - "~*(?:\b)Havij(?:\b|)" 3; - "~*(?:\b)HEADMasterSEO(?:\b|)" 3; - "~*(?:\b)Heritrix(?:\b|)" 3; - "~*(?:\b)Hloader(?:\b|)" 3; - "~*(?:\b)HMView(?:\b|)" 3; - "~*(?:\b)HTMLparser(?:\b|)" 3; - "~*(?:\b)HTTP::Lite(?:\b|)" 3; - "~*(?:\b)HTTrack(?:\b|)" 3; - "~*(?:\b)Humanlinks(?:\b|)" 3; - "~*(?:\b)HybridBot(?:\b|)" 3; - "~*(?:\b)Iblog(?:\b|)" 3; - "~*(?:\b)IDBot(?:\b|)" 3; - "~*(?:\b)Id-search(?:\b|)" 3; - "~*(?:\b)IlseBot(?:\b|)" 3; - "~*(?:\b)Image\ Fetch(?:\b|)" 3; - "~*(?:\b)Image\ Sucker(?:\b|)" 3; - "~*(?:\b)IndeedBot(?:\b|)" 3; - "~*(?:\b)Indy\ Library(?:\b|)" 3; - "~*(?:\b)InfoNaviRobot(?:\b|)" 3; - "~*(?:\b)InfoTekies(?:\b|)" 3; - "~*(?:\b)instabid(?:\b|)" 3; - "~*(?:\b)Intelliseek(?:\b|)" 3; - "~*(?:\b)InterGET(?:\b|)" 3; - "~*(?:\b)Internet\ Ninja(?:\b|)" 3; - "~*(?:\b)InternetSeer(?:\b|)" 3; - "~*(?:\b)internetVista\ monitor(?:\b|)" 3; - "~*(?:\b)ips-agent(?:\b|)" 3; - "~*(?:\b)Iria(?:\b|)" 3; - "~*(?:\b)IRLbot(?:\b|)" 3; - "~*(?:\b)Iskanie(?:\b|)" 3; - "~*(?:\b)IstellaBot(?:\b|)" 3; - "~*(?:\b)JamesBOT(?:\b|)" 3; - "~*(?:\b)Jbrofuzz(?:\b|)" 3; - "~*(?:\b)JennyBot(?:\b|)" 3; - "~*(?:\b)JetCar(?:\b|)" 3; - "~*(?:\b)Jetty(?:\b|)" 3; - "~*(?:\b)JikeSpider(?:\b|)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; - "~*(?:\b)Joomla(?:\b|)" 3; - "~*(?:\b)Jorgee(?:\b|)" 3; - "~*(?:\b)JustView(?:\b|)" 3; - "~*(?:\b)Jyxobot(?:\b|)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; - "~*(?:\b)Keyword\ Density(?:\b|)" 3; - "~*(?:\b)Kozmosbot(?:\b|)" 3; - "~*(?:\b)Lanshanbot(?:\b|)" 3; - "~*(?:\b)Larbin(?:\b|)" 3; - "~*(?:\b)LeechFTP(?:\b|)" 3; - "~*(?:\b)LeechGet(?:\b|)" 3; - "~*(?:\b)LexiBot(?:\b|)" 3; - "~*(?:\b)Lftp(?:\b|)" 3; - "~*(?:\b)LibWeb(?:\b|)" 3; - "~*(?:\b)Libwhisker(?:\b|)" 3; - "~*(?:\b)Lightspeedsystems(?:\b|)" 3; - "~*(?:\b)Likse(?:\b|)" 3; - "~*(?:\b)Linkdexbot(?:\b|)" 3; - "~*(?:\b)LinkextractorPro(?:\b|)" 3; - "~*(?:\b)LinkpadBot(?:\b|)" 3; - "~*(?:\b)LinkScan(?:\b|)" 3; - "~*(?:\b)LinksManager(?:\b|)" 3; - "~*(?:\b)LinkWalker(?:\b|)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; - "~*(?:\b)Lipperhey(?:\b|)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; - "~*(?:\b)Litemage_walker(?:\b|)" 3; - "~*(?:\b)Lmspider(?:\b|)" 3; - "~*(?:\b)LNSpiderguy(?:\b|)" 3; - "~*(?:\b)Ltx71(?:\b|)" 3; - "~*(?:\b)lwp-request(?:\b|)" 3; - "~*(?:\b)LWP::Simple(?:\b|)" 3; - "~*(?:\b)lwp-trivial(?:\b|)" 3; - "~*(?:\b)Magnet(?:\b|)" 3; - "~*(?:\b)Mag-Net(?:\b|)" 3; - "~*(?:\b)magpie-crawler(?:\b|)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; - "~*(?:\b)Majestic12(?:\b|)" 3; - "~*(?:\b)Majestic-SEO(?:\b|)" 3; - "~*(?:\b)Majestic\ SEO(?:\b|)" 3; - "~*(?:\b)MarkMonitor(?:\b|)" 3; - "~*(?:\b)MarkWatch(?:\b|)" 3; - "~*(?:\b)Masscan(?:\b|)" 3; - "~*(?:\b)Mass\ Downloader(?:\b|)" 3; - "~*(?:\b)Mata\ Hari(?:\b|)" 3; - "~*(?:\b)MauiBot(?:\b|)" 3; - "~*(?:\b)meanpathbot(?:\b|)" 3; - "~*(?:\b)Meanpathbot(?:\b|)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; - "~*(?:\b)mediawords(?:\b|)" 3; - "~*(?:\b)MegaIndex.ru(?:\b|)" 3; - "~*(?:\b)Metauri(?:\b|)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; - "~*(?:\b)MIDown\ tool(?:\b|)" 3; - "~*(?:\b)MIIxpc(?:\b|)" 3; - "~*(?:\b)Mister\ PiX(?:\b|)" 3; - "~*(?:\b)MJ12bot(?:\b|)" 3; - "~*(?:\b)Mojeek(?:\b|)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; - "~*(?:\b)Mr.4x3(?:\b|)" 3; - "~*(?:\b)MSFrontPage(?:\b|)" 3; - "~*(?:\b)MSIECrawler(?:\b|)" 3; - "~*(?:\b)Msrabot(?:\b|)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; - "~*(?:\b)muhstik-scan(?:\b|)" 3; - "~*(?:\b)Musobot(?:\b|)" 3; - "~*(?:\b)Name\ Intelligence(?:\b|)" 3; - "~*(?:\b)Nameprotect(?:\b|)" 3; - "~*(?:\b)Navroad(?:\b|)" 3; - "~*(?:\b)NearSite(?:\b|)" 3; - "~*(?:\b)Needle(?:\b|)" 3; - "~*(?:\b)Nessus(?:\b|)" 3; - "~*(?:\b)NetAnts(?:\b|)" 3; - "~*(?:\b)Netcraft(?:\b|)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; - "~*(?:\b)NetLyzer(?:\b|)" 3; - "~*(?:\b)NetMechanic(?:\b|)" 3; - "~*(?:\b)NetSpider(?:\b|)" 3; - "~*(?:\b)Nettrack(?:\b|)" 3; - "~*(?:\b)Net\ Vampire(?:\b|)" 3; - "~*(?:\b)Netvibes(?:\b|)" 3; - "~*(?:\b)NetZIP(?:\b|)" 3; - "~*(?:\b)NextGenSearchBot(?:\b|)" 3; - "~*(?:\b)Nibbler(?:\b|)" 3; - "~*(?:\b)NICErsPRO(?:\b|)" 3; - "~*(?:\b)Niki-bot(?:\b|)" 3; - "~*(?:\b)Nikto(?:\b|)" 3; - "~*(?:\b)NimbleCrawler(?:\b|)" 3; - "~*(?:\b)Nimbostratus(?:\b|)" 3; - "~*(?:\b)Ninja(?:\b|)" 3; - "~*(?:\b)Nmap(?:\b|)" 3; - "~*(?:\b)NPbot(?:\b|)" 3; - "~*(?:\b)Nutch(?:\b|)" 3; - "~*(?:\b)oBot(?:\b|)" 3; - "~*(?:\b)Octopus(?:\b|)" 3; - "~*(?:\b)Offline\ Explorer(?:\b|)" 3; - "~*(?:\b)Offline\ Navigator(?:\b|)" 3; - "~*(?:\b)OnCrawl(?:\b|)" 3; - "~*(?:\b)Openfind(?:\b|)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; - "~*(?:\b)Openvas(?:\b|)" 3; - "~*(?:\b)OpenVAS(?:\b|)" 3; - "~*(?:\b)OrangeBot(?:\b|)" 3; - "~*(?:\b)OrangeSpider(?:\b|)" 3; - "~*(?:\b)OutclicksBot(?:\b|)" 3; - "~*(?:\b)OutfoxBot(?:\b|)" 3; - "~*(?:\b)PageAnalyzer(?:\b|)" 3; - "~*(?:\b)Page\ Analyzer(?:\b|)" 3; - "~*(?:\b)PageGrabber(?:\b|)" 3; - "~*(?:\b)page\ scorer(?:\b|)" 3; - "~*(?:\b)PageScorer(?:\b|)" 3; - "~*(?:\b)Pandalytics(?:\b|)" 3; - "~*(?:\b)Panscient(?:\b|)" 3; - "~*(?:\b)Papa\ Foto(?:\b|)" 3; - "~*(?:\b)Pavuk(?:\b|)" 3; - "~*(?:\b)pcBrowser(?:\b|)" 3; - "~*(?:\b)PECL::HTTP(?:\b|)" 3; - "~*(?:\b)PeoplePal(?:\b|)" 3; - "~*(?:\b)PHPCrawl(?:\b|)" 3; - "~*(?:\b)Picscout(?:\b|)" 3; - "~*(?:\b)Picsearch(?:\b|)" 3; - "~*(?:\b)PictureFinder(?:\b|)" 3; - "~*(?:\b)Pimonster(?:\b|)" 3; - "~*(?:\b)Pi-Monster(?:\b|)" 3; - "~*(?:\b)Pixray(?:\b|)" 3; - "~*(?:\b)PleaseCrawl(?:\b|)" 3; - "~*(?:\b)plumanalytics(?:\b|)" 3; - "~*(?:\b)Pockey(?:\b|)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; - "~*(?:\b)Probethenet(?:\b|)" 3; - "~*(?:\b)ProPowerBot(?:\b|)" 3; - "~*(?:\b)ProWebWalker(?:\b|)" 3; - "~*(?:\b)Psbot(?:\b|)" 3; - "~*(?:\b)Pump(?:\b|)" 3; - "~*(?:\b)PxBroker(?:\b|)" 3; - "~*(?:\b)PyCurl(?:\b|)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; - "~*(?:\b)Quick-Crawler(?:\b|)" 3; - "~*(?:\b)RankActive(?:\b|)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; - "~*(?:\b)RankFlex(?:\b|)" 3; - "~*(?:\b)RankingBot(?:\b|)" 3; - "~*(?:\b)RankingBot2(?:\b|)" 3; - "~*(?:\b)Rankivabot(?:\b|)" 3; - "~*(?:\b)RankurBot(?:\b|)" 3; - "~*(?:\b)RealDownload(?:\b|)" 3; - "~*(?:\b)Reaper(?:\b|)" 3; - "~*(?:\b)RebelMouse(?:\b|)" 3; - "~*(?:\b)Recorder(?:\b|)" 3; - "~*(?:\b)RedesScrapy(?:\b|)" 3; - "~*(?:\b)ReGet(?:\b|)" 3; - "~*(?:\b)RepoMonkey(?:\b|)" 3; - "~*(?:\b)Ripper(?:\b|)" 3; - "~*(?:\b)RocketCrawler(?:\b|)" 3; - "~*(?:\b)Rogerbot(?:\b|)" 3; - "~*(?:\b)s1z.ru(?:\b|)" 3; - "~*(?:\b)SalesIntelligent(?:\b|)" 3; - "~*(?:\b)SBIder(?:\b|)" 3; - "~*(?:\b)ScanAlert(?:\b|)" 3; - "~*(?:\b)Scanbot(?:\b|)" 3; - "~*(?:\b)scan.lol(?:\b|)" 3; - "~*(?:\b)ScoutJet(?:\b|)" 3; - "~*(?:\b)Scrapy(?:\b|)" 3; - "~*(?:\b)Screaming(?:\b|)" 3; - "~*(?:\b)ScreenerBot(?:\b|)" 3; - "~*(?:\b)Searchestate(?:\b|)" 3; - "~*(?:\b)SearchmetricsBot(?:\b|)" 3; - "~*(?:\b)Semrush(?:\b|)" 3; - "~*(?:\b)SemrushBot(?:\b|)" 3; - "~*(?:\b)SEOkicks(?:\b|)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; - "~*(?:\b)Seomoz(?:\b|)" 3; - "~*(?:\b)SEOprofiler(?:\b|)" 3; - "~*(?:\b)seoscanners(?:\b|)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; - "~*(?:\b)SEOstats(?:\b|)" 3; - "~*(?:\b)serpstatbot(?:\b|)" 3; - "~*(?:\b)sexsearcher(?:\b|)" 3; - "~*(?:\b)Shodan(?:\b|)" 3; - "~*(?:\b)Siphon(?:\b|)" 3; - "~*(?:\b)SISTRIX(?:\b|)" 3; - "~*(?:\b)Sitebeam(?:\b|)" 3; - "~*(?:\b)SiteExplorer(?:\b|)" 3; - "~*(?:\b)Siteimprove(?:\b|)" 3; - "~*(?:\b)SiteLockSpider(?:\b|)" 3; - "~*(?:\b)SiteSnagger(?:\b|)" 3; - "~*(?:\b)SiteSucker(?:\b|)" 3; - "~*(?:\b)Site\ Sucker(?:\b|)" 3; - "~*(?:\b)Sitevigil(?:\b|)" 3; - "~*(?:\b)SlySearch(?:\b|)" 3; - "~*(?:\b)SmartDownload(?:\b|)" 3; - "~*(?:\b)SMTBot(?:\b|)" 3; - "~*(?:\b)Snake(?:\b|)" 3; - "~*(?:\b)Snapbot(?:\b|)" 3; - "~*(?:\b)Snoopy(?:\b|)" 3; - "~*(?:\b)SocialRankIOBot(?:\b|)" 3; - "~*(?:\b)Sociscraper(?:\b|)" 3; - "~*(?:\b)sogouspider(?:\b|)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; - "~*(?:\b)Sosospider(?:\b|)" 3; - "~*(?:\b)Sottopop(?:\b|)" 3; - "~*(?:\b)SpaceBison(?:\b|)" 3; - "~*(?:\b)Spammen(?:\b|)" 3; - "~*(?:\b)SpankBot(?:\b|)" 3; - "~*(?:\b)Spanner(?:\b|)" 3; - "~*(?:\b)sp_auditbot(?:\b|)" 3; - "~*(?:\b)Spbot(?:\b|)" 3; - "~*(?:\b)Spinn3r(?:\b|)" 3; - "~*(?:\b)SputnikBot(?:\b|)" 3; - "~*(?:\b)spyfu(?:\b|)" 3; - "~*(?:\b)Sqlmap(?:\b|)" 3; - "~*(?:\b)Sqlworm(?:\b|)" 3; - "~*(?:\b)Sqworm(?:\b|)" 3; - "~*(?:\b)Steeler(?:\b|)" 3; - "~*(?:\b)Stripper(?:\b|)" 3; - "~*(?:\b)Sucker(?:\b|)" 3; - "~*(?:\b)Sucuri(?:\b|)" 3; - "~*(?:\b)SuperBot(?:\b|)" 3; - "~*(?:\b)SuperHTTP(?:\b|)" 3; - "~*(?:\b)Surfbot(?:\b|)" 3; - "~*(?:\b)SurveyBot(?:\b|)" 3; - "~*(?:\b)Suzuran(?:\b|)" 3; - "~*(?:\b)Swiftbot(?:\b|)" 3; - "~*(?:\b)sysscan(?:\b|)" 3; - "~*(?:\b)Szukacz(?:\b|)" 3; - "~*(?:\b)T0PHackTeam(?:\b|)" 3; - "~*(?:\b)T8Abot(?:\b|)" 3; - "~*(?:\b)tAkeOut(?:\b|)" 3; - "~*(?:\b)Teleport(?:\b|)" 3; - "~*(?:\b)TeleportPro(?:\b|)" 3; - "~*(?:\b)Telesoft(?:\b|)" 3; - "~*(?:\b)Telesphoreo(?:\b|)" 3; - "~*(?:\b)Telesphorep(?:\b|)" 3; - "~*(?:\b)The\ Intraformant(?:\b|)" 3; - "~*(?:\b)TheNomad(?:\b|)" 3; - "~*(?:\b)Thumbor(?:\b|)" 3; - "~*(?:\b)TightTwatBot(?:\b|)" 3; - "~*(?:\b)Titan(?:\b|)" 3; - "~*(?:\b)Toata(?:\b|)" 3; - "~*(?:\b)Toweyabot(?:\b|)" 3; - "~*(?:\b)Tracemyfile(?:\b|)" 3; - "~*(?:\b)Trendiction(?:\b|)" 3; - "~*(?:\b)Trendictionbot(?:\b|)" 3; - "~*(?:\b)trendiction.com(?:\b|)" 3; - "~*(?:\b)trendiction.de(?:\b|)" 3; - "~*(?:\b)True_Robot(?:\b|)" 3; - "~*(?:\b)Turingos(?:\b|)" 3; - "~*(?:\b)Turnitin(?:\b|)" 3; - "~*(?:\b)TurnitinBot(?:\b|)" 3; - "~*(?:\b)TwengaBot(?:\b|)" 3; - "~*(?:\b)Twice(?:\b|)" 3; - "~*(?:\b)Typhoeus(?:\b|)" 3; - "~*(?:\b)UnisterBot(?:\b|)" 3; - "~*(?:\b)Upflow(?:\b|)" 3; - "~*(?:\b)URLy.Warning(?:\b|)" 3; - "~*(?:\b)URLy\ Warning(?:\b|)" 3; - "~*(?:\b)Vacuum(?:\b|)" 3; - "~*(?:\b)Vagabondo(?:\b|)" 3; - "~*(?:\b)VB\ Project(?:\b|)" 3; - "~*(?:\b)VCI(?:\b|)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; - "~*(?:\b)VidibleScraper(?:\b|)" 3; - "~*(?:\b)Virusdie(?:\b|)" 3; - "~*(?:\b)VoidEYE(?:\b|)" 3; - "~*(?:\b)Voil(?:\b|)" 3; - "~*(?:\b)Voltron(?:\b|)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; - "~*(?:\b)WallpapersHD(?:\b|)" 3; - "~*(?:\b)WASALive-Bot(?:\b|)" 3; - "~*(?:\b)WBSearchBot(?:\b|)" 3; - "~*(?:\b)Webalta(?:\b|)" 3; - "~*(?:\b)WebAuto(?:\b|)" 3; - "~*(?:\b)Web\ Auto(?:\b|)" 3; - "~*(?:\b)WebBandit(?:\b|)" 3; - "~*(?:\b)WebCollage(?:\b|)" 3; - "~*(?:\b)Web\ Collage(?:\b|)" 3; - "~*(?:\b)WebCopier(?:\b|)" 3; - "~*(?:\b)WEBDAV(?:\b|)" 3; - "~*(?:\b)WebEnhancer(?:\b|)" 3; - "~*(?:\b)Web\ Enhancer(?:\b|)" 3; - "~*(?:\b)WebFetch(?:\b|)" 3; - "~*(?:\b)Web\ Fetch(?:\b|)" 3; - "~*(?:\b)WebFuck(?:\b|)" 3; - "~*(?:\b)Web\ Fuck(?:\b|)" 3; - "~*(?:\b)WebGo\ IS(?:\b|)" 3; - "~*(?:\b)WebImageCollector(?:\b|)" 3; - "~*(?:\b)WebLeacher(?:\b|)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; - "~*(?:\b)webmeup-crawler(?:\b|)" 3; - "~*(?:\b)WebPix(?:\b|)" 3; - "~*(?:\b)Web\ Pix(?:\b|)" 3; - "~*(?:\b)WebReaper(?:\b|)" 3; - "~*(?:\b)WebSauger(?:\b|)" 3; - "~*(?:\b)Web\ Sauger(?:\b|)" 3; - "~*(?:\b)Webshag(?:\b|)" 3; - "~*(?:\b)WebsiteExtractor(?:\b|)" 3; - "~*(?:\b)WebsiteQuester(?:\b|)" 3; - "~*(?:\b)Website\ Quester(?:\b|)" 3; - "~*(?:\b)Webster(?:\b|)" 3; - "~*(?:\b)WebStripper(?:\b|)" 3; - "~*(?:\b)WebSucker(?:\b|)" 3; - "~*(?:\b)Web\ Sucker(?:\b|)" 3; - "~*(?:\b)WebWhacker(?:\b|)" 3; - "~*(?:\b)WebZIP(?:\b|)" 3; - "~*(?:\b)WeSEE(?:\b|)" 3; - "~*(?:\b)Whack(?:\b|)" 3; - "~*(?:\b)Whacker(?:\b|)" 3; - "~*(?:\b)Whatweb(?:\b|)" 3; - "~*(?:\b)Who.is\ Bot(?:\b|)" 3; - "~*(?:\b)Widow(?:\b|)" 3; - "~*(?:\b)WinHTTrack(?:\b|)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; - "~*(?:\b)WISENutbot(?:\b|)" 3; - "~*(?:\b)Wonderbot(?:\b|)" 3; - "~*(?:\b)Woobot(?:\b|)" 3; - "~*(?:\b)Wotbox(?:\b|)" 3; - "~*(?:\b)Wprecon(?:\b|)" 3; - "~*(?:\b)WPScan(?:\b|)" 3; - "~*(?:\b)WWW-Collector-E(?:\b|)" 3; - "~*(?:\b)WWW-Mechanize(?:\b|)" 3; - "~*(?:\b)WWW::Mechanize(?:\b|)" 3; - "~*(?:\b)WWWOFFLE(?:\b|)" 3; - "~*(?:\b)x09Mozilla(?:\b|)" 3; - "~*(?:\b)x22Mozilla(?:\b|)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; - "~*(?:\b)Xenu(?:\b|)" 3; - "~*(?:\b)xpymep1.exe(?:\b|)" 3; - "~*(?:\b)YoudaoBot(?:\b|)" 3; - "~*(?:\b)Zade(?:\b|)" 3; - "~*(?:\b)Zauba(?:\b|)" 3; - "~*(?:\b)zauba.io(?:\b|)" 3; - "~*(?:\b)Zermelo(?:\b|)" 3; - "~*(?:\b)Zeus(?:\b|)" 3; - "~*(?:\b)zgrab(?:\b|)" 3; - "~*(?:\b)Zitebot(?:\b|)" 3; - "~*(?:\b)ZmEu(?:\b|)" 3; - "~*(?:\b)ZumBot(?:\b|)" 3; - "~*(?:\b)ZyBorg(?:\b|)" 3; + "~*(?:\b)360Spider(?:\b)" 3; + "~*(?:\b)404checker(?:\b)" 3; + "~*(?:\b)404enemy(?:\b)" 3; + "~*(?:\b)80legs(?:\b)" 3; + "~*(?:\b)Abonti(?:\b)" 3; + "~*(?:\b)Aboundex(?:\b)" 3; + "~*(?:\b)Aboundexbot(?:\b)" 3; + "~*(?:\b)Acunetix(?:\b)" 3; + "~*(?:\b)ADmantX(?:\b)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; + "~*(?:\b)AhrefsBot(?:\b)" 3; + "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)AiHitBot(?:\b)" 3; + "~*(?:\b)Aipbot(?:\b)" 3; + "~*(?:\b)Alexibot(?:\b)" 3; + "~*(?:\b)Alligator(?:\b)" 3; + "~*(?:\b)AllSubmitter(?:\b)" 3; + "~*(?:\b)AlphaBot(?:\b)" 3; + "~*(?:\b)Anarchie(?:\b)" 3; + "~*(?:\b)Apexoo(?:\b)" 3; + "~*(?:\b)archive.org_bot(?:\b)" 3; + "~*(?:\b)ASPSeek(?:\b)" 3; + "~*(?:\b)Asterias(?:\b)" 3; + "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)BackDoorBot(?:\b)" 3; + "~*(?:\b)Backlink-Ceck(?:\b)" 3; + "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)BacklinkCrawler(?:\b)" 3; + "~*(?:\b)BackStreet(?:\b)" 3; + "~*(?:\b)BackWeb(?:\b)" 3; + "~*(?:\b)Badass(?:\b)" 3; + "~*(?:\b)Bandit(?:\b)" 3; + "~*(?:\b)Barkrowler(?:\b)" 3; + "~*(?:\b)BatchFTP(?:\b)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; + "~*(?:\b)BBBike(?:\b)" 3; + "~*(?:\b)BDCbot(?:\b)" 3; + "~*(?:\b)BDFetch(?:\b)" 3; + "~*(?:\b)BetaBot(?:\b)" 3; + "~*(?:\b)Bigfoot(?:\b)" 3; + "~*(?:\b)Bitacle(?:\b)" 3; + "~*(?:\b)Blackboard(?:\b)" 3; + "~*(?:\b)Black\ Hole(?:\b)" 3; + "~*(?:\b)BlackWidow(?:\b)" 3; + "~*(?:\b)BLEXBot(?:\b)" 3; + "~*(?:\b)Blow(?:\b)" 3; + "~*(?:\b)BlowFish(?:\b)" 3; + "~*(?:\b)Boardreader(?:\b)" 3; + "~*(?:\b)Bolt(?:\b)" 3; + "~*(?:\b)BotALot(?:\b)" 3; + "~*(?:\b)Brandprotect(?:\b)" 3; + "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buddy(?:\b)" 3; + "~*(?:\b)BuiltBotTough(?:\b)" 3; + "~*(?:\b)BuiltWith(?:\b)" 3; + "~*(?:\b)Bullseye(?:\b)" 3; + "~*(?:\b)BunnySlippers(?:\b)" 3; + "~*(?:\b)BuzzSumo(?:\b)" 3; + "~*(?:\b)Calculon(?:\b)" 3; + "~*(?:\b)CATExplorador(?:\b)" 3; + "~*(?:\b)CazoodleBot(?:\b)" 3; + "~*(?:\b)CCBot(?:\b)" 3; + "~*(?:\b)Cegbfeieh(?:\b)" 3; + "~*(?:\b)CheeseBot(?:\b)" 3; + "~*(?:\b)CherryPicker(?:\b)" 3; + "~*(?:\b)CheTeam(?:\b)" 3; + "~*(?:\b)ChinaClaw(?:\b)" 3; + "~*(?:\b)Chlooe(?:\b)" 3; + "~*(?:\b)Claritybot(?:\b)" 3; + "~*(?:\b)Cliqzbot(?:\b)" 3; + "~*(?:\b)Cloud\ mapping(?:\b)" 3; + "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cogentbot(?:\b)" 3; + "~*(?:\b)cognitiveseo(?:\b)" 3; + "~*(?:\b)Collector(?:\b)" 3; + "~*(?:\b)com.plumanalytics(?:\b)" 3; + "~*(?:\b)Copier(?:\b)" 3; + "~*(?:\b)CopyRightCheck(?:\b)" 3; + "~*(?:\b)Copyscape(?:\b)" 3; + "~*(?:\b)Cosmos(?:\b)" 3; + "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)crawler4j(?:\b)" 3; + "~*(?:\b)crawler.feedback(?:\b)" 3; + "~*(?:\b)crawl.sogou.com(?:\b)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b)" 3; + "~*(?:\b)Crescent(?:\b)" 3; + "~*(?:\b)CrunchBot(?:\b)" 3; + "~*(?:\b)CSHttp(?:\b)" 3; + "~*(?:\b)Curious(?:\b)" 3; + "~*(?:\b)Custo(?:\b)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; + "~*(?:\b)DataCha0s(?:\b)" 3; + "~*(?:\b)DBLBot(?:\b)" 3; + "~*(?:\b)demandbase-bot(?:\b)" 3; + "~*(?:\b)Demon(?:\b)" 3; + "~*(?:\b)Deusu(?:\b)" 3; + "~*(?:\b)Devil(?:\b)" 3; + "~*(?:\b)Digincore(?:\b)" 3; + "~*(?:\b)DigitalPebble(?:\b)" 3; + "~*(?:\b)DIIbot(?:\b)" 3; + "~*(?:\b)Dirbuster(?:\b)" 3; + "~*(?:\b)Disco(?:\b)" 3; + "~*(?:\b)Discobot(?:\b)" 3; + "~*(?:\b)Discoverybot(?:\b)" 3; + "~*(?:\b)Dispatch(?:\b)" 3; + "~*(?:\b)DittoSpyder(?:\b)" 3; + "~*(?:\b)DnyzBot(?:\b)" 3; + "~*(?:\b)DomainAppender(?:\b)" 3; + "~*(?:\b)DomainCrawler(?:\b)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)DomainStatsBot(?:\b)" 3; + "~*(?:\b)Dotbot(?:\b)" 3; + "~*(?:\b)Download\ Wonder(?:\b)" 3; + "~*(?:\b)Dragonfly(?:\b)" 3; + "~*(?:\b)Drip(?:\b)" 3; + "~*(?:\b)DSearch(?:\b)" 3; + "~*(?:\b)DTS\ Agent(?:\b)" 3; + "~*(?:\b)EasyDL(?:\b)" 3; + "~*(?:\b)Ebingbong(?:\b)" 3; + "~*(?:\b)eCatch(?:\b)" 3; + "~*(?:\b)ECCP/1.0(?:\b)" 3; + "~*(?:\b)Ecxi(?:\b)" 3; + "~*(?:\b)EirGrabber(?:\b)" 3; + "~*(?:\b)EMail\ Siphon(?:\b)" 3; + "~*(?:\b)EMail\ Wolf(?:\b)" 3; + "~*(?:\b)EroCrawler(?:\b)" 3; + "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)Evil(?:\b)" 3; + "~*(?:\b)Exabot(?:\b)" 3; + "~*(?:\b)Express\ WebPictures(?:\b)" 3; + "~*(?:\b)ExtLinksBot(?:\b)" 3; + "~*(?:\b)Extractor(?:\b)" 3; + "~*(?:\b)ExtractorPro(?:\b)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; + "~*(?:\b)EyeNetIE(?:\b)" 3; + "~*(?:\b)Ezooms(?:\b)" 3; + "~*(?:\b)facebookscraper(?:\b)" 3; + "~*(?:\b)FDM(?:\b)" 3; + "~*(?:\b)FemtosearchBot(?:\b)" 3; + "~*(?:\b)FHscan(?:\b)" 3; + "~*(?:\b)Fimap(?:\b)" 3; + "~*(?:\b)Firefox/7.0(?:\b)" 3; + "~*(?:\b)FlashGet(?:\b)" 3; + "~*(?:\b)Flunky(?:\b)" 3; + "~*(?:\b)Foobot(?:\b)" 3; + "~*(?:\b)Freeuploader(?:\b)" 3; + "~*(?:\b)FrontPage(?:\b)" 3; + "~*(?:\b)FyberSpider(?:\b)" 3; + "~*(?:\b)Fyrebot(?:\b)" 3; + "~*(?:\b)GalaxyBot(?:\b)" 3; + "~*(?:\b)Genieo(?:\b)" 3; + "~*(?:\b)GermCrawler(?:\b)" 3; + "~*(?:\b)Getintent(?:\b)" 3; + "~*(?:\b)GetRight(?:\b)" 3; + "~*(?:\b)GetWeb(?:\b)" 3; + "~*(?:\b)Gigablast(?:\b)" 3; + "~*(?:\b)Gigabot(?:\b)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; + "~*(?:\b)Gotit(?:\b)" 3; + "~*(?:\b)GoZilla(?:\b)" 3; + "~*(?:\b)Go!Zilla(?:\b)" 3; + "~*(?:\b)Grabber(?:\b)" 3; + "~*(?:\b)GrabNet(?:\b)" 3; + "~*(?:\b)Grafula(?:\b)" 3; + "~*(?:\b)GrapeFX(?:\b)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b)" 3; + "~*(?:\b)GridBot(?:\b)" 3; + "~*(?:\b)GT::WWW(?:\b)" 3; + "~*(?:\b)Haansoft(?:\b)" 3; + "~*(?:\b)HaosouSpider(?:\b)" 3; + "~*(?:\b)Harvest(?:\b)" 3; + "~*(?:\b)Havij(?:\b)" 3; + "~*(?:\b)HEADMasterSEO(?:\b)" 3; + "~*(?:\b)Heritrix(?:\b)" 3; + "~*(?:\b)Hloader(?:\b)" 3; + "~*(?:\b)HMView(?:\b)" 3; + "~*(?:\b)HTMLparser(?:\b)" 3; + "~*(?:\b)HTTP::Lite(?:\b)" 3; + "~*(?:\b)HTTrack(?:\b)" 3; + "~*(?:\b)Humanlinks(?:\b)" 3; + "~*(?:\b)HybridBot(?:\b)" 3; + "~*(?:\b)Iblog(?:\b)" 3; + "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)Id-search(?:\b)" 3; + "~*(?:\b)IlseBot(?:\b)" 3; + "~*(?:\b)Image\ Fetch(?:\b)" 3; + "~*(?:\b)Image\ Sucker(?:\b)" 3; + "~*(?:\b)IndeedBot(?:\b)" 3; + "~*(?:\b)Indy\ Library(?:\b)" 3; + "~*(?:\b)InfoNaviRobot(?:\b)" 3; + "~*(?:\b)InfoTekies(?:\b)" 3; + "~*(?:\b)instabid(?:\b)" 3; + "~*(?:\b)Intelliseek(?:\b)" 3; + "~*(?:\b)InterGET(?:\b)" 3; + "~*(?:\b)Internet\ Ninja(?:\b)" 3; + "~*(?:\b)InternetSeer(?:\b)" 3; + "~*(?:\b)internetVista\ monitor(?:\b)" 3; + "~*(?:\b)ips-agent(?:\b)" 3; + "~*(?:\b)Iria(?:\b)" 3; + "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)Iskanie(?:\b)" 3; + "~*(?:\b)IstellaBot(?:\b)" 3; + "~*(?:\b)JamesBOT(?:\b)" 3; + "~*(?:\b)Jbrofuzz(?:\b)" 3; + "~*(?:\b)JennyBot(?:\b)" 3; + "~*(?:\b)JetCar(?:\b)" 3; + "~*(?:\b)Jetty(?:\b)" 3; + "~*(?:\b)JikeSpider(?:\b)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; + "~*(?:\b)Joomla(?:\b)" 3; + "~*(?:\b)Jorgee(?:\b)" 3; + "~*(?:\b)JustView(?:\b)" 3; + "~*(?:\b)Jyxobot(?:\b)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b)" 3; + "~*(?:\b)Keyword\ Density(?:\b)" 3; + "~*(?:\b)Kozmosbot(?:\b)" 3; + "~*(?:\b)Lanshanbot(?:\b)" 3; + "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)LeechFTP(?:\b)" 3; + "~*(?:\b)LeechGet(?:\b)" 3; + "~*(?:\b)LexiBot(?:\b)" 3; + "~*(?:\b)Lftp(?:\b)" 3; + "~*(?:\b)LibWeb(?:\b)" 3; + "~*(?:\b)Libwhisker(?:\b)" 3; + "~*(?:\b)Lightspeedsystems(?:\b)" 3; + "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkdexbot(?:\b)" 3; + "~*(?:\b)LinkextractorPro(?:\b)" 3; + "~*(?:\b)LinkpadBot(?:\b)" 3; + "~*(?:\b)LinkScan(?:\b)" 3; + "~*(?:\b)LinksManager(?:\b)" 3; + "~*(?:\b)LinkWalker(?:\b)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; + "~*(?:\b)Lipperhey(?:\b)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; + "~*(?:\b)Litemage_walker(?:\b)" 3; + "~*(?:\b)Lmspider(?:\b)" 3; + "~*(?:\b)LNSpiderguy(?:\b)" 3; + "~*(?:\b)Ltx71(?:\b)" 3; + "~*(?:\b)lwp-request(?:\b)" 3; + "~*(?:\b)LWP::Simple(?:\b)" 3; + "~*(?:\b)lwp-trivial(?:\b)" 3; + "~*(?:\b)Magnet(?:\b)" 3; + "~*(?:\b)Mag-Net(?:\b)" 3; + "~*(?:\b)magpie-crawler(?:\b)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b)" 3; + "~*(?:\b)Majestic12(?:\b)" 3; + "~*(?:\b)Majestic-SEO(?:\b)" 3; + "~*(?:\b)Majestic\ SEO(?:\b)" 3; + "~*(?:\b)MarkMonitor(?:\b)" 3; + "~*(?:\b)MarkWatch(?:\b)" 3; + "~*(?:\b)Masscan(?:\b)" 3; + "~*(?:\b)Mass\ Downloader(?:\b)" 3; + "~*(?:\b)Mata\ Hari(?:\b)" 3; + "~*(?:\b)MauiBot(?:\b)" 3; + "~*(?:\b)meanpathbot(?:\b)" 3; + "~*(?:\b)Meanpathbot(?:\b)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b)" 3; + "~*(?:\b)mediawords(?:\b)" 3; + "~*(?:\b)MegaIndex.ru(?:\b)" 3; + "~*(?:\b)Metauri(?:\b)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; + "~*(?:\b)MIDown\ tool(?:\b)" 3; + "~*(?:\b)MIIxpc(?:\b)" 3; + "~*(?:\b)Mister\ PiX(?:\b)" 3; + "~*(?:\b)MJ12bot(?:\b)" 3; + "~*(?:\b)Mojeek(?:\b)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; + "~*(?:\b)Mr.4x3(?:\b)" 3; + "~*(?:\b)MSFrontPage(?:\b)" 3; + "~*(?:\b)MSIECrawler(?:\b)" 3; + "~*(?:\b)Msrabot(?:\b)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; + "~*(?:\b)muhstik-scan(?:\b)" 3; + "~*(?:\b)Musobot(?:\b)" 3; + "~*(?:\b)Name\ Intelligence(?:\b)" 3; + "~*(?:\b)Nameprotect(?:\b)" 3; + "~*(?:\b)Navroad(?:\b)" 3; + "~*(?:\b)NearSite(?:\b)" 3; + "~*(?:\b)Needle(?:\b)" 3; + "~*(?:\b)Nessus(?:\b)" 3; + "~*(?:\b)NetAnts(?:\b)" 3; + "~*(?:\b)Netcraft(?:\b)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; + "~*(?:\b)NetLyzer(?:\b)" 3; + "~*(?:\b)NetMechanic(?:\b)" 3; + "~*(?:\b)NetSpider(?:\b)" 3; + "~*(?:\b)Nettrack(?:\b)" 3; + "~*(?:\b)Net\ Vampire(?:\b)" 3; + "~*(?:\b)Netvibes(?:\b)" 3; + "~*(?:\b)NetZIP(?:\b)" 3; + "~*(?:\b)NextGenSearchBot(?:\b)" 3; + "~*(?:\b)Nibbler(?:\b)" 3; + "~*(?:\b)NICErsPRO(?:\b)" 3; + "~*(?:\b)Niki-bot(?:\b)" 3; + "~*(?:\b)Nikto(?:\b)" 3; + "~*(?:\b)NimbleCrawler(?:\b)" 3; + "~*(?:\b)Nimbostratus(?:\b)" 3; + "~*(?:\b)Ninja(?:\b)" 3; + "~*(?:\b)Nmap(?:\b)" 3; + "~*(?:\b)NPbot(?:\b)" 3; + "~*(?:\b)Nutch(?:\b)" 3; + "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)Octopus(?:\b)" 3; + "~*(?:\b)Offline\ Explorer(?:\b)" 3; + "~*(?:\b)Offline\ Navigator(?:\b)" 3; + "~*(?:\b)OnCrawl(?:\b)" 3; + "~*(?:\b)Openfind(?:\b)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b)" 3; + "~*(?:\b)Openvas(?:\b)" 3; + "~*(?:\b)OpenVAS(?:\b)" 3; + "~*(?:\b)OrangeBot(?:\b)" 3; + "~*(?:\b)OrangeSpider(?:\b)" 3; + "~*(?:\b)OutclicksBot(?:\b)" 3; + "~*(?:\b)OutfoxBot(?:\b)" 3; + "~*(?:\b)PageAnalyzer(?:\b)" 3; + "~*(?:\b)Page\ Analyzer(?:\b)" 3; + "~*(?:\b)PageGrabber(?:\b)" 3; + "~*(?:\b)page\ scorer(?:\b)" 3; + "~*(?:\b)PageScorer(?:\b)" 3; + "~*(?:\b)Pandalytics(?:\b)" 3; + "~*(?:\b)Panscient(?:\b)" 3; + "~*(?:\b)Papa\ Foto(?:\b)" 3; + "~*(?:\b)Pavuk(?:\b)" 3; + "~*(?:\b)pcBrowser(?:\b)" 3; + "~*(?:\b)PECL::HTTP(?:\b)" 3; + "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)PHPCrawl(?:\b)" 3; + "~*(?:\b)Picscout(?:\b)" 3; + "~*(?:\b)Picsearch(?:\b)" 3; + "~*(?:\b)PictureFinder(?:\b)" 3; + "~*(?:\b)Pimonster(?:\b)" 3; + "~*(?:\b)Pi-Monster(?:\b)" 3; + "~*(?:\b)Pixray(?:\b)" 3; + "~*(?:\b)PleaseCrawl(?:\b)" 3; + "~*(?:\b)plumanalytics(?:\b)" 3; + "~*(?:\b)Pockey(?:\b)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; + "~*(?:\b)Probethenet(?:\b)" 3; + "~*(?:\b)ProPowerBot(?:\b)" 3; + "~*(?:\b)ProWebWalker(?:\b)" 3; + "~*(?:\b)Psbot(?:\b)" 3; + "~*(?:\b)Pump(?:\b)" 3; + "~*(?:\b)PxBroker(?:\b)" 3; + "~*(?:\b)PyCurl(?:\b)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; + "~*(?:\b)Quick-Crawler(?:\b)" 3; + "~*(?:\b)RankActive(?:\b)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b)" 3; + "~*(?:\b)RankFlex(?:\b)" 3; + "~*(?:\b)RankingBot(?:\b)" 3; + "~*(?:\b)RankingBot2(?:\b)" 3; + "~*(?:\b)Rankivabot(?:\b)" 3; + "~*(?:\b)RankurBot(?:\b)" 3; + "~*(?:\b)RealDownload(?:\b)" 3; + "~*(?:\b)Reaper(?:\b)" 3; + "~*(?:\b)RebelMouse(?:\b)" 3; + "~*(?:\b)Recorder(?:\b)" 3; + "~*(?:\b)RedesScrapy(?:\b)" 3; + "~*(?:\b)ReGet(?:\b)" 3; + "~*(?:\b)RepoMonkey(?:\b)" 3; + "~*(?:\b)Ripper(?:\b)" 3; + "~*(?:\b)RocketCrawler(?:\b)" 3; + "~*(?:\b)Rogerbot(?:\b)" 3; + "~*(?:\b)s1z.ru(?:\b)" 3; + "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)SBIder(?:\b)" 3; + "~*(?:\b)ScanAlert(?:\b)" 3; + "~*(?:\b)Scanbot(?:\b)" 3; + "~*(?:\b)scan.lol(?:\b)" 3; + "~*(?:\b)ScoutJet(?:\b)" 3; + "~*(?:\b)Scrapy(?:\b)" 3; + "~*(?:\b)Screaming(?:\b)" 3; + "~*(?:\b)ScreenerBot(?:\b)" 3; + "~*(?:\b)Searchestate(?:\b)" 3; + "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Semrush(?:\b)" 3; + "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SEOkicks(?:\b)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; + "~*(?:\b)Seomoz(?:\b)" 3; + "~*(?:\b)SEOprofiler(?:\b)" 3; + "~*(?:\b)seoscanners(?:\b)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b)" 3; + "~*(?:\b)SEOstats(?:\b)" 3; + "~*(?:\b)serpstatbot(?:\b)" 3; + "~*(?:\b)sexsearcher(?:\b)" 3; + "~*(?:\b)Shodan(?:\b)" 3; + "~*(?:\b)Siphon(?:\b)" 3; + "~*(?:\b)SISTRIX(?:\b)" 3; + "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteExplorer(?:\b)" 3; + "~*(?:\b)Siteimprove(?:\b)" 3; + "~*(?:\b)SiteLockSpider(?:\b)" 3; + "~*(?:\b)SiteSnagger(?:\b)" 3; + "~*(?:\b)SiteSucker(?:\b)" 3; + "~*(?:\b)Site\ Sucker(?:\b)" 3; + "~*(?:\b)Sitevigil(?:\b)" 3; + "~*(?:\b)SlySearch(?:\b)" 3; + "~*(?:\b)SmartDownload(?:\b)" 3; + "~*(?:\b)SMTBot(?:\b)" 3; + "~*(?:\b)Snake(?:\b)" 3; + "~*(?:\b)Snapbot(?:\b)" 3; + "~*(?:\b)Snoopy(?:\b)" 3; + "~*(?:\b)SocialRankIOBot(?:\b)" 3; + "~*(?:\b)Sociscraper(?:\b)" 3; + "~*(?:\b)sogouspider(?:\b)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; + "~*(?:\b)Sosospider(?:\b)" 3; + "~*(?:\b)Sottopop(?:\b)" 3; + "~*(?:\b)SpaceBison(?:\b)" 3; + "~*(?:\b)Spammen(?:\b)" 3; + "~*(?:\b)SpankBot(?:\b)" 3; + "~*(?:\b)Spanner(?:\b)" 3; + "~*(?:\b)sp_auditbot(?:\b)" 3; + "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spinn3r(?:\b)" 3; + "~*(?:\b)SputnikBot(?:\b)" 3; + "~*(?:\b)spyfu(?:\b)" 3; + "~*(?:\b)Sqlmap(?:\b)" 3; + "~*(?:\b)Sqlworm(?:\b)" 3; + "~*(?:\b)Sqworm(?:\b)" 3; + "~*(?:\b)Steeler(?:\b)" 3; + "~*(?:\b)Stripper(?:\b)" 3; + "~*(?:\b)Sucker(?:\b)" 3; + "~*(?:\b)Sucuri(?:\b)" 3; + "~*(?:\b)SuperBot(?:\b)" 3; + "~*(?:\b)SuperHTTP(?:\b)" 3; + "~*(?:\b)Surfbot(?:\b)" 3; + "~*(?:\b)SurveyBot(?:\b)" 3; + "~*(?:\b)Suzuran(?:\b)" 3; + "~*(?:\b)Swiftbot(?:\b)" 3; + "~*(?:\b)sysscan(?:\b)" 3; + "~*(?:\b)Szukacz(?:\b)" 3; + "~*(?:\b)T0PHackTeam(?:\b)" 3; + "~*(?:\b)T8Abot(?:\b)" 3; + "~*(?:\b)tAkeOut(?:\b)" 3; + "~*(?:\b)Teleport(?:\b)" 3; + "~*(?:\b)TeleportPro(?:\b)" 3; + "~*(?:\b)Telesoft(?:\b)" 3; + "~*(?:\b)Telesphoreo(?:\b)" 3; + "~*(?:\b)Telesphorep(?:\b)" 3; + "~*(?:\b)The\ Intraformant(?:\b)" 3; + "~*(?:\b)TheNomad(?:\b)" 3; + "~*(?:\b)Thumbor(?:\b)" 3; + "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)Titan(?:\b)" 3; + "~*(?:\b)Toata(?:\b)" 3; + "~*(?:\b)Toweyabot(?:\b)" 3; + "~*(?:\b)Tracemyfile(?:\b)" 3; + "~*(?:\b)Trendiction(?:\b)" 3; + "~*(?:\b)Trendictionbot(?:\b)" 3; + "~*(?:\b)trendiction.com(?:\b)" 3; + "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)True_Robot(?:\b)" 3; + "~*(?:\b)Turingos(?:\b)" 3; + "~*(?:\b)Turnitin(?:\b)" 3; + "~*(?:\b)TurnitinBot(?:\b)" 3; + "~*(?:\b)TwengaBot(?:\b)" 3; + "~*(?:\b)Twice(?:\b)" 3; + "~*(?:\b)Typhoeus(?:\b)" 3; + "~*(?:\b)UnisterBot(?:\b)" 3; + "~*(?:\b)Upflow(?:\b)" 3; + "~*(?:\b)URLy.Warning(?:\b)" 3; + "~*(?:\b)URLy\ Warning(?:\b)" 3; + "~*(?:\b)Vacuum(?:\b)" 3; + "~*(?:\b)Vagabondo(?:\b)" 3; + "~*(?:\b)VB\ Project(?:\b)" 3; + "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b)" 3; + "~*(?:\b)VidibleScraper(?:\b)" 3; + "~*(?:\b)Virusdie(?:\b)" 3; + "~*(?:\b)VoidEYE(?:\b)" 3; + "~*(?:\b)Voil(?:\b)" 3; + "~*(?:\b)Voltron(?:\b)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b)" 3; + "~*(?:\b)WallpapersHD(?:\b)" 3; + "~*(?:\b)WASALive-Bot(?:\b)" 3; + "~*(?:\b)WBSearchBot(?:\b)" 3; + "~*(?:\b)Webalta(?:\b)" 3; + "~*(?:\b)WebAuto(?:\b)" 3; + "~*(?:\b)Web\ Auto(?:\b)" 3; + "~*(?:\b)WebBandit(?:\b)" 3; + "~*(?:\b)WebCollage(?:\b)" 3; + "~*(?:\b)Web\ Collage(?:\b)" 3; + "~*(?:\b)WebCopier(?:\b)" 3; + "~*(?:\b)WEBDAV(?:\b)" 3; + "~*(?:\b)WebEnhancer(?:\b)" 3; + "~*(?:\b)Web\ Enhancer(?:\b)" 3; + "~*(?:\b)WebFetch(?:\b)" 3; + "~*(?:\b)Web\ Fetch(?:\b)" 3; + "~*(?:\b)WebFuck(?:\b)" 3; + "~*(?:\b)Web\ Fuck(?:\b)" 3; + "~*(?:\b)WebGo\ IS(?:\b)" 3; + "~*(?:\b)WebImageCollector(?:\b)" 3; + "~*(?:\b)WebLeacher(?:\b)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; + "~*(?:\b)webmeup-crawler(?:\b)" 3; + "~*(?:\b)WebPix(?:\b)" 3; + "~*(?:\b)Web\ Pix(?:\b)" 3; + "~*(?:\b)WebReaper(?:\b)" 3; + "~*(?:\b)WebSauger(?:\b)" 3; + "~*(?:\b)Web\ Sauger(?:\b)" 3; + "~*(?:\b)Webshag(?:\b)" 3; + "~*(?:\b)WebsiteExtractor(?:\b)" 3; + "~*(?:\b)WebsiteQuester(?:\b)" 3; + "~*(?:\b)Website\ Quester(?:\b)" 3; + "~*(?:\b)Webster(?:\b)" 3; + "~*(?:\b)WebStripper(?:\b)" 3; + "~*(?:\b)WebSucker(?:\b)" 3; + "~*(?:\b)Web\ Sucker(?:\b)" 3; + "~*(?:\b)WebWhacker(?:\b)" 3; + "~*(?:\b)WebZIP(?:\b)" 3; + "~*(?:\b)WeSEE(?:\b)" 3; + "~*(?:\b)Whack(?:\b)" 3; + "~*(?:\b)Whacker(?:\b)" 3; + "~*(?:\b)Whatweb(?:\b)" 3; + "~*(?:\b)Who.is\ Bot(?:\b)" 3; + "~*(?:\b)Widow(?:\b)" 3; + "~*(?:\b)WinHTTrack(?:\b)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; + "~*(?:\b)WISENutbot(?:\b)" 3; + "~*(?:\b)Wonderbot(?:\b)" 3; + "~*(?:\b)Woobot(?:\b)" 3; + "~*(?:\b)Wotbox(?:\b)" 3; + "~*(?:\b)Wprecon(?:\b)" 3; + "~*(?:\b)WPScan(?:\b)" 3; + "~*(?:\b)WWW-Collector-E(?:\b)" 3; + "~*(?:\b)WWW-Mechanize(?:\b)" 3; + "~*(?:\b)WWW::Mechanize(?:\b)" 3; + "~*(?:\b)WWWOFFLE(?:\b)" 3; + "~*(?:\b)x09Mozilla(?:\b)" 3; + "~*(?:\b)x22Mozilla(?:\b)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; + "~*(?:\b)Xenu(?:\b)" 3; + "~*(?:\b)xpymep1.exe(?:\b)" 3; + "~*(?:\b)YoudaoBot(?:\b)" 3; + "~*(?:\b)Zade(?:\b)" 3; + "~*(?:\b)Zauba(?:\b)" 3; + "~*(?:\b)zauba.io(?:\b)" 3; + "~*(?:\b)Zermelo(?:\b)" 3; + "~*(?:\b)Zeus(?:\b)" 3; + "~*(?:\b)zgrab(?:\b)" 3; + "~*(?:\b)Zitebot(?:\b)" 3; + "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZumBot(?:\b)" 3; + "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------- @@ -717,38 +717,38 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)adidxbot(?:\b|)" 0; - "~*(?:\b)AdsBot-Google(?:\b|)" 0; - "~*(?:\b)aolbuild(?:\b|)" 0; - "~*(?:\b)bingbot(?:\b|)" 0; - "~*(?:\b)bingpreview(?:\b|)" 0; - "~*(?:\b)DoCoMo(?:\b|)" 0; - "~*(?:\b)duckduckgo(?:\b|)" 0; - "~*(?:\b)facebookexternalhit(?:\b|)" 0; - "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; - "~*(?:\b)Googlebot(?:\b|)" 0; - "~*(?:\b)Googlebot-Image(?:\b|)" 0; - "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; - "~*(?:\b)Googlebot-News(?:\b|)" 0; - "~*(?:\b)Googlebot/Test(?:\b|)" 0; - "~*(?:\b)Googlebot-Video(?:\b|)" 0; - "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; - "~*(?:\b)Gravityscan(?:\b|)" 0; - "~*(?:\b)gsa-crawler(?:\b|)" 0; - "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; - "~*(?:\b)Kraken/0.1(?:\b|)" 0; - "~*(?:\b)LinkedInBot(?:\b|)" 0; - "~*(?:\b)Mediapartners-Google(?:\b|)" 0; - "~*(?:\b)msnbot(?:\b|)" 0; - "~*(?:\b)msnbot-media(?:\b|)" 0; - "~*(?:\b)SAMSUNG(?:\b|)" 0; - "~*(?:\b)Slackbot(?:\b|)" 0; - "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; - "~*(?:\b)slurp(?:\b|)" 0; - "~*(?:\b)teoma(?:\b|)" 0; - "~*(?:\b)TwitterBot(?:\b|)" 0; - "~*(?:\b)Wordpress(?:\b|)" 0; - "~*(?:\b)yahoo(?:\b|)" 0; + "~*(?:\b)adidxbot(?:\b)" 0; + "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bingbot(?:\b)" 0; + "~*(?:\b)bingpreview(?:\b)" 0; + "~*(?:\b)DoCoMo(?:\b)" 0; + "~*(?:\b)duckduckgo(?:\b)" 0; + "~*(?:\b)facebookexternalhit(?:\b)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b)" 0; + "~*(?:\b)Googlebot(?:\b)" 0; + "~*(?:\b)Googlebot-Image(?:\b)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b)" 0; + "~*(?:\b)Googlebot-News(?:\b)" 0; + "~*(?:\b)Googlebot/Test(?:\b)" 0; + "~*(?:\b)Googlebot-Video(?:\b)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; + "~*(?:\b)Gravityscan(?:\b)" 0; + "~*(?:\b)gsa-crawler(?:\b)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b)" 0; + "~*(?:\b)Kraken/0.1(?:\b)" 0; + "~*(?:\b)LinkedInBot(?:\b)" 0; + "~*(?:\b)Mediapartners-Google(?:\b)" 0; + "~*(?:\b)msnbot(?:\b)" 0; + "~*(?:\b)msnbot-media(?:\b)" 0; + "~*(?:\b)SAMSUNG(?:\b)" 0; + "~*(?:\b)Slackbot(?:\b)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 0; + "~*(?:\b)slurp(?:\b)" 0; + "~*(?:\b)teoma(?:\b)" 0; + "~*(?:\b)TwitterBot(?:\b)" 0; + "~*(?:\b)Wordpress(?:\b)" 0; + "~*(?:\b)yahoo(?:\b)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------------------- @@ -760,12 +760,12 @@ map $http_user_agent $bad_bot { #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)jetmon(?:\b|)" 1; - "~*(?:\b)libwww-perl(?:\b|)" 1; - "~*(?:\b)Lynx(?:\b|)" 1; - "~*(?:\b)munin(?:\b|)" 1; - "~*(?:\b)Presto(?:\b|)" 1; - "~*(?:\b)Wget/1.15(?:\b|)" 1; + "~*(?:\b)jetmon(?:\b)" 1; + "~*(?:\b)libwww-perl(?:\b)" 1; + "~*(?:\b)Lynx(?:\b)" 1; + "~*(?:\b)munin(?:\b)" 1; + "~*(?:\b)Presto(?:\b)" 1; + "~*(?:\b)Wget/1.15(?:\b)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ------------------------------------------------------- @@ -778,23 +778,23 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)Alexa(?:\b|)" 2; - "~*(?:\b)archive.org(?:\b|)" 2; - "~*(?:\b)Baidu(?:\b|)" 2; - "~*(?:\b)BUbiNG(?:\b|)" 2; - "~*(?:\b)FlipboardProxy(?:\b|)" 2; - "~*(?:\b)ia_archiver(?:\b|)" 2; - "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; - "~*(?:\b)Proximic(?:\b|)" 2; - "~*(?:\b)R6_CommentReader(?:\b|)" 2; - "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; - "~*(?:\b)RED/1(?:\b|)" 2; - "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; - "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; - "~*(?:\b)Spaidu(?:\b|)" 2; - "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; - "~*(?:\b)YandexBot(?:\b|)" 2; - "~*(?:\b)YandexImages(?:\b|)" 2; + "~*(?:\b)Alexa(?:\b)" 2; + "~*(?:\b)archive.org(?:\b)" 2; + "~*(?:\b)Baidu(?:\b)" 2; + "~*(?:\b)BUbiNG(?:\b)" 2; + "~*(?:\b)FlipboardProxy(?:\b)" 2; + "~*(?:\b)ia_archiver(?:\b)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b)" 2; + "~*(?:\b)Proximic(?:\b)" 2; + "~*(?:\b)R6_CommentReader(?:\b)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b)" 2; + "~*(?:\b)RED/1(?:\b)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b)" 2; + "~*(?:\b)Spaidu(?:\b)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b)" 2; + "~*(?:\b)YandexBot(?:\b)" 2; + "~*(?:\b)YandexImages(?:\b)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 2e2965ffc..12f18b91b 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.188.73.34 1; +35.202.145.110 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index d3d331ed4..c3c284efc 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.188.73.34 0; +35.202.145.110 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 7b00caacd..65f28ba04 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1704 -### Updated: Sat Jun 29 21:12:07 SAST 2019 +### Version: V3.2019.07.1705 +### Updated: Mon Jul 1 08:42:46 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### @@ -147,562 +147,562 @@ map $http_user_agent $bad_bot { # -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b|)" 3; - "~*(?:\b)404checker(?:\b|)" 3; - "~*(?:\b)404enemy(?:\b|)" 3; - "~*(?:\b)80legs(?:\b|)" 3; - "~*(?:\b)Abonti(?:\b|)" 3; - "~*(?:\b)Aboundex(?:\b|)" 3; - "~*(?:\b)Aboundexbot(?:\b|)" 3; - "~*(?:\b)Acunetix(?:\b|)" 3; - "~*(?:\b)ADmantX(?:\b|)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; - "~*(?:\b)AhrefsBot(?:\b|)" 3; - "~*(?:\b)AIBOT(?:\b|)" 3; - "~*(?:\b)AiHitBot(?:\b|)" 3; - "~*(?:\b)Aipbot(?:\b|)" 3; - "~*(?:\b)Alexibot(?:\b|)" 3; - "~*(?:\b)Alligator(?:\b|)" 3; - "~*(?:\b)AllSubmitter(?:\b|)" 3; - "~*(?:\b)AlphaBot(?:\b|)" 3; - "~*(?:\b)Anarchie(?:\b|)" 3; - "~*(?:\b)Apexoo(?:\b|)" 3; - "~*(?:\b)archive.org_bot(?:\b|)" 3; - "~*(?:\b)ASPSeek(?:\b|)" 3; - "~*(?:\b)Asterias(?:\b|)" 3; - "~*(?:\b)Attach(?:\b|)" 3; - "~*(?:\b)autoemailspider(?:\b|)" 3; - "~*(?:\b)BackDoorBot(?:\b|)" 3; - "~*(?:\b)Backlink-Ceck(?:\b|)" 3; - "~*(?:\b)backlink-check(?:\b|)" 3; - "~*(?:\b)BacklinkCrawler(?:\b|)" 3; - "~*(?:\b)BackStreet(?:\b|)" 3; - "~*(?:\b)BackWeb(?:\b|)" 3; - "~*(?:\b)Badass(?:\b|)" 3; - "~*(?:\b)Bandit(?:\b|)" 3; - "~*(?:\b)Barkrowler(?:\b|)" 3; - "~*(?:\b)BatchFTP(?:\b|)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; - "~*(?:\b)BBBike(?:\b|)" 3; - "~*(?:\b)BDCbot(?:\b|)" 3; - "~*(?:\b)BDFetch(?:\b|)" 3; - "~*(?:\b)BetaBot(?:\b|)" 3; - "~*(?:\b)Bigfoot(?:\b|)" 3; - "~*(?:\b)Bitacle(?:\b|)" 3; - "~*(?:\b)Blackboard(?:\b|)" 3; - "~*(?:\b)Black\ Hole(?:\b|)" 3; - "~*(?:\b)BlackWidow(?:\b|)" 3; - "~*(?:\b)BLEXBot(?:\b|)" 3; - "~*(?:\b)Blow(?:\b|)" 3; - "~*(?:\b)BlowFish(?:\b|)" 3; - "~*(?:\b)Boardreader(?:\b|)" 3; - "~*(?:\b)Bolt(?:\b|)" 3; - "~*(?:\b)BotALot(?:\b|)" 3; - "~*(?:\b)Brandprotect(?:\b|)" 3; - "~*(?:\b)Brandwatch(?:\b|)" 3; - "~*(?:\b)Buddy(?:\b|)" 3; - "~*(?:\b)BuiltBotTough(?:\b|)" 3; - "~*(?:\b)BuiltWith(?:\b|)" 3; - "~*(?:\b)Bullseye(?:\b|)" 3; - "~*(?:\b)BunnySlippers(?:\b|)" 3; - "~*(?:\b)BuzzSumo(?:\b|)" 3; - "~*(?:\b)Calculon(?:\b|)" 3; - "~*(?:\b)CATExplorador(?:\b|)" 3; - "~*(?:\b)CazoodleBot(?:\b|)" 3; - "~*(?:\b)CCBot(?:\b|)" 3; - "~*(?:\b)Cegbfeieh(?:\b|)" 3; - "~*(?:\b)CheeseBot(?:\b|)" 3; - "~*(?:\b)CherryPicker(?:\b|)" 3; - "~*(?:\b)CheTeam(?:\b|)" 3; - "~*(?:\b)ChinaClaw(?:\b|)" 3; - "~*(?:\b)Chlooe(?:\b|)" 3; - "~*(?:\b)Claritybot(?:\b|)" 3; - "~*(?:\b)Cliqzbot(?:\b|)" 3; - "~*(?:\b)Cloud\ mapping(?:\b|)" 3; - "~*(?:\b)coccocbot-web(?:\b|)" 3; - "~*(?:\b)Cogentbot(?:\b|)" 3; - "~*(?:\b)cognitiveseo(?:\b|)" 3; - "~*(?:\b)Collector(?:\b|)" 3; - "~*(?:\b)com.plumanalytics(?:\b|)" 3; - "~*(?:\b)Copier(?:\b|)" 3; - "~*(?:\b)CopyRightCheck(?:\b|)" 3; - "~*(?:\b)Copyscape(?:\b|)" 3; - "~*(?:\b)Cosmos(?:\b|)" 3; - "~*(?:\b)Craftbot(?:\b|)" 3; - "~*(?:\b)crawler4j(?:\b|)" 3; - "~*(?:\b)crawler.feedback(?:\b|)" 3; - "~*(?:\b)crawl.sogou.com(?:\b|)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; - "~*(?:\b)Crescent(?:\b|)" 3; - "~*(?:\b)CrunchBot(?:\b|)" 3; - "~*(?:\b)CSHttp(?:\b|)" 3; - "~*(?:\b)Curious(?:\b|)" 3; - "~*(?:\b)Custo(?:\b|)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; - "~*(?:\b)DataCha0s(?:\b|)" 3; - "~*(?:\b)DBLBot(?:\b|)" 3; - "~*(?:\b)demandbase-bot(?:\b|)" 3; - "~*(?:\b)Demon(?:\b|)" 3; - "~*(?:\b)Deusu(?:\b|)" 3; - "~*(?:\b)Devil(?:\b|)" 3; - "~*(?:\b)Digincore(?:\b|)" 3; - "~*(?:\b)DigitalPebble(?:\b|)" 3; - "~*(?:\b)DIIbot(?:\b|)" 3; - "~*(?:\b)Dirbuster(?:\b|)" 3; - "~*(?:\b)Disco(?:\b|)" 3; - "~*(?:\b)Discobot(?:\b|)" 3; - "~*(?:\b)Discoverybot(?:\b|)" 3; - "~*(?:\b)Dispatch(?:\b|)" 3; - "~*(?:\b)DittoSpyder(?:\b|)" 3; - "~*(?:\b)DnyzBot(?:\b|)" 3; - "~*(?:\b)DomainAppender(?:\b|)" 3; - "~*(?:\b)DomainCrawler(?:\b|)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; - "~*(?:\b)DomainStatsBot(?:\b|)" 3; - "~*(?:\b)Dotbot(?:\b|)" 3; - "~*(?:\b)Download\ Wonder(?:\b|)" 3; - "~*(?:\b)Dragonfly(?:\b|)" 3; - "~*(?:\b)Drip(?:\b|)" 3; - "~*(?:\b)DSearch(?:\b|)" 3; - "~*(?:\b)DTS\ Agent(?:\b|)" 3; - "~*(?:\b)EasyDL(?:\b|)" 3; - "~*(?:\b)Ebingbong(?:\b|)" 3; - "~*(?:\b)eCatch(?:\b|)" 3; - "~*(?:\b)ECCP/1.0(?:\b|)" 3; - "~*(?:\b)Ecxi(?:\b|)" 3; - "~*(?:\b)EirGrabber(?:\b|)" 3; - "~*(?:\b)EMail\ Siphon(?:\b|)" 3; - "~*(?:\b)EMail\ Wolf(?:\b|)" 3; - "~*(?:\b)EroCrawler(?:\b|)" 3; - "~*(?:\b)evc-batch(?:\b|)" 3; - "~*(?:\b)Evil(?:\b|)" 3; - "~*(?:\b)Exabot(?:\b|)" 3; - "~*(?:\b)Express\ WebPictures(?:\b|)" 3; - "~*(?:\b)ExtLinksBot(?:\b|)" 3; - "~*(?:\b)Extractor(?:\b|)" 3; - "~*(?:\b)ExtractorPro(?:\b|)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; - "~*(?:\b)EyeNetIE(?:\b|)" 3; - "~*(?:\b)Ezooms(?:\b|)" 3; - "~*(?:\b)facebookscraper(?:\b|)" 3; - "~*(?:\b)FDM(?:\b|)" 3; - "~*(?:\b)FemtosearchBot(?:\b|)" 3; - "~*(?:\b)FHscan(?:\b|)" 3; - "~*(?:\b)Fimap(?:\b|)" 3; - "~*(?:\b)Firefox/7.0(?:\b|)" 3; - "~*(?:\b)FlashGet(?:\b|)" 3; - "~*(?:\b)Flunky(?:\b|)" 3; - "~*(?:\b)Foobot(?:\b|)" 3; - "~*(?:\b)Freeuploader(?:\b|)" 3; - "~*(?:\b)FrontPage(?:\b|)" 3; - "~*(?:\b)FyberSpider(?:\b|)" 3; - "~*(?:\b)Fyrebot(?:\b|)" 3; - "~*(?:\b)GalaxyBot(?:\b|)" 3; - "~*(?:\b)Genieo(?:\b|)" 3; - "~*(?:\b)GermCrawler(?:\b|)" 3; - "~*(?:\b)Getintent(?:\b|)" 3; - "~*(?:\b)GetRight(?:\b|)" 3; - "~*(?:\b)GetWeb(?:\b|)" 3; - "~*(?:\b)Gigablast(?:\b|)" 3; - "~*(?:\b)Gigabot(?:\b|)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; - "~*(?:\b)Gotit(?:\b|)" 3; - "~*(?:\b)GoZilla(?:\b|)" 3; - "~*(?:\b)Go!Zilla(?:\b|)" 3; - "~*(?:\b)Grabber(?:\b|)" 3; - "~*(?:\b)GrabNet(?:\b|)" 3; - "~*(?:\b)Grafula(?:\b|)" 3; - "~*(?:\b)GrapeFX(?:\b|)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; - "~*(?:\b)GridBot(?:\b|)" 3; - "~*(?:\b)GT::WWW(?:\b|)" 3; - "~*(?:\b)Haansoft(?:\b|)" 3; - "~*(?:\b)HaosouSpider(?:\b|)" 3; - "~*(?:\b)Harvest(?:\b|)" 3; - "~*(?:\b)Havij(?:\b|)" 3; - "~*(?:\b)HEADMasterSEO(?:\b|)" 3; - "~*(?:\b)Heritrix(?:\b|)" 3; - "~*(?:\b)Hloader(?:\b|)" 3; - "~*(?:\b)HMView(?:\b|)" 3; - "~*(?:\b)HTMLparser(?:\b|)" 3; - "~*(?:\b)HTTP::Lite(?:\b|)" 3; - "~*(?:\b)HTTrack(?:\b|)" 3; - "~*(?:\b)Humanlinks(?:\b|)" 3; - "~*(?:\b)HybridBot(?:\b|)" 3; - "~*(?:\b)Iblog(?:\b|)" 3; - "~*(?:\b)IDBot(?:\b|)" 3; - "~*(?:\b)Id-search(?:\b|)" 3; - "~*(?:\b)IlseBot(?:\b|)" 3; - "~*(?:\b)Image\ Fetch(?:\b|)" 3; - "~*(?:\b)Image\ Sucker(?:\b|)" 3; - "~*(?:\b)IndeedBot(?:\b|)" 3; - "~*(?:\b)Indy\ Library(?:\b|)" 3; - "~*(?:\b)InfoNaviRobot(?:\b|)" 3; - "~*(?:\b)InfoTekies(?:\b|)" 3; - "~*(?:\b)instabid(?:\b|)" 3; - "~*(?:\b)Intelliseek(?:\b|)" 3; - "~*(?:\b)InterGET(?:\b|)" 3; - "~*(?:\b)Internet\ Ninja(?:\b|)" 3; - "~*(?:\b)InternetSeer(?:\b|)" 3; - "~*(?:\b)internetVista\ monitor(?:\b|)" 3; - "~*(?:\b)ips-agent(?:\b|)" 3; - "~*(?:\b)Iria(?:\b|)" 3; - "~*(?:\b)IRLbot(?:\b|)" 3; - "~*(?:\b)Iskanie(?:\b|)" 3; - "~*(?:\b)IstellaBot(?:\b|)" 3; - "~*(?:\b)JamesBOT(?:\b|)" 3; - "~*(?:\b)Jbrofuzz(?:\b|)" 3; - "~*(?:\b)JennyBot(?:\b|)" 3; - "~*(?:\b)JetCar(?:\b|)" 3; - "~*(?:\b)Jetty(?:\b|)" 3; - "~*(?:\b)JikeSpider(?:\b|)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; - "~*(?:\b)Joomla(?:\b|)" 3; - "~*(?:\b)Jorgee(?:\b|)" 3; - "~*(?:\b)JustView(?:\b|)" 3; - "~*(?:\b)Jyxobot(?:\b|)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; - "~*(?:\b)Keyword\ Density(?:\b|)" 3; - "~*(?:\b)Kozmosbot(?:\b|)" 3; - "~*(?:\b)Lanshanbot(?:\b|)" 3; - "~*(?:\b)Larbin(?:\b|)" 3; - "~*(?:\b)LeechFTP(?:\b|)" 3; - "~*(?:\b)LeechGet(?:\b|)" 3; - "~*(?:\b)LexiBot(?:\b|)" 3; - "~*(?:\b)Lftp(?:\b|)" 3; - "~*(?:\b)LibWeb(?:\b|)" 3; - "~*(?:\b)Libwhisker(?:\b|)" 3; - "~*(?:\b)Lightspeedsystems(?:\b|)" 3; - "~*(?:\b)Likse(?:\b|)" 3; - "~*(?:\b)Linkdexbot(?:\b|)" 3; - "~*(?:\b)LinkextractorPro(?:\b|)" 3; - "~*(?:\b)LinkpadBot(?:\b|)" 3; - "~*(?:\b)LinkScan(?:\b|)" 3; - "~*(?:\b)LinksManager(?:\b|)" 3; - "~*(?:\b)LinkWalker(?:\b|)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; - "~*(?:\b)Lipperhey(?:\b|)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; - "~*(?:\b)Litemage_walker(?:\b|)" 3; - "~*(?:\b)Lmspider(?:\b|)" 3; - "~*(?:\b)LNSpiderguy(?:\b|)" 3; - "~*(?:\b)Ltx71(?:\b|)" 3; - "~*(?:\b)lwp-request(?:\b|)" 3; - "~*(?:\b)LWP::Simple(?:\b|)" 3; - "~*(?:\b)lwp-trivial(?:\b|)" 3; - "~*(?:\b)Magnet(?:\b|)" 3; - "~*(?:\b)Mag-Net(?:\b|)" 3; - "~*(?:\b)magpie-crawler(?:\b|)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; - "~*(?:\b)Majestic12(?:\b|)" 3; - "~*(?:\b)Majestic-SEO(?:\b|)" 3; - "~*(?:\b)Majestic\ SEO(?:\b|)" 3; - "~*(?:\b)MarkMonitor(?:\b|)" 3; - "~*(?:\b)MarkWatch(?:\b|)" 3; - "~*(?:\b)Masscan(?:\b|)" 3; - "~*(?:\b)Mass\ Downloader(?:\b|)" 3; - "~*(?:\b)Mata\ Hari(?:\b|)" 3; - "~*(?:\b)MauiBot(?:\b|)" 3; - "~*(?:\b)meanpathbot(?:\b|)" 3; - "~*(?:\b)Meanpathbot(?:\b|)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; - "~*(?:\b)mediawords(?:\b|)" 3; - "~*(?:\b)MegaIndex.ru(?:\b|)" 3; - "~*(?:\b)Metauri(?:\b|)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; - "~*(?:\b)MIDown\ tool(?:\b|)" 3; - "~*(?:\b)MIIxpc(?:\b|)" 3; - "~*(?:\b)Mister\ PiX(?:\b|)" 3; - "~*(?:\b)MJ12bot(?:\b|)" 3; - "~*(?:\b)Mojeek(?:\b|)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; - "~*(?:\b)Mr.4x3(?:\b|)" 3; - "~*(?:\b)MSFrontPage(?:\b|)" 3; - "~*(?:\b)MSIECrawler(?:\b|)" 3; - "~*(?:\b)Msrabot(?:\b|)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; - "~*(?:\b)muhstik-scan(?:\b|)" 3; - "~*(?:\b)Musobot(?:\b|)" 3; - "~*(?:\b)Name\ Intelligence(?:\b|)" 3; - "~*(?:\b)Nameprotect(?:\b|)" 3; - "~*(?:\b)Navroad(?:\b|)" 3; - "~*(?:\b)NearSite(?:\b|)" 3; - "~*(?:\b)Needle(?:\b|)" 3; - "~*(?:\b)Nessus(?:\b|)" 3; - "~*(?:\b)NetAnts(?:\b|)" 3; - "~*(?:\b)Netcraft(?:\b|)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; - "~*(?:\b)NetLyzer(?:\b|)" 3; - "~*(?:\b)NetMechanic(?:\b|)" 3; - "~*(?:\b)NetSpider(?:\b|)" 3; - "~*(?:\b)Nettrack(?:\b|)" 3; - "~*(?:\b)Net\ Vampire(?:\b|)" 3; - "~*(?:\b)Netvibes(?:\b|)" 3; - "~*(?:\b)NetZIP(?:\b|)" 3; - "~*(?:\b)NextGenSearchBot(?:\b|)" 3; - "~*(?:\b)Nibbler(?:\b|)" 3; - "~*(?:\b)NICErsPRO(?:\b|)" 3; - "~*(?:\b)Niki-bot(?:\b|)" 3; - "~*(?:\b)Nikto(?:\b|)" 3; - "~*(?:\b)NimbleCrawler(?:\b|)" 3; - "~*(?:\b)Nimbostratus(?:\b|)" 3; - "~*(?:\b)Ninja(?:\b|)" 3; - "~*(?:\b)Nmap(?:\b|)" 3; - "~*(?:\b)NPbot(?:\b|)" 3; - "~*(?:\b)Nutch(?:\b|)" 3; - "~*(?:\b)oBot(?:\b|)" 3; - "~*(?:\b)Octopus(?:\b|)" 3; - "~*(?:\b)Offline\ Explorer(?:\b|)" 3; - "~*(?:\b)Offline\ Navigator(?:\b|)" 3; - "~*(?:\b)OnCrawl(?:\b|)" 3; - "~*(?:\b)Openfind(?:\b|)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; - "~*(?:\b)Openvas(?:\b|)" 3; - "~*(?:\b)OpenVAS(?:\b|)" 3; - "~*(?:\b)OrangeBot(?:\b|)" 3; - "~*(?:\b)OrangeSpider(?:\b|)" 3; - "~*(?:\b)OutclicksBot(?:\b|)" 3; - "~*(?:\b)OutfoxBot(?:\b|)" 3; - "~*(?:\b)PageAnalyzer(?:\b|)" 3; - "~*(?:\b)Page\ Analyzer(?:\b|)" 3; - "~*(?:\b)PageGrabber(?:\b|)" 3; - "~*(?:\b)page\ scorer(?:\b|)" 3; - "~*(?:\b)PageScorer(?:\b|)" 3; - "~*(?:\b)Pandalytics(?:\b|)" 3; - "~*(?:\b)Panscient(?:\b|)" 3; - "~*(?:\b)Papa\ Foto(?:\b|)" 3; - "~*(?:\b)Pavuk(?:\b|)" 3; - "~*(?:\b)pcBrowser(?:\b|)" 3; - "~*(?:\b)PECL::HTTP(?:\b|)" 3; - "~*(?:\b)PeoplePal(?:\b|)" 3; - "~*(?:\b)PHPCrawl(?:\b|)" 3; - "~*(?:\b)Picscout(?:\b|)" 3; - "~*(?:\b)Picsearch(?:\b|)" 3; - "~*(?:\b)PictureFinder(?:\b|)" 3; - "~*(?:\b)Pimonster(?:\b|)" 3; - "~*(?:\b)Pi-Monster(?:\b|)" 3; - "~*(?:\b)Pixray(?:\b|)" 3; - "~*(?:\b)PleaseCrawl(?:\b|)" 3; - "~*(?:\b)plumanalytics(?:\b|)" 3; - "~*(?:\b)Pockey(?:\b|)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; - "~*(?:\b)Probethenet(?:\b|)" 3; - "~*(?:\b)ProPowerBot(?:\b|)" 3; - "~*(?:\b)ProWebWalker(?:\b|)" 3; - "~*(?:\b)Psbot(?:\b|)" 3; - "~*(?:\b)Pump(?:\b|)" 3; - "~*(?:\b)PxBroker(?:\b|)" 3; - "~*(?:\b)PyCurl(?:\b|)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; - "~*(?:\b)Quick-Crawler(?:\b|)" 3; - "~*(?:\b)RankActive(?:\b|)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; - "~*(?:\b)RankFlex(?:\b|)" 3; - "~*(?:\b)RankingBot(?:\b|)" 3; - "~*(?:\b)RankingBot2(?:\b|)" 3; - "~*(?:\b)Rankivabot(?:\b|)" 3; - "~*(?:\b)RankurBot(?:\b|)" 3; - "~*(?:\b)RealDownload(?:\b|)" 3; - "~*(?:\b)Reaper(?:\b|)" 3; - "~*(?:\b)RebelMouse(?:\b|)" 3; - "~*(?:\b)Recorder(?:\b|)" 3; - "~*(?:\b)RedesScrapy(?:\b|)" 3; - "~*(?:\b)ReGet(?:\b|)" 3; - "~*(?:\b)RepoMonkey(?:\b|)" 3; - "~*(?:\b)Ripper(?:\b|)" 3; - "~*(?:\b)RocketCrawler(?:\b|)" 3; - "~*(?:\b)Rogerbot(?:\b|)" 3; - "~*(?:\b)s1z.ru(?:\b|)" 3; - "~*(?:\b)SalesIntelligent(?:\b|)" 3; - "~*(?:\b)SBIder(?:\b|)" 3; - "~*(?:\b)ScanAlert(?:\b|)" 3; - "~*(?:\b)Scanbot(?:\b|)" 3; - "~*(?:\b)scan.lol(?:\b|)" 3; - "~*(?:\b)ScoutJet(?:\b|)" 3; - "~*(?:\b)Scrapy(?:\b|)" 3; - "~*(?:\b)Screaming(?:\b|)" 3; - "~*(?:\b)ScreenerBot(?:\b|)" 3; - "~*(?:\b)Searchestate(?:\b|)" 3; - "~*(?:\b)SearchmetricsBot(?:\b|)" 3; - "~*(?:\b)Semrush(?:\b|)" 3; - "~*(?:\b)SemrushBot(?:\b|)" 3; - "~*(?:\b)SEOkicks(?:\b|)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; - "~*(?:\b)Seomoz(?:\b|)" 3; - "~*(?:\b)SEOprofiler(?:\b|)" 3; - "~*(?:\b)seoscanners(?:\b|)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; - "~*(?:\b)SEOstats(?:\b|)" 3; - "~*(?:\b)serpstatbot(?:\b|)" 3; - "~*(?:\b)sexsearcher(?:\b|)" 3; - "~*(?:\b)Shodan(?:\b|)" 3; - "~*(?:\b)Siphon(?:\b|)" 3; - "~*(?:\b)SISTRIX(?:\b|)" 3; - "~*(?:\b)Sitebeam(?:\b|)" 3; - "~*(?:\b)SiteExplorer(?:\b|)" 3; - "~*(?:\b)Siteimprove(?:\b|)" 3; - "~*(?:\b)SiteLockSpider(?:\b|)" 3; - "~*(?:\b)SiteSnagger(?:\b|)" 3; - "~*(?:\b)SiteSucker(?:\b|)" 3; - "~*(?:\b)Site\ Sucker(?:\b|)" 3; - "~*(?:\b)Sitevigil(?:\b|)" 3; - "~*(?:\b)SlySearch(?:\b|)" 3; - "~*(?:\b)SmartDownload(?:\b|)" 3; - "~*(?:\b)SMTBot(?:\b|)" 3; - "~*(?:\b)Snake(?:\b|)" 3; - "~*(?:\b)Snapbot(?:\b|)" 3; - "~*(?:\b)Snoopy(?:\b|)" 3; - "~*(?:\b)SocialRankIOBot(?:\b|)" 3; - "~*(?:\b)Sociscraper(?:\b|)" 3; - "~*(?:\b)sogouspider(?:\b|)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; - "~*(?:\b)Sosospider(?:\b|)" 3; - "~*(?:\b)Sottopop(?:\b|)" 3; - "~*(?:\b)SpaceBison(?:\b|)" 3; - "~*(?:\b)Spammen(?:\b|)" 3; - "~*(?:\b)SpankBot(?:\b|)" 3; - "~*(?:\b)Spanner(?:\b|)" 3; - "~*(?:\b)sp_auditbot(?:\b|)" 3; - "~*(?:\b)Spbot(?:\b|)" 3; - "~*(?:\b)Spinn3r(?:\b|)" 3; - "~*(?:\b)SputnikBot(?:\b|)" 3; - "~*(?:\b)spyfu(?:\b|)" 3; - "~*(?:\b)Sqlmap(?:\b|)" 3; - "~*(?:\b)Sqlworm(?:\b|)" 3; - "~*(?:\b)Sqworm(?:\b|)" 3; - "~*(?:\b)Steeler(?:\b|)" 3; - "~*(?:\b)Stripper(?:\b|)" 3; - "~*(?:\b)Sucker(?:\b|)" 3; - "~*(?:\b)Sucuri(?:\b|)" 3; - "~*(?:\b)SuperBot(?:\b|)" 3; - "~*(?:\b)SuperHTTP(?:\b|)" 3; - "~*(?:\b)Surfbot(?:\b|)" 3; - "~*(?:\b)SurveyBot(?:\b|)" 3; - "~*(?:\b)Suzuran(?:\b|)" 3; - "~*(?:\b)Swiftbot(?:\b|)" 3; - "~*(?:\b)sysscan(?:\b|)" 3; - "~*(?:\b)Szukacz(?:\b|)" 3; - "~*(?:\b)T0PHackTeam(?:\b|)" 3; - "~*(?:\b)T8Abot(?:\b|)" 3; - "~*(?:\b)tAkeOut(?:\b|)" 3; - "~*(?:\b)Teleport(?:\b|)" 3; - "~*(?:\b)TeleportPro(?:\b|)" 3; - "~*(?:\b)Telesoft(?:\b|)" 3; - "~*(?:\b)Telesphoreo(?:\b|)" 3; - "~*(?:\b)Telesphorep(?:\b|)" 3; - "~*(?:\b)The\ Intraformant(?:\b|)" 3; - "~*(?:\b)TheNomad(?:\b|)" 3; - "~*(?:\b)Thumbor(?:\b|)" 3; - "~*(?:\b)TightTwatBot(?:\b|)" 3; - "~*(?:\b)Titan(?:\b|)" 3; - "~*(?:\b)Toata(?:\b|)" 3; - "~*(?:\b)Toweyabot(?:\b|)" 3; - "~*(?:\b)Tracemyfile(?:\b|)" 3; - "~*(?:\b)Trendiction(?:\b|)" 3; - "~*(?:\b)Trendictionbot(?:\b|)" 3; - "~*(?:\b)trendiction.com(?:\b|)" 3; - "~*(?:\b)trendiction.de(?:\b|)" 3; - "~*(?:\b)True_Robot(?:\b|)" 3; - "~*(?:\b)Turingos(?:\b|)" 3; - "~*(?:\b)Turnitin(?:\b|)" 3; - "~*(?:\b)TurnitinBot(?:\b|)" 3; - "~*(?:\b)TwengaBot(?:\b|)" 3; - "~*(?:\b)Twice(?:\b|)" 3; - "~*(?:\b)Typhoeus(?:\b|)" 3; - "~*(?:\b)UnisterBot(?:\b|)" 3; - "~*(?:\b)Upflow(?:\b|)" 3; - "~*(?:\b)URLy.Warning(?:\b|)" 3; - "~*(?:\b)URLy\ Warning(?:\b|)" 3; - "~*(?:\b)Vacuum(?:\b|)" 3; - "~*(?:\b)Vagabondo(?:\b|)" 3; - "~*(?:\b)VB\ Project(?:\b|)" 3; - "~*(?:\b)VCI(?:\b|)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; - "~*(?:\b)VidibleScraper(?:\b|)" 3; - "~*(?:\b)Virusdie(?:\b|)" 3; - "~*(?:\b)VoidEYE(?:\b|)" 3; - "~*(?:\b)Voil(?:\b|)" 3; - "~*(?:\b)Voltron(?:\b|)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; - "~*(?:\b)WallpapersHD(?:\b|)" 3; - "~*(?:\b)WASALive-Bot(?:\b|)" 3; - "~*(?:\b)WBSearchBot(?:\b|)" 3; - "~*(?:\b)Webalta(?:\b|)" 3; - "~*(?:\b)WebAuto(?:\b|)" 3; - "~*(?:\b)Web\ Auto(?:\b|)" 3; - "~*(?:\b)WebBandit(?:\b|)" 3; - "~*(?:\b)WebCollage(?:\b|)" 3; - "~*(?:\b)Web\ Collage(?:\b|)" 3; - "~*(?:\b)WebCopier(?:\b|)" 3; - "~*(?:\b)WEBDAV(?:\b|)" 3; - "~*(?:\b)WebEnhancer(?:\b|)" 3; - "~*(?:\b)Web\ Enhancer(?:\b|)" 3; - "~*(?:\b)WebFetch(?:\b|)" 3; - "~*(?:\b)Web\ Fetch(?:\b|)" 3; - "~*(?:\b)WebFuck(?:\b|)" 3; - "~*(?:\b)Web\ Fuck(?:\b|)" 3; - "~*(?:\b)WebGo\ IS(?:\b|)" 3; - "~*(?:\b)WebImageCollector(?:\b|)" 3; - "~*(?:\b)WebLeacher(?:\b|)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; - "~*(?:\b)webmeup-crawler(?:\b|)" 3; - "~*(?:\b)WebPix(?:\b|)" 3; - "~*(?:\b)Web\ Pix(?:\b|)" 3; - "~*(?:\b)WebReaper(?:\b|)" 3; - "~*(?:\b)WebSauger(?:\b|)" 3; - "~*(?:\b)Web\ Sauger(?:\b|)" 3; - "~*(?:\b)Webshag(?:\b|)" 3; - "~*(?:\b)WebsiteExtractor(?:\b|)" 3; - "~*(?:\b)WebsiteQuester(?:\b|)" 3; - "~*(?:\b)Website\ Quester(?:\b|)" 3; - "~*(?:\b)Webster(?:\b|)" 3; - "~*(?:\b)WebStripper(?:\b|)" 3; - "~*(?:\b)WebSucker(?:\b|)" 3; - "~*(?:\b)Web\ Sucker(?:\b|)" 3; - "~*(?:\b)WebWhacker(?:\b|)" 3; - "~*(?:\b)WebZIP(?:\b|)" 3; - "~*(?:\b)WeSEE(?:\b|)" 3; - "~*(?:\b)Whack(?:\b|)" 3; - "~*(?:\b)Whacker(?:\b|)" 3; - "~*(?:\b)Whatweb(?:\b|)" 3; - "~*(?:\b)Who.is\ Bot(?:\b|)" 3; - "~*(?:\b)Widow(?:\b|)" 3; - "~*(?:\b)WinHTTrack(?:\b|)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; - "~*(?:\b)WISENutbot(?:\b|)" 3; - "~*(?:\b)Wonderbot(?:\b|)" 3; - "~*(?:\b)Woobot(?:\b|)" 3; - "~*(?:\b)Wotbox(?:\b|)" 3; - "~*(?:\b)Wprecon(?:\b|)" 3; - "~*(?:\b)WPScan(?:\b|)" 3; - "~*(?:\b)WWW-Collector-E(?:\b|)" 3; - "~*(?:\b)WWW-Mechanize(?:\b|)" 3; - "~*(?:\b)WWW::Mechanize(?:\b|)" 3; - "~*(?:\b)WWWOFFLE(?:\b|)" 3; - "~*(?:\b)x09Mozilla(?:\b|)" 3; - "~*(?:\b)x22Mozilla(?:\b|)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; - "~*(?:\b)Xenu(?:\b|)" 3; - "~*(?:\b)xpymep1.exe(?:\b|)" 3; - "~*(?:\b)YoudaoBot(?:\b|)" 3; - "~*(?:\b)Zade(?:\b|)" 3; - "~*(?:\b)Zauba(?:\b|)" 3; - "~*(?:\b)zauba.io(?:\b|)" 3; - "~*(?:\b)Zermelo(?:\b|)" 3; - "~*(?:\b)Zeus(?:\b|)" 3; - "~*(?:\b)zgrab(?:\b|)" 3; - "~*(?:\b)Zitebot(?:\b|)" 3; - "~*(?:\b)ZmEu(?:\b|)" 3; - "~*(?:\b)ZumBot(?:\b|)" 3; - "~*(?:\b)ZyBorg(?:\b|)" 3; + "~*(?:\b)360Spider(?:\b)" 3; + "~*(?:\b)404checker(?:\b)" 3; + "~*(?:\b)404enemy(?:\b)" 3; + "~*(?:\b)80legs(?:\b)" 3; + "~*(?:\b)Abonti(?:\b)" 3; + "~*(?:\b)Aboundex(?:\b)" 3; + "~*(?:\b)Aboundexbot(?:\b)" 3; + "~*(?:\b)Acunetix(?:\b)" 3; + "~*(?:\b)ADmantX(?:\b)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; + "~*(?:\b)AhrefsBot(?:\b)" 3; + "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)AiHitBot(?:\b)" 3; + "~*(?:\b)Aipbot(?:\b)" 3; + "~*(?:\b)Alexibot(?:\b)" 3; + "~*(?:\b)Alligator(?:\b)" 3; + "~*(?:\b)AllSubmitter(?:\b)" 3; + "~*(?:\b)AlphaBot(?:\b)" 3; + "~*(?:\b)Anarchie(?:\b)" 3; + "~*(?:\b)Apexoo(?:\b)" 3; + "~*(?:\b)archive.org_bot(?:\b)" 3; + "~*(?:\b)ASPSeek(?:\b)" 3; + "~*(?:\b)Asterias(?:\b)" 3; + "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)BackDoorBot(?:\b)" 3; + "~*(?:\b)Backlink-Ceck(?:\b)" 3; + "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)BacklinkCrawler(?:\b)" 3; + "~*(?:\b)BackStreet(?:\b)" 3; + "~*(?:\b)BackWeb(?:\b)" 3; + "~*(?:\b)Badass(?:\b)" 3; + "~*(?:\b)Bandit(?:\b)" 3; + "~*(?:\b)Barkrowler(?:\b)" 3; + "~*(?:\b)BatchFTP(?:\b)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; + "~*(?:\b)BBBike(?:\b)" 3; + "~*(?:\b)BDCbot(?:\b)" 3; + "~*(?:\b)BDFetch(?:\b)" 3; + "~*(?:\b)BetaBot(?:\b)" 3; + "~*(?:\b)Bigfoot(?:\b)" 3; + "~*(?:\b)Bitacle(?:\b)" 3; + "~*(?:\b)Blackboard(?:\b)" 3; + "~*(?:\b)Black\ Hole(?:\b)" 3; + "~*(?:\b)BlackWidow(?:\b)" 3; + "~*(?:\b)BLEXBot(?:\b)" 3; + "~*(?:\b)Blow(?:\b)" 3; + "~*(?:\b)BlowFish(?:\b)" 3; + "~*(?:\b)Boardreader(?:\b)" 3; + "~*(?:\b)Bolt(?:\b)" 3; + "~*(?:\b)BotALot(?:\b)" 3; + "~*(?:\b)Brandprotect(?:\b)" 3; + "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buddy(?:\b)" 3; + "~*(?:\b)BuiltBotTough(?:\b)" 3; + "~*(?:\b)BuiltWith(?:\b)" 3; + "~*(?:\b)Bullseye(?:\b)" 3; + "~*(?:\b)BunnySlippers(?:\b)" 3; + "~*(?:\b)BuzzSumo(?:\b)" 3; + "~*(?:\b)Calculon(?:\b)" 3; + "~*(?:\b)CATExplorador(?:\b)" 3; + "~*(?:\b)CazoodleBot(?:\b)" 3; + "~*(?:\b)CCBot(?:\b)" 3; + "~*(?:\b)Cegbfeieh(?:\b)" 3; + "~*(?:\b)CheeseBot(?:\b)" 3; + "~*(?:\b)CherryPicker(?:\b)" 3; + "~*(?:\b)CheTeam(?:\b)" 3; + "~*(?:\b)ChinaClaw(?:\b)" 3; + "~*(?:\b)Chlooe(?:\b)" 3; + "~*(?:\b)Claritybot(?:\b)" 3; + "~*(?:\b)Cliqzbot(?:\b)" 3; + "~*(?:\b)Cloud\ mapping(?:\b)" 3; + "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cogentbot(?:\b)" 3; + "~*(?:\b)cognitiveseo(?:\b)" 3; + "~*(?:\b)Collector(?:\b)" 3; + "~*(?:\b)com.plumanalytics(?:\b)" 3; + "~*(?:\b)Copier(?:\b)" 3; + "~*(?:\b)CopyRightCheck(?:\b)" 3; + "~*(?:\b)Copyscape(?:\b)" 3; + "~*(?:\b)Cosmos(?:\b)" 3; + "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)crawler4j(?:\b)" 3; + "~*(?:\b)crawler.feedback(?:\b)" 3; + "~*(?:\b)crawl.sogou.com(?:\b)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b)" 3; + "~*(?:\b)Crescent(?:\b)" 3; + "~*(?:\b)CrunchBot(?:\b)" 3; + "~*(?:\b)CSHttp(?:\b)" 3; + "~*(?:\b)Curious(?:\b)" 3; + "~*(?:\b)Custo(?:\b)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; + "~*(?:\b)DataCha0s(?:\b)" 3; + "~*(?:\b)DBLBot(?:\b)" 3; + "~*(?:\b)demandbase-bot(?:\b)" 3; + "~*(?:\b)Demon(?:\b)" 3; + "~*(?:\b)Deusu(?:\b)" 3; + "~*(?:\b)Devil(?:\b)" 3; + "~*(?:\b)Digincore(?:\b)" 3; + "~*(?:\b)DigitalPebble(?:\b)" 3; + "~*(?:\b)DIIbot(?:\b)" 3; + "~*(?:\b)Dirbuster(?:\b)" 3; + "~*(?:\b)Disco(?:\b)" 3; + "~*(?:\b)Discobot(?:\b)" 3; + "~*(?:\b)Discoverybot(?:\b)" 3; + "~*(?:\b)Dispatch(?:\b)" 3; + "~*(?:\b)DittoSpyder(?:\b)" 3; + "~*(?:\b)DnyzBot(?:\b)" 3; + "~*(?:\b)DomainAppender(?:\b)" 3; + "~*(?:\b)DomainCrawler(?:\b)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)DomainStatsBot(?:\b)" 3; + "~*(?:\b)Dotbot(?:\b)" 3; + "~*(?:\b)Download\ Wonder(?:\b)" 3; + "~*(?:\b)Dragonfly(?:\b)" 3; + "~*(?:\b)Drip(?:\b)" 3; + "~*(?:\b)DSearch(?:\b)" 3; + "~*(?:\b)DTS\ Agent(?:\b)" 3; + "~*(?:\b)EasyDL(?:\b)" 3; + "~*(?:\b)Ebingbong(?:\b)" 3; + "~*(?:\b)eCatch(?:\b)" 3; + "~*(?:\b)ECCP/1.0(?:\b)" 3; + "~*(?:\b)Ecxi(?:\b)" 3; + "~*(?:\b)EirGrabber(?:\b)" 3; + "~*(?:\b)EMail\ Siphon(?:\b)" 3; + "~*(?:\b)EMail\ Wolf(?:\b)" 3; + "~*(?:\b)EroCrawler(?:\b)" 3; + "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)Evil(?:\b)" 3; + "~*(?:\b)Exabot(?:\b)" 3; + "~*(?:\b)Express\ WebPictures(?:\b)" 3; + "~*(?:\b)ExtLinksBot(?:\b)" 3; + "~*(?:\b)Extractor(?:\b)" 3; + "~*(?:\b)ExtractorPro(?:\b)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; + "~*(?:\b)EyeNetIE(?:\b)" 3; + "~*(?:\b)Ezooms(?:\b)" 3; + "~*(?:\b)facebookscraper(?:\b)" 3; + "~*(?:\b)FDM(?:\b)" 3; + "~*(?:\b)FemtosearchBot(?:\b)" 3; + "~*(?:\b)FHscan(?:\b)" 3; + "~*(?:\b)Fimap(?:\b)" 3; + "~*(?:\b)Firefox/7.0(?:\b)" 3; + "~*(?:\b)FlashGet(?:\b)" 3; + "~*(?:\b)Flunky(?:\b)" 3; + "~*(?:\b)Foobot(?:\b)" 3; + "~*(?:\b)Freeuploader(?:\b)" 3; + "~*(?:\b)FrontPage(?:\b)" 3; + "~*(?:\b)FyberSpider(?:\b)" 3; + "~*(?:\b)Fyrebot(?:\b)" 3; + "~*(?:\b)GalaxyBot(?:\b)" 3; + "~*(?:\b)Genieo(?:\b)" 3; + "~*(?:\b)GermCrawler(?:\b)" 3; + "~*(?:\b)Getintent(?:\b)" 3; + "~*(?:\b)GetRight(?:\b)" 3; + "~*(?:\b)GetWeb(?:\b)" 3; + "~*(?:\b)Gigablast(?:\b)" 3; + "~*(?:\b)Gigabot(?:\b)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; + "~*(?:\b)Gotit(?:\b)" 3; + "~*(?:\b)GoZilla(?:\b)" 3; + "~*(?:\b)Go!Zilla(?:\b)" 3; + "~*(?:\b)Grabber(?:\b)" 3; + "~*(?:\b)GrabNet(?:\b)" 3; + "~*(?:\b)Grafula(?:\b)" 3; + "~*(?:\b)GrapeFX(?:\b)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b)" 3; + "~*(?:\b)GridBot(?:\b)" 3; + "~*(?:\b)GT::WWW(?:\b)" 3; + "~*(?:\b)Haansoft(?:\b)" 3; + "~*(?:\b)HaosouSpider(?:\b)" 3; + "~*(?:\b)Harvest(?:\b)" 3; + "~*(?:\b)Havij(?:\b)" 3; + "~*(?:\b)HEADMasterSEO(?:\b)" 3; + "~*(?:\b)Heritrix(?:\b)" 3; + "~*(?:\b)Hloader(?:\b)" 3; + "~*(?:\b)HMView(?:\b)" 3; + "~*(?:\b)HTMLparser(?:\b)" 3; + "~*(?:\b)HTTP::Lite(?:\b)" 3; + "~*(?:\b)HTTrack(?:\b)" 3; + "~*(?:\b)Humanlinks(?:\b)" 3; + "~*(?:\b)HybridBot(?:\b)" 3; + "~*(?:\b)Iblog(?:\b)" 3; + "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)Id-search(?:\b)" 3; + "~*(?:\b)IlseBot(?:\b)" 3; + "~*(?:\b)Image\ Fetch(?:\b)" 3; + "~*(?:\b)Image\ Sucker(?:\b)" 3; + "~*(?:\b)IndeedBot(?:\b)" 3; + "~*(?:\b)Indy\ Library(?:\b)" 3; + "~*(?:\b)InfoNaviRobot(?:\b)" 3; + "~*(?:\b)InfoTekies(?:\b)" 3; + "~*(?:\b)instabid(?:\b)" 3; + "~*(?:\b)Intelliseek(?:\b)" 3; + "~*(?:\b)InterGET(?:\b)" 3; + "~*(?:\b)Internet\ Ninja(?:\b)" 3; + "~*(?:\b)InternetSeer(?:\b)" 3; + "~*(?:\b)internetVista\ monitor(?:\b)" 3; + "~*(?:\b)ips-agent(?:\b)" 3; + "~*(?:\b)Iria(?:\b)" 3; + "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)Iskanie(?:\b)" 3; + "~*(?:\b)IstellaBot(?:\b)" 3; + "~*(?:\b)JamesBOT(?:\b)" 3; + "~*(?:\b)Jbrofuzz(?:\b)" 3; + "~*(?:\b)JennyBot(?:\b)" 3; + "~*(?:\b)JetCar(?:\b)" 3; + "~*(?:\b)Jetty(?:\b)" 3; + "~*(?:\b)JikeSpider(?:\b)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; + "~*(?:\b)Joomla(?:\b)" 3; + "~*(?:\b)Jorgee(?:\b)" 3; + "~*(?:\b)JustView(?:\b)" 3; + "~*(?:\b)Jyxobot(?:\b)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b)" 3; + "~*(?:\b)Keyword\ Density(?:\b)" 3; + "~*(?:\b)Kozmosbot(?:\b)" 3; + "~*(?:\b)Lanshanbot(?:\b)" 3; + "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)LeechFTP(?:\b)" 3; + "~*(?:\b)LeechGet(?:\b)" 3; + "~*(?:\b)LexiBot(?:\b)" 3; + "~*(?:\b)Lftp(?:\b)" 3; + "~*(?:\b)LibWeb(?:\b)" 3; + "~*(?:\b)Libwhisker(?:\b)" 3; + "~*(?:\b)Lightspeedsystems(?:\b)" 3; + "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkdexbot(?:\b)" 3; + "~*(?:\b)LinkextractorPro(?:\b)" 3; + "~*(?:\b)LinkpadBot(?:\b)" 3; + "~*(?:\b)LinkScan(?:\b)" 3; + "~*(?:\b)LinksManager(?:\b)" 3; + "~*(?:\b)LinkWalker(?:\b)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; + "~*(?:\b)Lipperhey(?:\b)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; + "~*(?:\b)Litemage_walker(?:\b)" 3; + "~*(?:\b)Lmspider(?:\b)" 3; + "~*(?:\b)LNSpiderguy(?:\b)" 3; + "~*(?:\b)Ltx71(?:\b)" 3; + "~*(?:\b)lwp-request(?:\b)" 3; + "~*(?:\b)LWP::Simple(?:\b)" 3; + "~*(?:\b)lwp-trivial(?:\b)" 3; + "~*(?:\b)Magnet(?:\b)" 3; + "~*(?:\b)Mag-Net(?:\b)" 3; + "~*(?:\b)magpie-crawler(?:\b)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b)" 3; + "~*(?:\b)Majestic12(?:\b)" 3; + "~*(?:\b)Majestic-SEO(?:\b)" 3; + "~*(?:\b)Majestic\ SEO(?:\b)" 3; + "~*(?:\b)MarkMonitor(?:\b)" 3; + "~*(?:\b)MarkWatch(?:\b)" 3; + "~*(?:\b)Masscan(?:\b)" 3; + "~*(?:\b)Mass\ Downloader(?:\b)" 3; + "~*(?:\b)Mata\ Hari(?:\b)" 3; + "~*(?:\b)MauiBot(?:\b)" 3; + "~*(?:\b)meanpathbot(?:\b)" 3; + "~*(?:\b)Meanpathbot(?:\b)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b)" 3; + "~*(?:\b)mediawords(?:\b)" 3; + "~*(?:\b)MegaIndex.ru(?:\b)" 3; + "~*(?:\b)Metauri(?:\b)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; + "~*(?:\b)MIDown\ tool(?:\b)" 3; + "~*(?:\b)MIIxpc(?:\b)" 3; + "~*(?:\b)Mister\ PiX(?:\b)" 3; + "~*(?:\b)MJ12bot(?:\b)" 3; + "~*(?:\b)Mojeek(?:\b)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; + "~*(?:\b)Mr.4x3(?:\b)" 3; + "~*(?:\b)MSFrontPage(?:\b)" 3; + "~*(?:\b)MSIECrawler(?:\b)" 3; + "~*(?:\b)Msrabot(?:\b)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; + "~*(?:\b)muhstik-scan(?:\b)" 3; + "~*(?:\b)Musobot(?:\b)" 3; + "~*(?:\b)Name\ Intelligence(?:\b)" 3; + "~*(?:\b)Nameprotect(?:\b)" 3; + "~*(?:\b)Navroad(?:\b)" 3; + "~*(?:\b)NearSite(?:\b)" 3; + "~*(?:\b)Needle(?:\b)" 3; + "~*(?:\b)Nessus(?:\b)" 3; + "~*(?:\b)NetAnts(?:\b)" 3; + "~*(?:\b)Netcraft(?:\b)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; + "~*(?:\b)NetLyzer(?:\b)" 3; + "~*(?:\b)NetMechanic(?:\b)" 3; + "~*(?:\b)NetSpider(?:\b)" 3; + "~*(?:\b)Nettrack(?:\b)" 3; + "~*(?:\b)Net\ Vampire(?:\b)" 3; + "~*(?:\b)Netvibes(?:\b)" 3; + "~*(?:\b)NetZIP(?:\b)" 3; + "~*(?:\b)NextGenSearchBot(?:\b)" 3; + "~*(?:\b)Nibbler(?:\b)" 3; + "~*(?:\b)NICErsPRO(?:\b)" 3; + "~*(?:\b)Niki-bot(?:\b)" 3; + "~*(?:\b)Nikto(?:\b)" 3; + "~*(?:\b)NimbleCrawler(?:\b)" 3; + "~*(?:\b)Nimbostratus(?:\b)" 3; + "~*(?:\b)Ninja(?:\b)" 3; + "~*(?:\b)Nmap(?:\b)" 3; + "~*(?:\b)NPbot(?:\b)" 3; + "~*(?:\b)Nutch(?:\b)" 3; + "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)Octopus(?:\b)" 3; + "~*(?:\b)Offline\ Explorer(?:\b)" 3; + "~*(?:\b)Offline\ Navigator(?:\b)" 3; + "~*(?:\b)OnCrawl(?:\b)" 3; + "~*(?:\b)Openfind(?:\b)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b)" 3; + "~*(?:\b)Openvas(?:\b)" 3; + "~*(?:\b)OpenVAS(?:\b)" 3; + "~*(?:\b)OrangeBot(?:\b)" 3; + "~*(?:\b)OrangeSpider(?:\b)" 3; + "~*(?:\b)OutclicksBot(?:\b)" 3; + "~*(?:\b)OutfoxBot(?:\b)" 3; + "~*(?:\b)PageAnalyzer(?:\b)" 3; + "~*(?:\b)Page\ Analyzer(?:\b)" 3; + "~*(?:\b)PageGrabber(?:\b)" 3; + "~*(?:\b)page\ scorer(?:\b)" 3; + "~*(?:\b)PageScorer(?:\b)" 3; + "~*(?:\b)Pandalytics(?:\b)" 3; + "~*(?:\b)Panscient(?:\b)" 3; + "~*(?:\b)Papa\ Foto(?:\b)" 3; + "~*(?:\b)Pavuk(?:\b)" 3; + "~*(?:\b)pcBrowser(?:\b)" 3; + "~*(?:\b)PECL::HTTP(?:\b)" 3; + "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)PHPCrawl(?:\b)" 3; + "~*(?:\b)Picscout(?:\b)" 3; + "~*(?:\b)Picsearch(?:\b)" 3; + "~*(?:\b)PictureFinder(?:\b)" 3; + "~*(?:\b)Pimonster(?:\b)" 3; + "~*(?:\b)Pi-Monster(?:\b)" 3; + "~*(?:\b)Pixray(?:\b)" 3; + "~*(?:\b)PleaseCrawl(?:\b)" 3; + "~*(?:\b)plumanalytics(?:\b)" 3; + "~*(?:\b)Pockey(?:\b)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; + "~*(?:\b)Probethenet(?:\b)" 3; + "~*(?:\b)ProPowerBot(?:\b)" 3; + "~*(?:\b)ProWebWalker(?:\b)" 3; + "~*(?:\b)Psbot(?:\b)" 3; + "~*(?:\b)Pump(?:\b)" 3; + "~*(?:\b)PxBroker(?:\b)" 3; + "~*(?:\b)PyCurl(?:\b)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; + "~*(?:\b)Quick-Crawler(?:\b)" 3; + "~*(?:\b)RankActive(?:\b)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b)" 3; + "~*(?:\b)RankFlex(?:\b)" 3; + "~*(?:\b)RankingBot(?:\b)" 3; + "~*(?:\b)RankingBot2(?:\b)" 3; + "~*(?:\b)Rankivabot(?:\b)" 3; + "~*(?:\b)RankurBot(?:\b)" 3; + "~*(?:\b)RealDownload(?:\b)" 3; + "~*(?:\b)Reaper(?:\b)" 3; + "~*(?:\b)RebelMouse(?:\b)" 3; + "~*(?:\b)Recorder(?:\b)" 3; + "~*(?:\b)RedesScrapy(?:\b)" 3; + "~*(?:\b)ReGet(?:\b)" 3; + "~*(?:\b)RepoMonkey(?:\b)" 3; + "~*(?:\b)Ripper(?:\b)" 3; + "~*(?:\b)RocketCrawler(?:\b)" 3; + "~*(?:\b)Rogerbot(?:\b)" 3; + "~*(?:\b)s1z.ru(?:\b)" 3; + "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)SBIder(?:\b)" 3; + "~*(?:\b)ScanAlert(?:\b)" 3; + "~*(?:\b)Scanbot(?:\b)" 3; + "~*(?:\b)scan.lol(?:\b)" 3; + "~*(?:\b)ScoutJet(?:\b)" 3; + "~*(?:\b)Scrapy(?:\b)" 3; + "~*(?:\b)Screaming(?:\b)" 3; + "~*(?:\b)ScreenerBot(?:\b)" 3; + "~*(?:\b)Searchestate(?:\b)" 3; + "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Semrush(?:\b)" 3; + "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SEOkicks(?:\b)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; + "~*(?:\b)Seomoz(?:\b)" 3; + "~*(?:\b)SEOprofiler(?:\b)" 3; + "~*(?:\b)seoscanners(?:\b)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b)" 3; + "~*(?:\b)SEOstats(?:\b)" 3; + "~*(?:\b)serpstatbot(?:\b)" 3; + "~*(?:\b)sexsearcher(?:\b)" 3; + "~*(?:\b)Shodan(?:\b)" 3; + "~*(?:\b)Siphon(?:\b)" 3; + "~*(?:\b)SISTRIX(?:\b)" 3; + "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteExplorer(?:\b)" 3; + "~*(?:\b)Siteimprove(?:\b)" 3; + "~*(?:\b)SiteLockSpider(?:\b)" 3; + "~*(?:\b)SiteSnagger(?:\b)" 3; + "~*(?:\b)SiteSucker(?:\b)" 3; + "~*(?:\b)Site\ Sucker(?:\b)" 3; + "~*(?:\b)Sitevigil(?:\b)" 3; + "~*(?:\b)SlySearch(?:\b)" 3; + "~*(?:\b)SmartDownload(?:\b)" 3; + "~*(?:\b)SMTBot(?:\b)" 3; + "~*(?:\b)Snake(?:\b)" 3; + "~*(?:\b)Snapbot(?:\b)" 3; + "~*(?:\b)Snoopy(?:\b)" 3; + "~*(?:\b)SocialRankIOBot(?:\b)" 3; + "~*(?:\b)Sociscraper(?:\b)" 3; + "~*(?:\b)sogouspider(?:\b)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; + "~*(?:\b)Sosospider(?:\b)" 3; + "~*(?:\b)Sottopop(?:\b)" 3; + "~*(?:\b)SpaceBison(?:\b)" 3; + "~*(?:\b)Spammen(?:\b)" 3; + "~*(?:\b)SpankBot(?:\b)" 3; + "~*(?:\b)Spanner(?:\b)" 3; + "~*(?:\b)sp_auditbot(?:\b)" 3; + "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spinn3r(?:\b)" 3; + "~*(?:\b)SputnikBot(?:\b)" 3; + "~*(?:\b)spyfu(?:\b)" 3; + "~*(?:\b)Sqlmap(?:\b)" 3; + "~*(?:\b)Sqlworm(?:\b)" 3; + "~*(?:\b)Sqworm(?:\b)" 3; + "~*(?:\b)Steeler(?:\b)" 3; + "~*(?:\b)Stripper(?:\b)" 3; + "~*(?:\b)Sucker(?:\b)" 3; + "~*(?:\b)Sucuri(?:\b)" 3; + "~*(?:\b)SuperBot(?:\b)" 3; + "~*(?:\b)SuperHTTP(?:\b)" 3; + "~*(?:\b)Surfbot(?:\b)" 3; + "~*(?:\b)SurveyBot(?:\b)" 3; + "~*(?:\b)Suzuran(?:\b)" 3; + "~*(?:\b)Swiftbot(?:\b)" 3; + "~*(?:\b)sysscan(?:\b)" 3; + "~*(?:\b)Szukacz(?:\b)" 3; + "~*(?:\b)T0PHackTeam(?:\b)" 3; + "~*(?:\b)T8Abot(?:\b)" 3; + "~*(?:\b)tAkeOut(?:\b)" 3; + "~*(?:\b)Teleport(?:\b)" 3; + "~*(?:\b)TeleportPro(?:\b)" 3; + "~*(?:\b)Telesoft(?:\b)" 3; + "~*(?:\b)Telesphoreo(?:\b)" 3; + "~*(?:\b)Telesphorep(?:\b)" 3; + "~*(?:\b)The\ Intraformant(?:\b)" 3; + "~*(?:\b)TheNomad(?:\b)" 3; + "~*(?:\b)Thumbor(?:\b)" 3; + "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)Titan(?:\b)" 3; + "~*(?:\b)Toata(?:\b)" 3; + "~*(?:\b)Toweyabot(?:\b)" 3; + "~*(?:\b)Tracemyfile(?:\b)" 3; + "~*(?:\b)Trendiction(?:\b)" 3; + "~*(?:\b)Trendictionbot(?:\b)" 3; + "~*(?:\b)trendiction.com(?:\b)" 3; + "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)True_Robot(?:\b)" 3; + "~*(?:\b)Turingos(?:\b)" 3; + "~*(?:\b)Turnitin(?:\b)" 3; + "~*(?:\b)TurnitinBot(?:\b)" 3; + "~*(?:\b)TwengaBot(?:\b)" 3; + "~*(?:\b)Twice(?:\b)" 3; + "~*(?:\b)Typhoeus(?:\b)" 3; + "~*(?:\b)UnisterBot(?:\b)" 3; + "~*(?:\b)Upflow(?:\b)" 3; + "~*(?:\b)URLy.Warning(?:\b)" 3; + "~*(?:\b)URLy\ Warning(?:\b)" 3; + "~*(?:\b)Vacuum(?:\b)" 3; + "~*(?:\b)Vagabondo(?:\b)" 3; + "~*(?:\b)VB\ Project(?:\b)" 3; + "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b)" 3; + "~*(?:\b)VidibleScraper(?:\b)" 3; + "~*(?:\b)Virusdie(?:\b)" 3; + "~*(?:\b)VoidEYE(?:\b)" 3; + "~*(?:\b)Voil(?:\b)" 3; + "~*(?:\b)Voltron(?:\b)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b)" 3; + "~*(?:\b)WallpapersHD(?:\b)" 3; + "~*(?:\b)WASALive-Bot(?:\b)" 3; + "~*(?:\b)WBSearchBot(?:\b)" 3; + "~*(?:\b)Webalta(?:\b)" 3; + "~*(?:\b)WebAuto(?:\b)" 3; + "~*(?:\b)Web\ Auto(?:\b)" 3; + "~*(?:\b)WebBandit(?:\b)" 3; + "~*(?:\b)WebCollage(?:\b)" 3; + "~*(?:\b)Web\ Collage(?:\b)" 3; + "~*(?:\b)WebCopier(?:\b)" 3; + "~*(?:\b)WEBDAV(?:\b)" 3; + "~*(?:\b)WebEnhancer(?:\b)" 3; + "~*(?:\b)Web\ Enhancer(?:\b)" 3; + "~*(?:\b)WebFetch(?:\b)" 3; + "~*(?:\b)Web\ Fetch(?:\b)" 3; + "~*(?:\b)WebFuck(?:\b)" 3; + "~*(?:\b)Web\ Fuck(?:\b)" 3; + "~*(?:\b)WebGo\ IS(?:\b)" 3; + "~*(?:\b)WebImageCollector(?:\b)" 3; + "~*(?:\b)WebLeacher(?:\b)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; + "~*(?:\b)webmeup-crawler(?:\b)" 3; + "~*(?:\b)WebPix(?:\b)" 3; + "~*(?:\b)Web\ Pix(?:\b)" 3; + "~*(?:\b)WebReaper(?:\b)" 3; + "~*(?:\b)WebSauger(?:\b)" 3; + "~*(?:\b)Web\ Sauger(?:\b)" 3; + "~*(?:\b)Webshag(?:\b)" 3; + "~*(?:\b)WebsiteExtractor(?:\b)" 3; + "~*(?:\b)WebsiteQuester(?:\b)" 3; + "~*(?:\b)Website\ Quester(?:\b)" 3; + "~*(?:\b)Webster(?:\b)" 3; + "~*(?:\b)WebStripper(?:\b)" 3; + "~*(?:\b)WebSucker(?:\b)" 3; + "~*(?:\b)Web\ Sucker(?:\b)" 3; + "~*(?:\b)WebWhacker(?:\b)" 3; + "~*(?:\b)WebZIP(?:\b)" 3; + "~*(?:\b)WeSEE(?:\b)" 3; + "~*(?:\b)Whack(?:\b)" 3; + "~*(?:\b)Whacker(?:\b)" 3; + "~*(?:\b)Whatweb(?:\b)" 3; + "~*(?:\b)Who.is\ Bot(?:\b)" 3; + "~*(?:\b)Widow(?:\b)" 3; + "~*(?:\b)WinHTTrack(?:\b)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; + "~*(?:\b)WISENutbot(?:\b)" 3; + "~*(?:\b)Wonderbot(?:\b)" 3; + "~*(?:\b)Woobot(?:\b)" 3; + "~*(?:\b)Wotbox(?:\b)" 3; + "~*(?:\b)Wprecon(?:\b)" 3; + "~*(?:\b)WPScan(?:\b)" 3; + "~*(?:\b)WWW-Collector-E(?:\b)" 3; + "~*(?:\b)WWW-Mechanize(?:\b)" 3; + "~*(?:\b)WWW::Mechanize(?:\b)" 3; + "~*(?:\b)WWWOFFLE(?:\b)" 3; + "~*(?:\b)x09Mozilla(?:\b)" 3; + "~*(?:\b)x22Mozilla(?:\b)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; + "~*(?:\b)Xenu(?:\b)" 3; + "~*(?:\b)xpymep1.exe(?:\b)" 3; + "~*(?:\b)YoudaoBot(?:\b)" 3; + "~*(?:\b)Zade(?:\b)" 3; + "~*(?:\b)Zauba(?:\b)" 3; + "~*(?:\b)zauba.io(?:\b)" 3; + "~*(?:\b)Zermelo(?:\b)" 3; + "~*(?:\b)Zeus(?:\b)" 3; + "~*(?:\b)zgrab(?:\b)" 3; + "~*(?:\b)Zitebot(?:\b)" 3; + "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZumBot(?:\b)" 3; + "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------- @@ -717,38 +717,38 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)adidxbot(?:\b|)" 0; - "~*(?:\b)AdsBot-Google(?:\b|)" 0; - "~*(?:\b)aolbuild(?:\b|)" 0; - "~*(?:\b)bingbot(?:\b|)" 0; - "~*(?:\b)bingpreview(?:\b|)" 0; - "~*(?:\b)DoCoMo(?:\b|)" 0; - "~*(?:\b)duckduckgo(?:\b|)" 0; - "~*(?:\b)facebookexternalhit(?:\b|)" 0; - "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; - "~*(?:\b)Googlebot(?:\b|)" 0; - "~*(?:\b)Googlebot-Image(?:\b|)" 0; - "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; - "~*(?:\b)Googlebot-News(?:\b|)" 0; - "~*(?:\b)Googlebot/Test(?:\b|)" 0; - "~*(?:\b)Googlebot-Video(?:\b|)" 0; - "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; - "~*(?:\b)Gravityscan(?:\b|)" 0; - "~*(?:\b)gsa-crawler(?:\b|)" 0; - "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; - "~*(?:\b)Kraken/0.1(?:\b|)" 0; - "~*(?:\b)LinkedInBot(?:\b|)" 0; - "~*(?:\b)Mediapartners-Google(?:\b|)" 0; - "~*(?:\b)msnbot(?:\b|)" 0; - "~*(?:\b)msnbot-media(?:\b|)" 0; - "~*(?:\b)SAMSUNG(?:\b|)" 0; - "~*(?:\b)Slackbot(?:\b|)" 0; - "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; - "~*(?:\b)slurp(?:\b|)" 0; - "~*(?:\b)teoma(?:\b|)" 0; - "~*(?:\b)TwitterBot(?:\b|)" 0; - "~*(?:\b)Wordpress(?:\b|)" 0; - "~*(?:\b)yahoo(?:\b|)" 0; + "~*(?:\b)adidxbot(?:\b)" 0; + "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bingbot(?:\b)" 0; + "~*(?:\b)bingpreview(?:\b)" 0; + "~*(?:\b)DoCoMo(?:\b)" 0; + "~*(?:\b)duckduckgo(?:\b)" 0; + "~*(?:\b)facebookexternalhit(?:\b)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b)" 0; + "~*(?:\b)Googlebot(?:\b)" 0; + "~*(?:\b)Googlebot-Image(?:\b)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b)" 0; + "~*(?:\b)Googlebot-News(?:\b)" 0; + "~*(?:\b)Googlebot/Test(?:\b)" 0; + "~*(?:\b)Googlebot-Video(?:\b)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; + "~*(?:\b)Gravityscan(?:\b)" 0; + "~*(?:\b)gsa-crawler(?:\b)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b)" 0; + "~*(?:\b)Kraken/0.1(?:\b)" 0; + "~*(?:\b)LinkedInBot(?:\b)" 0; + "~*(?:\b)Mediapartners-Google(?:\b)" 0; + "~*(?:\b)msnbot(?:\b)" 0; + "~*(?:\b)msnbot-media(?:\b)" 0; + "~*(?:\b)SAMSUNG(?:\b)" 0; + "~*(?:\b)Slackbot(?:\b)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 0; + "~*(?:\b)slurp(?:\b)" 0; + "~*(?:\b)teoma(?:\b)" 0; + "~*(?:\b)TwitterBot(?:\b)" 0; + "~*(?:\b)Wordpress(?:\b)" 0; + "~*(?:\b)yahoo(?:\b)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------------------- @@ -760,12 +760,12 @@ map $http_user_agent $bad_bot { #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)jetmon(?:\b|)" 1; - "~*(?:\b)libwww-perl(?:\b|)" 1; - "~*(?:\b)Lynx(?:\b|)" 1; - "~*(?:\b)munin(?:\b|)" 1; - "~*(?:\b)Presto(?:\b|)" 1; - "~*(?:\b)Wget/1.15(?:\b|)" 1; + "~*(?:\b)jetmon(?:\b)" 1; + "~*(?:\b)libwww-perl(?:\b)" 1; + "~*(?:\b)Lynx(?:\b)" 1; + "~*(?:\b)munin(?:\b)" 1; + "~*(?:\b)Presto(?:\b)" 1; + "~*(?:\b)Wget/1.15(?:\b)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ------------------------------------------------------- @@ -778,23 +778,23 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)Alexa(?:\b|)" 2; - "~*(?:\b)archive.org(?:\b|)" 2; - "~*(?:\b)Baidu(?:\b|)" 2; - "~*(?:\b)BUbiNG(?:\b|)" 2; - "~*(?:\b)FlipboardProxy(?:\b|)" 2; - "~*(?:\b)ia_archiver(?:\b|)" 2; - "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; - "~*(?:\b)Proximic(?:\b|)" 2; - "~*(?:\b)R6_CommentReader(?:\b|)" 2; - "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; - "~*(?:\b)RED/1(?:\b|)" 2; - "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; - "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; - "~*(?:\b)Spaidu(?:\b|)" 2; - "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; - "~*(?:\b)YandexBot(?:\b|)" 2; - "~*(?:\b)YandexImages(?:\b|)" 2; + "~*(?:\b)Alexa(?:\b)" 2; + "~*(?:\b)archive.org(?:\b)" 2; + "~*(?:\b)Baidu(?:\b)" 2; + "~*(?:\b)BUbiNG(?:\b)" 2; + "~*(?:\b)FlipboardProxy(?:\b)" 2; + "~*(?:\b)ia_archiver(?:\b)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b)" 2; + "~*(?:\b)Proximic(?:\b)" 2; + "~*(?:\b)R6_CommentReader(?:\b)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b)" 2; + "~*(?:\b)RED/1(?:\b)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b)" 2; + "~*(?:\b)Spaidu(?:\b)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b)" 2; + "~*(?:\b)YandexBot(?:\b)" 2; + "~*(?:\b)YandexImages(?:\b)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 7b00caacd..65f28ba04 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1704 -### Updated: Sat Jun 29 21:12:07 SAST 2019 +### Version: V3.2019.07.1705 +### Updated: Mon Jul 1 08:42:46 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### @@ -147,562 +147,562 @@ map $http_user_agent $bad_bot { # -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b|)" 3; - "~*(?:\b)404checker(?:\b|)" 3; - "~*(?:\b)404enemy(?:\b|)" 3; - "~*(?:\b)80legs(?:\b|)" 3; - "~*(?:\b)Abonti(?:\b|)" 3; - "~*(?:\b)Aboundex(?:\b|)" 3; - "~*(?:\b)Aboundexbot(?:\b|)" 3; - "~*(?:\b)Acunetix(?:\b|)" 3; - "~*(?:\b)ADmantX(?:\b|)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; - "~*(?:\b)AhrefsBot(?:\b|)" 3; - "~*(?:\b)AIBOT(?:\b|)" 3; - "~*(?:\b)AiHitBot(?:\b|)" 3; - "~*(?:\b)Aipbot(?:\b|)" 3; - "~*(?:\b)Alexibot(?:\b|)" 3; - "~*(?:\b)Alligator(?:\b|)" 3; - "~*(?:\b)AllSubmitter(?:\b|)" 3; - "~*(?:\b)AlphaBot(?:\b|)" 3; - "~*(?:\b)Anarchie(?:\b|)" 3; - "~*(?:\b)Apexoo(?:\b|)" 3; - "~*(?:\b)archive.org_bot(?:\b|)" 3; - "~*(?:\b)ASPSeek(?:\b|)" 3; - "~*(?:\b)Asterias(?:\b|)" 3; - "~*(?:\b)Attach(?:\b|)" 3; - "~*(?:\b)autoemailspider(?:\b|)" 3; - "~*(?:\b)BackDoorBot(?:\b|)" 3; - "~*(?:\b)Backlink-Ceck(?:\b|)" 3; - "~*(?:\b)backlink-check(?:\b|)" 3; - "~*(?:\b)BacklinkCrawler(?:\b|)" 3; - "~*(?:\b)BackStreet(?:\b|)" 3; - "~*(?:\b)BackWeb(?:\b|)" 3; - "~*(?:\b)Badass(?:\b|)" 3; - "~*(?:\b)Bandit(?:\b|)" 3; - "~*(?:\b)Barkrowler(?:\b|)" 3; - "~*(?:\b)BatchFTP(?:\b|)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; - "~*(?:\b)BBBike(?:\b|)" 3; - "~*(?:\b)BDCbot(?:\b|)" 3; - "~*(?:\b)BDFetch(?:\b|)" 3; - "~*(?:\b)BetaBot(?:\b|)" 3; - "~*(?:\b)Bigfoot(?:\b|)" 3; - "~*(?:\b)Bitacle(?:\b|)" 3; - "~*(?:\b)Blackboard(?:\b|)" 3; - "~*(?:\b)Black\ Hole(?:\b|)" 3; - "~*(?:\b)BlackWidow(?:\b|)" 3; - "~*(?:\b)BLEXBot(?:\b|)" 3; - "~*(?:\b)Blow(?:\b|)" 3; - "~*(?:\b)BlowFish(?:\b|)" 3; - "~*(?:\b)Boardreader(?:\b|)" 3; - "~*(?:\b)Bolt(?:\b|)" 3; - "~*(?:\b)BotALot(?:\b|)" 3; - "~*(?:\b)Brandprotect(?:\b|)" 3; - "~*(?:\b)Brandwatch(?:\b|)" 3; - "~*(?:\b)Buddy(?:\b|)" 3; - "~*(?:\b)BuiltBotTough(?:\b|)" 3; - "~*(?:\b)BuiltWith(?:\b|)" 3; - "~*(?:\b)Bullseye(?:\b|)" 3; - "~*(?:\b)BunnySlippers(?:\b|)" 3; - "~*(?:\b)BuzzSumo(?:\b|)" 3; - "~*(?:\b)Calculon(?:\b|)" 3; - "~*(?:\b)CATExplorador(?:\b|)" 3; - "~*(?:\b)CazoodleBot(?:\b|)" 3; - "~*(?:\b)CCBot(?:\b|)" 3; - "~*(?:\b)Cegbfeieh(?:\b|)" 3; - "~*(?:\b)CheeseBot(?:\b|)" 3; - "~*(?:\b)CherryPicker(?:\b|)" 3; - "~*(?:\b)CheTeam(?:\b|)" 3; - "~*(?:\b)ChinaClaw(?:\b|)" 3; - "~*(?:\b)Chlooe(?:\b|)" 3; - "~*(?:\b)Claritybot(?:\b|)" 3; - "~*(?:\b)Cliqzbot(?:\b|)" 3; - "~*(?:\b)Cloud\ mapping(?:\b|)" 3; - "~*(?:\b)coccocbot-web(?:\b|)" 3; - "~*(?:\b)Cogentbot(?:\b|)" 3; - "~*(?:\b)cognitiveseo(?:\b|)" 3; - "~*(?:\b)Collector(?:\b|)" 3; - "~*(?:\b)com.plumanalytics(?:\b|)" 3; - "~*(?:\b)Copier(?:\b|)" 3; - "~*(?:\b)CopyRightCheck(?:\b|)" 3; - "~*(?:\b)Copyscape(?:\b|)" 3; - "~*(?:\b)Cosmos(?:\b|)" 3; - "~*(?:\b)Craftbot(?:\b|)" 3; - "~*(?:\b)crawler4j(?:\b|)" 3; - "~*(?:\b)crawler.feedback(?:\b|)" 3; - "~*(?:\b)crawl.sogou.com(?:\b|)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; - "~*(?:\b)Crescent(?:\b|)" 3; - "~*(?:\b)CrunchBot(?:\b|)" 3; - "~*(?:\b)CSHttp(?:\b|)" 3; - "~*(?:\b)Curious(?:\b|)" 3; - "~*(?:\b)Custo(?:\b|)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; - "~*(?:\b)DataCha0s(?:\b|)" 3; - "~*(?:\b)DBLBot(?:\b|)" 3; - "~*(?:\b)demandbase-bot(?:\b|)" 3; - "~*(?:\b)Demon(?:\b|)" 3; - "~*(?:\b)Deusu(?:\b|)" 3; - "~*(?:\b)Devil(?:\b|)" 3; - "~*(?:\b)Digincore(?:\b|)" 3; - "~*(?:\b)DigitalPebble(?:\b|)" 3; - "~*(?:\b)DIIbot(?:\b|)" 3; - "~*(?:\b)Dirbuster(?:\b|)" 3; - "~*(?:\b)Disco(?:\b|)" 3; - "~*(?:\b)Discobot(?:\b|)" 3; - "~*(?:\b)Discoverybot(?:\b|)" 3; - "~*(?:\b)Dispatch(?:\b|)" 3; - "~*(?:\b)DittoSpyder(?:\b|)" 3; - "~*(?:\b)DnyzBot(?:\b|)" 3; - "~*(?:\b)DomainAppender(?:\b|)" 3; - "~*(?:\b)DomainCrawler(?:\b|)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; - "~*(?:\b)DomainStatsBot(?:\b|)" 3; - "~*(?:\b)Dotbot(?:\b|)" 3; - "~*(?:\b)Download\ Wonder(?:\b|)" 3; - "~*(?:\b)Dragonfly(?:\b|)" 3; - "~*(?:\b)Drip(?:\b|)" 3; - "~*(?:\b)DSearch(?:\b|)" 3; - "~*(?:\b)DTS\ Agent(?:\b|)" 3; - "~*(?:\b)EasyDL(?:\b|)" 3; - "~*(?:\b)Ebingbong(?:\b|)" 3; - "~*(?:\b)eCatch(?:\b|)" 3; - "~*(?:\b)ECCP/1.0(?:\b|)" 3; - "~*(?:\b)Ecxi(?:\b|)" 3; - "~*(?:\b)EirGrabber(?:\b|)" 3; - "~*(?:\b)EMail\ Siphon(?:\b|)" 3; - "~*(?:\b)EMail\ Wolf(?:\b|)" 3; - "~*(?:\b)EroCrawler(?:\b|)" 3; - "~*(?:\b)evc-batch(?:\b|)" 3; - "~*(?:\b)Evil(?:\b|)" 3; - "~*(?:\b)Exabot(?:\b|)" 3; - "~*(?:\b)Express\ WebPictures(?:\b|)" 3; - "~*(?:\b)ExtLinksBot(?:\b|)" 3; - "~*(?:\b)Extractor(?:\b|)" 3; - "~*(?:\b)ExtractorPro(?:\b|)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; - "~*(?:\b)EyeNetIE(?:\b|)" 3; - "~*(?:\b)Ezooms(?:\b|)" 3; - "~*(?:\b)facebookscraper(?:\b|)" 3; - "~*(?:\b)FDM(?:\b|)" 3; - "~*(?:\b)FemtosearchBot(?:\b|)" 3; - "~*(?:\b)FHscan(?:\b|)" 3; - "~*(?:\b)Fimap(?:\b|)" 3; - "~*(?:\b)Firefox/7.0(?:\b|)" 3; - "~*(?:\b)FlashGet(?:\b|)" 3; - "~*(?:\b)Flunky(?:\b|)" 3; - "~*(?:\b)Foobot(?:\b|)" 3; - "~*(?:\b)Freeuploader(?:\b|)" 3; - "~*(?:\b)FrontPage(?:\b|)" 3; - "~*(?:\b)FyberSpider(?:\b|)" 3; - "~*(?:\b)Fyrebot(?:\b|)" 3; - "~*(?:\b)GalaxyBot(?:\b|)" 3; - "~*(?:\b)Genieo(?:\b|)" 3; - "~*(?:\b)GermCrawler(?:\b|)" 3; - "~*(?:\b)Getintent(?:\b|)" 3; - "~*(?:\b)GetRight(?:\b|)" 3; - "~*(?:\b)GetWeb(?:\b|)" 3; - "~*(?:\b)Gigablast(?:\b|)" 3; - "~*(?:\b)Gigabot(?:\b|)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; - "~*(?:\b)Gotit(?:\b|)" 3; - "~*(?:\b)GoZilla(?:\b|)" 3; - "~*(?:\b)Go!Zilla(?:\b|)" 3; - "~*(?:\b)Grabber(?:\b|)" 3; - "~*(?:\b)GrabNet(?:\b|)" 3; - "~*(?:\b)Grafula(?:\b|)" 3; - "~*(?:\b)GrapeFX(?:\b|)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; - "~*(?:\b)GridBot(?:\b|)" 3; - "~*(?:\b)GT::WWW(?:\b|)" 3; - "~*(?:\b)Haansoft(?:\b|)" 3; - "~*(?:\b)HaosouSpider(?:\b|)" 3; - "~*(?:\b)Harvest(?:\b|)" 3; - "~*(?:\b)Havij(?:\b|)" 3; - "~*(?:\b)HEADMasterSEO(?:\b|)" 3; - "~*(?:\b)Heritrix(?:\b|)" 3; - "~*(?:\b)Hloader(?:\b|)" 3; - "~*(?:\b)HMView(?:\b|)" 3; - "~*(?:\b)HTMLparser(?:\b|)" 3; - "~*(?:\b)HTTP::Lite(?:\b|)" 3; - "~*(?:\b)HTTrack(?:\b|)" 3; - "~*(?:\b)Humanlinks(?:\b|)" 3; - "~*(?:\b)HybridBot(?:\b|)" 3; - "~*(?:\b)Iblog(?:\b|)" 3; - "~*(?:\b)IDBot(?:\b|)" 3; - "~*(?:\b)Id-search(?:\b|)" 3; - "~*(?:\b)IlseBot(?:\b|)" 3; - "~*(?:\b)Image\ Fetch(?:\b|)" 3; - "~*(?:\b)Image\ Sucker(?:\b|)" 3; - "~*(?:\b)IndeedBot(?:\b|)" 3; - "~*(?:\b)Indy\ Library(?:\b|)" 3; - "~*(?:\b)InfoNaviRobot(?:\b|)" 3; - "~*(?:\b)InfoTekies(?:\b|)" 3; - "~*(?:\b)instabid(?:\b|)" 3; - "~*(?:\b)Intelliseek(?:\b|)" 3; - "~*(?:\b)InterGET(?:\b|)" 3; - "~*(?:\b)Internet\ Ninja(?:\b|)" 3; - "~*(?:\b)InternetSeer(?:\b|)" 3; - "~*(?:\b)internetVista\ monitor(?:\b|)" 3; - "~*(?:\b)ips-agent(?:\b|)" 3; - "~*(?:\b)Iria(?:\b|)" 3; - "~*(?:\b)IRLbot(?:\b|)" 3; - "~*(?:\b)Iskanie(?:\b|)" 3; - "~*(?:\b)IstellaBot(?:\b|)" 3; - "~*(?:\b)JamesBOT(?:\b|)" 3; - "~*(?:\b)Jbrofuzz(?:\b|)" 3; - "~*(?:\b)JennyBot(?:\b|)" 3; - "~*(?:\b)JetCar(?:\b|)" 3; - "~*(?:\b)Jetty(?:\b|)" 3; - "~*(?:\b)JikeSpider(?:\b|)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; - "~*(?:\b)Joomla(?:\b|)" 3; - "~*(?:\b)Jorgee(?:\b|)" 3; - "~*(?:\b)JustView(?:\b|)" 3; - "~*(?:\b)Jyxobot(?:\b|)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; - "~*(?:\b)Keyword\ Density(?:\b|)" 3; - "~*(?:\b)Kozmosbot(?:\b|)" 3; - "~*(?:\b)Lanshanbot(?:\b|)" 3; - "~*(?:\b)Larbin(?:\b|)" 3; - "~*(?:\b)LeechFTP(?:\b|)" 3; - "~*(?:\b)LeechGet(?:\b|)" 3; - "~*(?:\b)LexiBot(?:\b|)" 3; - "~*(?:\b)Lftp(?:\b|)" 3; - "~*(?:\b)LibWeb(?:\b|)" 3; - "~*(?:\b)Libwhisker(?:\b|)" 3; - "~*(?:\b)Lightspeedsystems(?:\b|)" 3; - "~*(?:\b)Likse(?:\b|)" 3; - "~*(?:\b)Linkdexbot(?:\b|)" 3; - "~*(?:\b)LinkextractorPro(?:\b|)" 3; - "~*(?:\b)LinkpadBot(?:\b|)" 3; - "~*(?:\b)LinkScan(?:\b|)" 3; - "~*(?:\b)LinksManager(?:\b|)" 3; - "~*(?:\b)LinkWalker(?:\b|)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; - "~*(?:\b)Lipperhey(?:\b|)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; - "~*(?:\b)Litemage_walker(?:\b|)" 3; - "~*(?:\b)Lmspider(?:\b|)" 3; - "~*(?:\b)LNSpiderguy(?:\b|)" 3; - "~*(?:\b)Ltx71(?:\b|)" 3; - "~*(?:\b)lwp-request(?:\b|)" 3; - "~*(?:\b)LWP::Simple(?:\b|)" 3; - "~*(?:\b)lwp-trivial(?:\b|)" 3; - "~*(?:\b)Magnet(?:\b|)" 3; - "~*(?:\b)Mag-Net(?:\b|)" 3; - "~*(?:\b)magpie-crawler(?:\b|)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; - "~*(?:\b)Majestic12(?:\b|)" 3; - "~*(?:\b)Majestic-SEO(?:\b|)" 3; - "~*(?:\b)Majestic\ SEO(?:\b|)" 3; - "~*(?:\b)MarkMonitor(?:\b|)" 3; - "~*(?:\b)MarkWatch(?:\b|)" 3; - "~*(?:\b)Masscan(?:\b|)" 3; - "~*(?:\b)Mass\ Downloader(?:\b|)" 3; - "~*(?:\b)Mata\ Hari(?:\b|)" 3; - "~*(?:\b)MauiBot(?:\b|)" 3; - "~*(?:\b)meanpathbot(?:\b|)" 3; - "~*(?:\b)Meanpathbot(?:\b|)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; - "~*(?:\b)mediawords(?:\b|)" 3; - "~*(?:\b)MegaIndex.ru(?:\b|)" 3; - "~*(?:\b)Metauri(?:\b|)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; - "~*(?:\b)MIDown\ tool(?:\b|)" 3; - "~*(?:\b)MIIxpc(?:\b|)" 3; - "~*(?:\b)Mister\ PiX(?:\b|)" 3; - "~*(?:\b)MJ12bot(?:\b|)" 3; - "~*(?:\b)Mojeek(?:\b|)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; - "~*(?:\b)Mr.4x3(?:\b|)" 3; - "~*(?:\b)MSFrontPage(?:\b|)" 3; - "~*(?:\b)MSIECrawler(?:\b|)" 3; - "~*(?:\b)Msrabot(?:\b|)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; - "~*(?:\b)muhstik-scan(?:\b|)" 3; - "~*(?:\b)Musobot(?:\b|)" 3; - "~*(?:\b)Name\ Intelligence(?:\b|)" 3; - "~*(?:\b)Nameprotect(?:\b|)" 3; - "~*(?:\b)Navroad(?:\b|)" 3; - "~*(?:\b)NearSite(?:\b|)" 3; - "~*(?:\b)Needle(?:\b|)" 3; - "~*(?:\b)Nessus(?:\b|)" 3; - "~*(?:\b)NetAnts(?:\b|)" 3; - "~*(?:\b)Netcraft(?:\b|)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; - "~*(?:\b)NetLyzer(?:\b|)" 3; - "~*(?:\b)NetMechanic(?:\b|)" 3; - "~*(?:\b)NetSpider(?:\b|)" 3; - "~*(?:\b)Nettrack(?:\b|)" 3; - "~*(?:\b)Net\ Vampire(?:\b|)" 3; - "~*(?:\b)Netvibes(?:\b|)" 3; - "~*(?:\b)NetZIP(?:\b|)" 3; - "~*(?:\b)NextGenSearchBot(?:\b|)" 3; - "~*(?:\b)Nibbler(?:\b|)" 3; - "~*(?:\b)NICErsPRO(?:\b|)" 3; - "~*(?:\b)Niki-bot(?:\b|)" 3; - "~*(?:\b)Nikto(?:\b|)" 3; - "~*(?:\b)NimbleCrawler(?:\b|)" 3; - "~*(?:\b)Nimbostratus(?:\b|)" 3; - "~*(?:\b)Ninja(?:\b|)" 3; - "~*(?:\b)Nmap(?:\b|)" 3; - "~*(?:\b)NPbot(?:\b|)" 3; - "~*(?:\b)Nutch(?:\b|)" 3; - "~*(?:\b)oBot(?:\b|)" 3; - "~*(?:\b)Octopus(?:\b|)" 3; - "~*(?:\b)Offline\ Explorer(?:\b|)" 3; - "~*(?:\b)Offline\ Navigator(?:\b|)" 3; - "~*(?:\b)OnCrawl(?:\b|)" 3; - "~*(?:\b)Openfind(?:\b|)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; - "~*(?:\b)Openvas(?:\b|)" 3; - "~*(?:\b)OpenVAS(?:\b|)" 3; - "~*(?:\b)OrangeBot(?:\b|)" 3; - "~*(?:\b)OrangeSpider(?:\b|)" 3; - "~*(?:\b)OutclicksBot(?:\b|)" 3; - "~*(?:\b)OutfoxBot(?:\b|)" 3; - "~*(?:\b)PageAnalyzer(?:\b|)" 3; - "~*(?:\b)Page\ Analyzer(?:\b|)" 3; - "~*(?:\b)PageGrabber(?:\b|)" 3; - "~*(?:\b)page\ scorer(?:\b|)" 3; - "~*(?:\b)PageScorer(?:\b|)" 3; - "~*(?:\b)Pandalytics(?:\b|)" 3; - "~*(?:\b)Panscient(?:\b|)" 3; - "~*(?:\b)Papa\ Foto(?:\b|)" 3; - "~*(?:\b)Pavuk(?:\b|)" 3; - "~*(?:\b)pcBrowser(?:\b|)" 3; - "~*(?:\b)PECL::HTTP(?:\b|)" 3; - "~*(?:\b)PeoplePal(?:\b|)" 3; - "~*(?:\b)PHPCrawl(?:\b|)" 3; - "~*(?:\b)Picscout(?:\b|)" 3; - "~*(?:\b)Picsearch(?:\b|)" 3; - "~*(?:\b)PictureFinder(?:\b|)" 3; - "~*(?:\b)Pimonster(?:\b|)" 3; - "~*(?:\b)Pi-Monster(?:\b|)" 3; - "~*(?:\b)Pixray(?:\b|)" 3; - "~*(?:\b)PleaseCrawl(?:\b|)" 3; - "~*(?:\b)plumanalytics(?:\b|)" 3; - "~*(?:\b)Pockey(?:\b|)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; - "~*(?:\b)Probethenet(?:\b|)" 3; - "~*(?:\b)ProPowerBot(?:\b|)" 3; - "~*(?:\b)ProWebWalker(?:\b|)" 3; - "~*(?:\b)Psbot(?:\b|)" 3; - "~*(?:\b)Pump(?:\b|)" 3; - "~*(?:\b)PxBroker(?:\b|)" 3; - "~*(?:\b)PyCurl(?:\b|)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; - "~*(?:\b)Quick-Crawler(?:\b|)" 3; - "~*(?:\b)RankActive(?:\b|)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; - "~*(?:\b)RankFlex(?:\b|)" 3; - "~*(?:\b)RankingBot(?:\b|)" 3; - "~*(?:\b)RankingBot2(?:\b|)" 3; - "~*(?:\b)Rankivabot(?:\b|)" 3; - "~*(?:\b)RankurBot(?:\b|)" 3; - "~*(?:\b)RealDownload(?:\b|)" 3; - "~*(?:\b)Reaper(?:\b|)" 3; - "~*(?:\b)RebelMouse(?:\b|)" 3; - "~*(?:\b)Recorder(?:\b|)" 3; - "~*(?:\b)RedesScrapy(?:\b|)" 3; - "~*(?:\b)ReGet(?:\b|)" 3; - "~*(?:\b)RepoMonkey(?:\b|)" 3; - "~*(?:\b)Ripper(?:\b|)" 3; - "~*(?:\b)RocketCrawler(?:\b|)" 3; - "~*(?:\b)Rogerbot(?:\b|)" 3; - "~*(?:\b)s1z.ru(?:\b|)" 3; - "~*(?:\b)SalesIntelligent(?:\b|)" 3; - "~*(?:\b)SBIder(?:\b|)" 3; - "~*(?:\b)ScanAlert(?:\b|)" 3; - "~*(?:\b)Scanbot(?:\b|)" 3; - "~*(?:\b)scan.lol(?:\b|)" 3; - "~*(?:\b)ScoutJet(?:\b|)" 3; - "~*(?:\b)Scrapy(?:\b|)" 3; - "~*(?:\b)Screaming(?:\b|)" 3; - "~*(?:\b)ScreenerBot(?:\b|)" 3; - "~*(?:\b)Searchestate(?:\b|)" 3; - "~*(?:\b)SearchmetricsBot(?:\b|)" 3; - "~*(?:\b)Semrush(?:\b|)" 3; - "~*(?:\b)SemrushBot(?:\b|)" 3; - "~*(?:\b)SEOkicks(?:\b|)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; - "~*(?:\b)Seomoz(?:\b|)" 3; - "~*(?:\b)SEOprofiler(?:\b|)" 3; - "~*(?:\b)seoscanners(?:\b|)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; - "~*(?:\b)SEOstats(?:\b|)" 3; - "~*(?:\b)serpstatbot(?:\b|)" 3; - "~*(?:\b)sexsearcher(?:\b|)" 3; - "~*(?:\b)Shodan(?:\b|)" 3; - "~*(?:\b)Siphon(?:\b|)" 3; - "~*(?:\b)SISTRIX(?:\b|)" 3; - "~*(?:\b)Sitebeam(?:\b|)" 3; - "~*(?:\b)SiteExplorer(?:\b|)" 3; - "~*(?:\b)Siteimprove(?:\b|)" 3; - "~*(?:\b)SiteLockSpider(?:\b|)" 3; - "~*(?:\b)SiteSnagger(?:\b|)" 3; - "~*(?:\b)SiteSucker(?:\b|)" 3; - "~*(?:\b)Site\ Sucker(?:\b|)" 3; - "~*(?:\b)Sitevigil(?:\b|)" 3; - "~*(?:\b)SlySearch(?:\b|)" 3; - "~*(?:\b)SmartDownload(?:\b|)" 3; - "~*(?:\b)SMTBot(?:\b|)" 3; - "~*(?:\b)Snake(?:\b|)" 3; - "~*(?:\b)Snapbot(?:\b|)" 3; - "~*(?:\b)Snoopy(?:\b|)" 3; - "~*(?:\b)SocialRankIOBot(?:\b|)" 3; - "~*(?:\b)Sociscraper(?:\b|)" 3; - "~*(?:\b)sogouspider(?:\b|)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; - "~*(?:\b)Sosospider(?:\b|)" 3; - "~*(?:\b)Sottopop(?:\b|)" 3; - "~*(?:\b)SpaceBison(?:\b|)" 3; - "~*(?:\b)Spammen(?:\b|)" 3; - "~*(?:\b)SpankBot(?:\b|)" 3; - "~*(?:\b)Spanner(?:\b|)" 3; - "~*(?:\b)sp_auditbot(?:\b|)" 3; - "~*(?:\b)Spbot(?:\b|)" 3; - "~*(?:\b)Spinn3r(?:\b|)" 3; - "~*(?:\b)SputnikBot(?:\b|)" 3; - "~*(?:\b)spyfu(?:\b|)" 3; - "~*(?:\b)Sqlmap(?:\b|)" 3; - "~*(?:\b)Sqlworm(?:\b|)" 3; - "~*(?:\b)Sqworm(?:\b|)" 3; - "~*(?:\b)Steeler(?:\b|)" 3; - "~*(?:\b)Stripper(?:\b|)" 3; - "~*(?:\b)Sucker(?:\b|)" 3; - "~*(?:\b)Sucuri(?:\b|)" 3; - "~*(?:\b)SuperBot(?:\b|)" 3; - "~*(?:\b)SuperHTTP(?:\b|)" 3; - "~*(?:\b)Surfbot(?:\b|)" 3; - "~*(?:\b)SurveyBot(?:\b|)" 3; - "~*(?:\b)Suzuran(?:\b|)" 3; - "~*(?:\b)Swiftbot(?:\b|)" 3; - "~*(?:\b)sysscan(?:\b|)" 3; - "~*(?:\b)Szukacz(?:\b|)" 3; - "~*(?:\b)T0PHackTeam(?:\b|)" 3; - "~*(?:\b)T8Abot(?:\b|)" 3; - "~*(?:\b)tAkeOut(?:\b|)" 3; - "~*(?:\b)Teleport(?:\b|)" 3; - "~*(?:\b)TeleportPro(?:\b|)" 3; - "~*(?:\b)Telesoft(?:\b|)" 3; - "~*(?:\b)Telesphoreo(?:\b|)" 3; - "~*(?:\b)Telesphorep(?:\b|)" 3; - "~*(?:\b)The\ Intraformant(?:\b|)" 3; - "~*(?:\b)TheNomad(?:\b|)" 3; - "~*(?:\b)Thumbor(?:\b|)" 3; - "~*(?:\b)TightTwatBot(?:\b|)" 3; - "~*(?:\b)Titan(?:\b|)" 3; - "~*(?:\b)Toata(?:\b|)" 3; - "~*(?:\b)Toweyabot(?:\b|)" 3; - "~*(?:\b)Tracemyfile(?:\b|)" 3; - "~*(?:\b)Trendiction(?:\b|)" 3; - "~*(?:\b)Trendictionbot(?:\b|)" 3; - "~*(?:\b)trendiction.com(?:\b|)" 3; - "~*(?:\b)trendiction.de(?:\b|)" 3; - "~*(?:\b)True_Robot(?:\b|)" 3; - "~*(?:\b)Turingos(?:\b|)" 3; - "~*(?:\b)Turnitin(?:\b|)" 3; - "~*(?:\b)TurnitinBot(?:\b|)" 3; - "~*(?:\b)TwengaBot(?:\b|)" 3; - "~*(?:\b)Twice(?:\b|)" 3; - "~*(?:\b)Typhoeus(?:\b|)" 3; - "~*(?:\b)UnisterBot(?:\b|)" 3; - "~*(?:\b)Upflow(?:\b|)" 3; - "~*(?:\b)URLy.Warning(?:\b|)" 3; - "~*(?:\b)URLy\ Warning(?:\b|)" 3; - "~*(?:\b)Vacuum(?:\b|)" 3; - "~*(?:\b)Vagabondo(?:\b|)" 3; - "~*(?:\b)VB\ Project(?:\b|)" 3; - "~*(?:\b)VCI(?:\b|)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; - "~*(?:\b)VidibleScraper(?:\b|)" 3; - "~*(?:\b)Virusdie(?:\b|)" 3; - "~*(?:\b)VoidEYE(?:\b|)" 3; - "~*(?:\b)Voil(?:\b|)" 3; - "~*(?:\b)Voltron(?:\b|)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; - "~*(?:\b)WallpapersHD(?:\b|)" 3; - "~*(?:\b)WASALive-Bot(?:\b|)" 3; - "~*(?:\b)WBSearchBot(?:\b|)" 3; - "~*(?:\b)Webalta(?:\b|)" 3; - "~*(?:\b)WebAuto(?:\b|)" 3; - "~*(?:\b)Web\ Auto(?:\b|)" 3; - "~*(?:\b)WebBandit(?:\b|)" 3; - "~*(?:\b)WebCollage(?:\b|)" 3; - "~*(?:\b)Web\ Collage(?:\b|)" 3; - "~*(?:\b)WebCopier(?:\b|)" 3; - "~*(?:\b)WEBDAV(?:\b|)" 3; - "~*(?:\b)WebEnhancer(?:\b|)" 3; - "~*(?:\b)Web\ Enhancer(?:\b|)" 3; - "~*(?:\b)WebFetch(?:\b|)" 3; - "~*(?:\b)Web\ Fetch(?:\b|)" 3; - "~*(?:\b)WebFuck(?:\b|)" 3; - "~*(?:\b)Web\ Fuck(?:\b|)" 3; - "~*(?:\b)WebGo\ IS(?:\b|)" 3; - "~*(?:\b)WebImageCollector(?:\b|)" 3; - "~*(?:\b)WebLeacher(?:\b|)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; - "~*(?:\b)webmeup-crawler(?:\b|)" 3; - "~*(?:\b)WebPix(?:\b|)" 3; - "~*(?:\b)Web\ Pix(?:\b|)" 3; - "~*(?:\b)WebReaper(?:\b|)" 3; - "~*(?:\b)WebSauger(?:\b|)" 3; - "~*(?:\b)Web\ Sauger(?:\b|)" 3; - "~*(?:\b)Webshag(?:\b|)" 3; - "~*(?:\b)WebsiteExtractor(?:\b|)" 3; - "~*(?:\b)WebsiteQuester(?:\b|)" 3; - "~*(?:\b)Website\ Quester(?:\b|)" 3; - "~*(?:\b)Webster(?:\b|)" 3; - "~*(?:\b)WebStripper(?:\b|)" 3; - "~*(?:\b)WebSucker(?:\b|)" 3; - "~*(?:\b)Web\ Sucker(?:\b|)" 3; - "~*(?:\b)WebWhacker(?:\b|)" 3; - "~*(?:\b)WebZIP(?:\b|)" 3; - "~*(?:\b)WeSEE(?:\b|)" 3; - "~*(?:\b)Whack(?:\b|)" 3; - "~*(?:\b)Whacker(?:\b|)" 3; - "~*(?:\b)Whatweb(?:\b|)" 3; - "~*(?:\b)Who.is\ Bot(?:\b|)" 3; - "~*(?:\b)Widow(?:\b|)" 3; - "~*(?:\b)WinHTTrack(?:\b|)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; - "~*(?:\b)WISENutbot(?:\b|)" 3; - "~*(?:\b)Wonderbot(?:\b|)" 3; - "~*(?:\b)Woobot(?:\b|)" 3; - "~*(?:\b)Wotbox(?:\b|)" 3; - "~*(?:\b)Wprecon(?:\b|)" 3; - "~*(?:\b)WPScan(?:\b|)" 3; - "~*(?:\b)WWW-Collector-E(?:\b|)" 3; - "~*(?:\b)WWW-Mechanize(?:\b|)" 3; - "~*(?:\b)WWW::Mechanize(?:\b|)" 3; - "~*(?:\b)WWWOFFLE(?:\b|)" 3; - "~*(?:\b)x09Mozilla(?:\b|)" 3; - "~*(?:\b)x22Mozilla(?:\b|)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; - "~*(?:\b)Xenu(?:\b|)" 3; - "~*(?:\b)xpymep1.exe(?:\b|)" 3; - "~*(?:\b)YoudaoBot(?:\b|)" 3; - "~*(?:\b)Zade(?:\b|)" 3; - "~*(?:\b)Zauba(?:\b|)" 3; - "~*(?:\b)zauba.io(?:\b|)" 3; - "~*(?:\b)Zermelo(?:\b|)" 3; - "~*(?:\b)Zeus(?:\b|)" 3; - "~*(?:\b)zgrab(?:\b|)" 3; - "~*(?:\b)Zitebot(?:\b|)" 3; - "~*(?:\b)ZmEu(?:\b|)" 3; - "~*(?:\b)ZumBot(?:\b|)" 3; - "~*(?:\b)ZyBorg(?:\b|)" 3; + "~*(?:\b)360Spider(?:\b)" 3; + "~*(?:\b)404checker(?:\b)" 3; + "~*(?:\b)404enemy(?:\b)" 3; + "~*(?:\b)80legs(?:\b)" 3; + "~*(?:\b)Abonti(?:\b)" 3; + "~*(?:\b)Aboundex(?:\b)" 3; + "~*(?:\b)Aboundexbot(?:\b)" 3; + "~*(?:\b)Acunetix(?:\b)" 3; + "~*(?:\b)ADmantX(?:\b)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; + "~*(?:\b)AhrefsBot(?:\b)" 3; + "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)AiHitBot(?:\b)" 3; + "~*(?:\b)Aipbot(?:\b)" 3; + "~*(?:\b)Alexibot(?:\b)" 3; + "~*(?:\b)Alligator(?:\b)" 3; + "~*(?:\b)AllSubmitter(?:\b)" 3; + "~*(?:\b)AlphaBot(?:\b)" 3; + "~*(?:\b)Anarchie(?:\b)" 3; + "~*(?:\b)Apexoo(?:\b)" 3; + "~*(?:\b)archive.org_bot(?:\b)" 3; + "~*(?:\b)ASPSeek(?:\b)" 3; + "~*(?:\b)Asterias(?:\b)" 3; + "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)BackDoorBot(?:\b)" 3; + "~*(?:\b)Backlink-Ceck(?:\b)" 3; + "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)BacklinkCrawler(?:\b)" 3; + "~*(?:\b)BackStreet(?:\b)" 3; + "~*(?:\b)BackWeb(?:\b)" 3; + "~*(?:\b)Badass(?:\b)" 3; + "~*(?:\b)Bandit(?:\b)" 3; + "~*(?:\b)Barkrowler(?:\b)" 3; + "~*(?:\b)BatchFTP(?:\b)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; + "~*(?:\b)BBBike(?:\b)" 3; + "~*(?:\b)BDCbot(?:\b)" 3; + "~*(?:\b)BDFetch(?:\b)" 3; + "~*(?:\b)BetaBot(?:\b)" 3; + "~*(?:\b)Bigfoot(?:\b)" 3; + "~*(?:\b)Bitacle(?:\b)" 3; + "~*(?:\b)Blackboard(?:\b)" 3; + "~*(?:\b)Black\ Hole(?:\b)" 3; + "~*(?:\b)BlackWidow(?:\b)" 3; + "~*(?:\b)BLEXBot(?:\b)" 3; + "~*(?:\b)Blow(?:\b)" 3; + "~*(?:\b)BlowFish(?:\b)" 3; + "~*(?:\b)Boardreader(?:\b)" 3; + "~*(?:\b)Bolt(?:\b)" 3; + "~*(?:\b)BotALot(?:\b)" 3; + "~*(?:\b)Brandprotect(?:\b)" 3; + "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buddy(?:\b)" 3; + "~*(?:\b)BuiltBotTough(?:\b)" 3; + "~*(?:\b)BuiltWith(?:\b)" 3; + "~*(?:\b)Bullseye(?:\b)" 3; + "~*(?:\b)BunnySlippers(?:\b)" 3; + "~*(?:\b)BuzzSumo(?:\b)" 3; + "~*(?:\b)Calculon(?:\b)" 3; + "~*(?:\b)CATExplorador(?:\b)" 3; + "~*(?:\b)CazoodleBot(?:\b)" 3; + "~*(?:\b)CCBot(?:\b)" 3; + "~*(?:\b)Cegbfeieh(?:\b)" 3; + "~*(?:\b)CheeseBot(?:\b)" 3; + "~*(?:\b)CherryPicker(?:\b)" 3; + "~*(?:\b)CheTeam(?:\b)" 3; + "~*(?:\b)ChinaClaw(?:\b)" 3; + "~*(?:\b)Chlooe(?:\b)" 3; + "~*(?:\b)Claritybot(?:\b)" 3; + "~*(?:\b)Cliqzbot(?:\b)" 3; + "~*(?:\b)Cloud\ mapping(?:\b)" 3; + "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cogentbot(?:\b)" 3; + "~*(?:\b)cognitiveseo(?:\b)" 3; + "~*(?:\b)Collector(?:\b)" 3; + "~*(?:\b)com.plumanalytics(?:\b)" 3; + "~*(?:\b)Copier(?:\b)" 3; + "~*(?:\b)CopyRightCheck(?:\b)" 3; + "~*(?:\b)Copyscape(?:\b)" 3; + "~*(?:\b)Cosmos(?:\b)" 3; + "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)crawler4j(?:\b)" 3; + "~*(?:\b)crawler.feedback(?:\b)" 3; + "~*(?:\b)crawl.sogou.com(?:\b)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b)" 3; + "~*(?:\b)Crescent(?:\b)" 3; + "~*(?:\b)CrunchBot(?:\b)" 3; + "~*(?:\b)CSHttp(?:\b)" 3; + "~*(?:\b)Curious(?:\b)" 3; + "~*(?:\b)Custo(?:\b)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; + "~*(?:\b)DataCha0s(?:\b)" 3; + "~*(?:\b)DBLBot(?:\b)" 3; + "~*(?:\b)demandbase-bot(?:\b)" 3; + "~*(?:\b)Demon(?:\b)" 3; + "~*(?:\b)Deusu(?:\b)" 3; + "~*(?:\b)Devil(?:\b)" 3; + "~*(?:\b)Digincore(?:\b)" 3; + "~*(?:\b)DigitalPebble(?:\b)" 3; + "~*(?:\b)DIIbot(?:\b)" 3; + "~*(?:\b)Dirbuster(?:\b)" 3; + "~*(?:\b)Disco(?:\b)" 3; + "~*(?:\b)Discobot(?:\b)" 3; + "~*(?:\b)Discoverybot(?:\b)" 3; + "~*(?:\b)Dispatch(?:\b)" 3; + "~*(?:\b)DittoSpyder(?:\b)" 3; + "~*(?:\b)DnyzBot(?:\b)" 3; + "~*(?:\b)DomainAppender(?:\b)" 3; + "~*(?:\b)DomainCrawler(?:\b)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)DomainStatsBot(?:\b)" 3; + "~*(?:\b)Dotbot(?:\b)" 3; + "~*(?:\b)Download\ Wonder(?:\b)" 3; + "~*(?:\b)Dragonfly(?:\b)" 3; + "~*(?:\b)Drip(?:\b)" 3; + "~*(?:\b)DSearch(?:\b)" 3; + "~*(?:\b)DTS\ Agent(?:\b)" 3; + "~*(?:\b)EasyDL(?:\b)" 3; + "~*(?:\b)Ebingbong(?:\b)" 3; + "~*(?:\b)eCatch(?:\b)" 3; + "~*(?:\b)ECCP/1.0(?:\b)" 3; + "~*(?:\b)Ecxi(?:\b)" 3; + "~*(?:\b)EirGrabber(?:\b)" 3; + "~*(?:\b)EMail\ Siphon(?:\b)" 3; + "~*(?:\b)EMail\ Wolf(?:\b)" 3; + "~*(?:\b)EroCrawler(?:\b)" 3; + "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)Evil(?:\b)" 3; + "~*(?:\b)Exabot(?:\b)" 3; + "~*(?:\b)Express\ WebPictures(?:\b)" 3; + "~*(?:\b)ExtLinksBot(?:\b)" 3; + "~*(?:\b)Extractor(?:\b)" 3; + "~*(?:\b)ExtractorPro(?:\b)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; + "~*(?:\b)EyeNetIE(?:\b)" 3; + "~*(?:\b)Ezooms(?:\b)" 3; + "~*(?:\b)facebookscraper(?:\b)" 3; + "~*(?:\b)FDM(?:\b)" 3; + "~*(?:\b)FemtosearchBot(?:\b)" 3; + "~*(?:\b)FHscan(?:\b)" 3; + "~*(?:\b)Fimap(?:\b)" 3; + "~*(?:\b)Firefox/7.0(?:\b)" 3; + "~*(?:\b)FlashGet(?:\b)" 3; + "~*(?:\b)Flunky(?:\b)" 3; + "~*(?:\b)Foobot(?:\b)" 3; + "~*(?:\b)Freeuploader(?:\b)" 3; + "~*(?:\b)FrontPage(?:\b)" 3; + "~*(?:\b)FyberSpider(?:\b)" 3; + "~*(?:\b)Fyrebot(?:\b)" 3; + "~*(?:\b)GalaxyBot(?:\b)" 3; + "~*(?:\b)Genieo(?:\b)" 3; + "~*(?:\b)GermCrawler(?:\b)" 3; + "~*(?:\b)Getintent(?:\b)" 3; + "~*(?:\b)GetRight(?:\b)" 3; + "~*(?:\b)GetWeb(?:\b)" 3; + "~*(?:\b)Gigablast(?:\b)" 3; + "~*(?:\b)Gigabot(?:\b)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; + "~*(?:\b)Gotit(?:\b)" 3; + "~*(?:\b)GoZilla(?:\b)" 3; + "~*(?:\b)Go!Zilla(?:\b)" 3; + "~*(?:\b)Grabber(?:\b)" 3; + "~*(?:\b)GrabNet(?:\b)" 3; + "~*(?:\b)Grafula(?:\b)" 3; + "~*(?:\b)GrapeFX(?:\b)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b)" 3; + "~*(?:\b)GridBot(?:\b)" 3; + "~*(?:\b)GT::WWW(?:\b)" 3; + "~*(?:\b)Haansoft(?:\b)" 3; + "~*(?:\b)HaosouSpider(?:\b)" 3; + "~*(?:\b)Harvest(?:\b)" 3; + "~*(?:\b)Havij(?:\b)" 3; + "~*(?:\b)HEADMasterSEO(?:\b)" 3; + "~*(?:\b)Heritrix(?:\b)" 3; + "~*(?:\b)Hloader(?:\b)" 3; + "~*(?:\b)HMView(?:\b)" 3; + "~*(?:\b)HTMLparser(?:\b)" 3; + "~*(?:\b)HTTP::Lite(?:\b)" 3; + "~*(?:\b)HTTrack(?:\b)" 3; + "~*(?:\b)Humanlinks(?:\b)" 3; + "~*(?:\b)HybridBot(?:\b)" 3; + "~*(?:\b)Iblog(?:\b)" 3; + "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)Id-search(?:\b)" 3; + "~*(?:\b)IlseBot(?:\b)" 3; + "~*(?:\b)Image\ Fetch(?:\b)" 3; + "~*(?:\b)Image\ Sucker(?:\b)" 3; + "~*(?:\b)IndeedBot(?:\b)" 3; + "~*(?:\b)Indy\ Library(?:\b)" 3; + "~*(?:\b)InfoNaviRobot(?:\b)" 3; + "~*(?:\b)InfoTekies(?:\b)" 3; + "~*(?:\b)instabid(?:\b)" 3; + "~*(?:\b)Intelliseek(?:\b)" 3; + "~*(?:\b)InterGET(?:\b)" 3; + "~*(?:\b)Internet\ Ninja(?:\b)" 3; + "~*(?:\b)InternetSeer(?:\b)" 3; + "~*(?:\b)internetVista\ monitor(?:\b)" 3; + "~*(?:\b)ips-agent(?:\b)" 3; + "~*(?:\b)Iria(?:\b)" 3; + "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)Iskanie(?:\b)" 3; + "~*(?:\b)IstellaBot(?:\b)" 3; + "~*(?:\b)JamesBOT(?:\b)" 3; + "~*(?:\b)Jbrofuzz(?:\b)" 3; + "~*(?:\b)JennyBot(?:\b)" 3; + "~*(?:\b)JetCar(?:\b)" 3; + "~*(?:\b)Jetty(?:\b)" 3; + "~*(?:\b)JikeSpider(?:\b)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; + "~*(?:\b)Joomla(?:\b)" 3; + "~*(?:\b)Jorgee(?:\b)" 3; + "~*(?:\b)JustView(?:\b)" 3; + "~*(?:\b)Jyxobot(?:\b)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b)" 3; + "~*(?:\b)Keyword\ Density(?:\b)" 3; + "~*(?:\b)Kozmosbot(?:\b)" 3; + "~*(?:\b)Lanshanbot(?:\b)" 3; + "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)LeechFTP(?:\b)" 3; + "~*(?:\b)LeechGet(?:\b)" 3; + "~*(?:\b)LexiBot(?:\b)" 3; + "~*(?:\b)Lftp(?:\b)" 3; + "~*(?:\b)LibWeb(?:\b)" 3; + "~*(?:\b)Libwhisker(?:\b)" 3; + "~*(?:\b)Lightspeedsystems(?:\b)" 3; + "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkdexbot(?:\b)" 3; + "~*(?:\b)LinkextractorPro(?:\b)" 3; + "~*(?:\b)LinkpadBot(?:\b)" 3; + "~*(?:\b)LinkScan(?:\b)" 3; + "~*(?:\b)LinksManager(?:\b)" 3; + "~*(?:\b)LinkWalker(?:\b)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; + "~*(?:\b)Lipperhey(?:\b)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; + "~*(?:\b)Litemage_walker(?:\b)" 3; + "~*(?:\b)Lmspider(?:\b)" 3; + "~*(?:\b)LNSpiderguy(?:\b)" 3; + "~*(?:\b)Ltx71(?:\b)" 3; + "~*(?:\b)lwp-request(?:\b)" 3; + "~*(?:\b)LWP::Simple(?:\b)" 3; + "~*(?:\b)lwp-trivial(?:\b)" 3; + "~*(?:\b)Magnet(?:\b)" 3; + "~*(?:\b)Mag-Net(?:\b)" 3; + "~*(?:\b)magpie-crawler(?:\b)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b)" 3; + "~*(?:\b)Majestic12(?:\b)" 3; + "~*(?:\b)Majestic-SEO(?:\b)" 3; + "~*(?:\b)Majestic\ SEO(?:\b)" 3; + "~*(?:\b)MarkMonitor(?:\b)" 3; + "~*(?:\b)MarkWatch(?:\b)" 3; + "~*(?:\b)Masscan(?:\b)" 3; + "~*(?:\b)Mass\ Downloader(?:\b)" 3; + "~*(?:\b)Mata\ Hari(?:\b)" 3; + "~*(?:\b)MauiBot(?:\b)" 3; + "~*(?:\b)meanpathbot(?:\b)" 3; + "~*(?:\b)Meanpathbot(?:\b)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b)" 3; + "~*(?:\b)mediawords(?:\b)" 3; + "~*(?:\b)MegaIndex.ru(?:\b)" 3; + "~*(?:\b)Metauri(?:\b)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; + "~*(?:\b)MIDown\ tool(?:\b)" 3; + "~*(?:\b)MIIxpc(?:\b)" 3; + "~*(?:\b)Mister\ PiX(?:\b)" 3; + "~*(?:\b)MJ12bot(?:\b)" 3; + "~*(?:\b)Mojeek(?:\b)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; + "~*(?:\b)Mr.4x3(?:\b)" 3; + "~*(?:\b)MSFrontPage(?:\b)" 3; + "~*(?:\b)MSIECrawler(?:\b)" 3; + "~*(?:\b)Msrabot(?:\b)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; + "~*(?:\b)muhstik-scan(?:\b)" 3; + "~*(?:\b)Musobot(?:\b)" 3; + "~*(?:\b)Name\ Intelligence(?:\b)" 3; + "~*(?:\b)Nameprotect(?:\b)" 3; + "~*(?:\b)Navroad(?:\b)" 3; + "~*(?:\b)NearSite(?:\b)" 3; + "~*(?:\b)Needle(?:\b)" 3; + "~*(?:\b)Nessus(?:\b)" 3; + "~*(?:\b)NetAnts(?:\b)" 3; + "~*(?:\b)Netcraft(?:\b)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; + "~*(?:\b)NetLyzer(?:\b)" 3; + "~*(?:\b)NetMechanic(?:\b)" 3; + "~*(?:\b)NetSpider(?:\b)" 3; + "~*(?:\b)Nettrack(?:\b)" 3; + "~*(?:\b)Net\ Vampire(?:\b)" 3; + "~*(?:\b)Netvibes(?:\b)" 3; + "~*(?:\b)NetZIP(?:\b)" 3; + "~*(?:\b)NextGenSearchBot(?:\b)" 3; + "~*(?:\b)Nibbler(?:\b)" 3; + "~*(?:\b)NICErsPRO(?:\b)" 3; + "~*(?:\b)Niki-bot(?:\b)" 3; + "~*(?:\b)Nikto(?:\b)" 3; + "~*(?:\b)NimbleCrawler(?:\b)" 3; + "~*(?:\b)Nimbostratus(?:\b)" 3; + "~*(?:\b)Ninja(?:\b)" 3; + "~*(?:\b)Nmap(?:\b)" 3; + "~*(?:\b)NPbot(?:\b)" 3; + "~*(?:\b)Nutch(?:\b)" 3; + "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)Octopus(?:\b)" 3; + "~*(?:\b)Offline\ Explorer(?:\b)" 3; + "~*(?:\b)Offline\ Navigator(?:\b)" 3; + "~*(?:\b)OnCrawl(?:\b)" 3; + "~*(?:\b)Openfind(?:\b)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b)" 3; + "~*(?:\b)Openvas(?:\b)" 3; + "~*(?:\b)OpenVAS(?:\b)" 3; + "~*(?:\b)OrangeBot(?:\b)" 3; + "~*(?:\b)OrangeSpider(?:\b)" 3; + "~*(?:\b)OutclicksBot(?:\b)" 3; + "~*(?:\b)OutfoxBot(?:\b)" 3; + "~*(?:\b)PageAnalyzer(?:\b)" 3; + "~*(?:\b)Page\ Analyzer(?:\b)" 3; + "~*(?:\b)PageGrabber(?:\b)" 3; + "~*(?:\b)page\ scorer(?:\b)" 3; + "~*(?:\b)PageScorer(?:\b)" 3; + "~*(?:\b)Pandalytics(?:\b)" 3; + "~*(?:\b)Panscient(?:\b)" 3; + "~*(?:\b)Papa\ Foto(?:\b)" 3; + "~*(?:\b)Pavuk(?:\b)" 3; + "~*(?:\b)pcBrowser(?:\b)" 3; + "~*(?:\b)PECL::HTTP(?:\b)" 3; + "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)PHPCrawl(?:\b)" 3; + "~*(?:\b)Picscout(?:\b)" 3; + "~*(?:\b)Picsearch(?:\b)" 3; + "~*(?:\b)PictureFinder(?:\b)" 3; + "~*(?:\b)Pimonster(?:\b)" 3; + "~*(?:\b)Pi-Monster(?:\b)" 3; + "~*(?:\b)Pixray(?:\b)" 3; + "~*(?:\b)PleaseCrawl(?:\b)" 3; + "~*(?:\b)plumanalytics(?:\b)" 3; + "~*(?:\b)Pockey(?:\b)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; + "~*(?:\b)Probethenet(?:\b)" 3; + "~*(?:\b)ProPowerBot(?:\b)" 3; + "~*(?:\b)ProWebWalker(?:\b)" 3; + "~*(?:\b)Psbot(?:\b)" 3; + "~*(?:\b)Pump(?:\b)" 3; + "~*(?:\b)PxBroker(?:\b)" 3; + "~*(?:\b)PyCurl(?:\b)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; + "~*(?:\b)Quick-Crawler(?:\b)" 3; + "~*(?:\b)RankActive(?:\b)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b)" 3; + "~*(?:\b)RankFlex(?:\b)" 3; + "~*(?:\b)RankingBot(?:\b)" 3; + "~*(?:\b)RankingBot2(?:\b)" 3; + "~*(?:\b)Rankivabot(?:\b)" 3; + "~*(?:\b)RankurBot(?:\b)" 3; + "~*(?:\b)RealDownload(?:\b)" 3; + "~*(?:\b)Reaper(?:\b)" 3; + "~*(?:\b)RebelMouse(?:\b)" 3; + "~*(?:\b)Recorder(?:\b)" 3; + "~*(?:\b)RedesScrapy(?:\b)" 3; + "~*(?:\b)ReGet(?:\b)" 3; + "~*(?:\b)RepoMonkey(?:\b)" 3; + "~*(?:\b)Ripper(?:\b)" 3; + "~*(?:\b)RocketCrawler(?:\b)" 3; + "~*(?:\b)Rogerbot(?:\b)" 3; + "~*(?:\b)s1z.ru(?:\b)" 3; + "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)SBIder(?:\b)" 3; + "~*(?:\b)ScanAlert(?:\b)" 3; + "~*(?:\b)Scanbot(?:\b)" 3; + "~*(?:\b)scan.lol(?:\b)" 3; + "~*(?:\b)ScoutJet(?:\b)" 3; + "~*(?:\b)Scrapy(?:\b)" 3; + "~*(?:\b)Screaming(?:\b)" 3; + "~*(?:\b)ScreenerBot(?:\b)" 3; + "~*(?:\b)Searchestate(?:\b)" 3; + "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Semrush(?:\b)" 3; + "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SEOkicks(?:\b)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; + "~*(?:\b)Seomoz(?:\b)" 3; + "~*(?:\b)SEOprofiler(?:\b)" 3; + "~*(?:\b)seoscanners(?:\b)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b)" 3; + "~*(?:\b)SEOstats(?:\b)" 3; + "~*(?:\b)serpstatbot(?:\b)" 3; + "~*(?:\b)sexsearcher(?:\b)" 3; + "~*(?:\b)Shodan(?:\b)" 3; + "~*(?:\b)Siphon(?:\b)" 3; + "~*(?:\b)SISTRIX(?:\b)" 3; + "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteExplorer(?:\b)" 3; + "~*(?:\b)Siteimprove(?:\b)" 3; + "~*(?:\b)SiteLockSpider(?:\b)" 3; + "~*(?:\b)SiteSnagger(?:\b)" 3; + "~*(?:\b)SiteSucker(?:\b)" 3; + "~*(?:\b)Site\ Sucker(?:\b)" 3; + "~*(?:\b)Sitevigil(?:\b)" 3; + "~*(?:\b)SlySearch(?:\b)" 3; + "~*(?:\b)SmartDownload(?:\b)" 3; + "~*(?:\b)SMTBot(?:\b)" 3; + "~*(?:\b)Snake(?:\b)" 3; + "~*(?:\b)Snapbot(?:\b)" 3; + "~*(?:\b)Snoopy(?:\b)" 3; + "~*(?:\b)SocialRankIOBot(?:\b)" 3; + "~*(?:\b)Sociscraper(?:\b)" 3; + "~*(?:\b)sogouspider(?:\b)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; + "~*(?:\b)Sosospider(?:\b)" 3; + "~*(?:\b)Sottopop(?:\b)" 3; + "~*(?:\b)SpaceBison(?:\b)" 3; + "~*(?:\b)Spammen(?:\b)" 3; + "~*(?:\b)SpankBot(?:\b)" 3; + "~*(?:\b)Spanner(?:\b)" 3; + "~*(?:\b)sp_auditbot(?:\b)" 3; + "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spinn3r(?:\b)" 3; + "~*(?:\b)SputnikBot(?:\b)" 3; + "~*(?:\b)spyfu(?:\b)" 3; + "~*(?:\b)Sqlmap(?:\b)" 3; + "~*(?:\b)Sqlworm(?:\b)" 3; + "~*(?:\b)Sqworm(?:\b)" 3; + "~*(?:\b)Steeler(?:\b)" 3; + "~*(?:\b)Stripper(?:\b)" 3; + "~*(?:\b)Sucker(?:\b)" 3; + "~*(?:\b)Sucuri(?:\b)" 3; + "~*(?:\b)SuperBot(?:\b)" 3; + "~*(?:\b)SuperHTTP(?:\b)" 3; + "~*(?:\b)Surfbot(?:\b)" 3; + "~*(?:\b)SurveyBot(?:\b)" 3; + "~*(?:\b)Suzuran(?:\b)" 3; + "~*(?:\b)Swiftbot(?:\b)" 3; + "~*(?:\b)sysscan(?:\b)" 3; + "~*(?:\b)Szukacz(?:\b)" 3; + "~*(?:\b)T0PHackTeam(?:\b)" 3; + "~*(?:\b)T8Abot(?:\b)" 3; + "~*(?:\b)tAkeOut(?:\b)" 3; + "~*(?:\b)Teleport(?:\b)" 3; + "~*(?:\b)TeleportPro(?:\b)" 3; + "~*(?:\b)Telesoft(?:\b)" 3; + "~*(?:\b)Telesphoreo(?:\b)" 3; + "~*(?:\b)Telesphorep(?:\b)" 3; + "~*(?:\b)The\ Intraformant(?:\b)" 3; + "~*(?:\b)TheNomad(?:\b)" 3; + "~*(?:\b)Thumbor(?:\b)" 3; + "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)Titan(?:\b)" 3; + "~*(?:\b)Toata(?:\b)" 3; + "~*(?:\b)Toweyabot(?:\b)" 3; + "~*(?:\b)Tracemyfile(?:\b)" 3; + "~*(?:\b)Trendiction(?:\b)" 3; + "~*(?:\b)Trendictionbot(?:\b)" 3; + "~*(?:\b)trendiction.com(?:\b)" 3; + "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)True_Robot(?:\b)" 3; + "~*(?:\b)Turingos(?:\b)" 3; + "~*(?:\b)Turnitin(?:\b)" 3; + "~*(?:\b)TurnitinBot(?:\b)" 3; + "~*(?:\b)TwengaBot(?:\b)" 3; + "~*(?:\b)Twice(?:\b)" 3; + "~*(?:\b)Typhoeus(?:\b)" 3; + "~*(?:\b)UnisterBot(?:\b)" 3; + "~*(?:\b)Upflow(?:\b)" 3; + "~*(?:\b)URLy.Warning(?:\b)" 3; + "~*(?:\b)URLy\ Warning(?:\b)" 3; + "~*(?:\b)Vacuum(?:\b)" 3; + "~*(?:\b)Vagabondo(?:\b)" 3; + "~*(?:\b)VB\ Project(?:\b)" 3; + "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b)" 3; + "~*(?:\b)VidibleScraper(?:\b)" 3; + "~*(?:\b)Virusdie(?:\b)" 3; + "~*(?:\b)VoidEYE(?:\b)" 3; + "~*(?:\b)Voil(?:\b)" 3; + "~*(?:\b)Voltron(?:\b)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b)" 3; + "~*(?:\b)WallpapersHD(?:\b)" 3; + "~*(?:\b)WASALive-Bot(?:\b)" 3; + "~*(?:\b)WBSearchBot(?:\b)" 3; + "~*(?:\b)Webalta(?:\b)" 3; + "~*(?:\b)WebAuto(?:\b)" 3; + "~*(?:\b)Web\ Auto(?:\b)" 3; + "~*(?:\b)WebBandit(?:\b)" 3; + "~*(?:\b)WebCollage(?:\b)" 3; + "~*(?:\b)Web\ Collage(?:\b)" 3; + "~*(?:\b)WebCopier(?:\b)" 3; + "~*(?:\b)WEBDAV(?:\b)" 3; + "~*(?:\b)WebEnhancer(?:\b)" 3; + "~*(?:\b)Web\ Enhancer(?:\b)" 3; + "~*(?:\b)WebFetch(?:\b)" 3; + "~*(?:\b)Web\ Fetch(?:\b)" 3; + "~*(?:\b)WebFuck(?:\b)" 3; + "~*(?:\b)Web\ Fuck(?:\b)" 3; + "~*(?:\b)WebGo\ IS(?:\b)" 3; + "~*(?:\b)WebImageCollector(?:\b)" 3; + "~*(?:\b)WebLeacher(?:\b)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; + "~*(?:\b)webmeup-crawler(?:\b)" 3; + "~*(?:\b)WebPix(?:\b)" 3; + "~*(?:\b)Web\ Pix(?:\b)" 3; + "~*(?:\b)WebReaper(?:\b)" 3; + "~*(?:\b)WebSauger(?:\b)" 3; + "~*(?:\b)Web\ Sauger(?:\b)" 3; + "~*(?:\b)Webshag(?:\b)" 3; + "~*(?:\b)WebsiteExtractor(?:\b)" 3; + "~*(?:\b)WebsiteQuester(?:\b)" 3; + "~*(?:\b)Website\ Quester(?:\b)" 3; + "~*(?:\b)Webster(?:\b)" 3; + "~*(?:\b)WebStripper(?:\b)" 3; + "~*(?:\b)WebSucker(?:\b)" 3; + "~*(?:\b)Web\ Sucker(?:\b)" 3; + "~*(?:\b)WebWhacker(?:\b)" 3; + "~*(?:\b)WebZIP(?:\b)" 3; + "~*(?:\b)WeSEE(?:\b)" 3; + "~*(?:\b)Whack(?:\b)" 3; + "~*(?:\b)Whacker(?:\b)" 3; + "~*(?:\b)Whatweb(?:\b)" 3; + "~*(?:\b)Who.is\ Bot(?:\b)" 3; + "~*(?:\b)Widow(?:\b)" 3; + "~*(?:\b)WinHTTrack(?:\b)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; + "~*(?:\b)WISENutbot(?:\b)" 3; + "~*(?:\b)Wonderbot(?:\b)" 3; + "~*(?:\b)Woobot(?:\b)" 3; + "~*(?:\b)Wotbox(?:\b)" 3; + "~*(?:\b)Wprecon(?:\b)" 3; + "~*(?:\b)WPScan(?:\b)" 3; + "~*(?:\b)WWW-Collector-E(?:\b)" 3; + "~*(?:\b)WWW-Mechanize(?:\b)" 3; + "~*(?:\b)WWW::Mechanize(?:\b)" 3; + "~*(?:\b)WWWOFFLE(?:\b)" 3; + "~*(?:\b)x09Mozilla(?:\b)" 3; + "~*(?:\b)x22Mozilla(?:\b)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; + "~*(?:\b)Xenu(?:\b)" 3; + "~*(?:\b)xpymep1.exe(?:\b)" 3; + "~*(?:\b)YoudaoBot(?:\b)" 3; + "~*(?:\b)Zade(?:\b)" 3; + "~*(?:\b)Zauba(?:\b)" 3; + "~*(?:\b)zauba.io(?:\b)" 3; + "~*(?:\b)Zermelo(?:\b)" 3; + "~*(?:\b)Zeus(?:\b)" 3; + "~*(?:\b)zgrab(?:\b)" 3; + "~*(?:\b)Zitebot(?:\b)" 3; + "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZumBot(?:\b)" 3; + "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------- @@ -717,38 +717,38 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)adidxbot(?:\b|)" 0; - "~*(?:\b)AdsBot-Google(?:\b|)" 0; - "~*(?:\b)aolbuild(?:\b|)" 0; - "~*(?:\b)bingbot(?:\b|)" 0; - "~*(?:\b)bingpreview(?:\b|)" 0; - "~*(?:\b)DoCoMo(?:\b|)" 0; - "~*(?:\b)duckduckgo(?:\b|)" 0; - "~*(?:\b)facebookexternalhit(?:\b|)" 0; - "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; - "~*(?:\b)Googlebot(?:\b|)" 0; - "~*(?:\b)Googlebot-Image(?:\b|)" 0; - "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; - "~*(?:\b)Googlebot-News(?:\b|)" 0; - "~*(?:\b)Googlebot/Test(?:\b|)" 0; - "~*(?:\b)Googlebot-Video(?:\b|)" 0; - "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; - "~*(?:\b)Gravityscan(?:\b|)" 0; - "~*(?:\b)gsa-crawler(?:\b|)" 0; - "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; - "~*(?:\b)Kraken/0.1(?:\b|)" 0; - "~*(?:\b)LinkedInBot(?:\b|)" 0; - "~*(?:\b)Mediapartners-Google(?:\b|)" 0; - "~*(?:\b)msnbot(?:\b|)" 0; - "~*(?:\b)msnbot-media(?:\b|)" 0; - "~*(?:\b)SAMSUNG(?:\b|)" 0; - "~*(?:\b)Slackbot(?:\b|)" 0; - "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; - "~*(?:\b)slurp(?:\b|)" 0; - "~*(?:\b)teoma(?:\b|)" 0; - "~*(?:\b)TwitterBot(?:\b|)" 0; - "~*(?:\b)Wordpress(?:\b|)" 0; - "~*(?:\b)yahoo(?:\b|)" 0; + "~*(?:\b)adidxbot(?:\b)" 0; + "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bingbot(?:\b)" 0; + "~*(?:\b)bingpreview(?:\b)" 0; + "~*(?:\b)DoCoMo(?:\b)" 0; + "~*(?:\b)duckduckgo(?:\b)" 0; + "~*(?:\b)facebookexternalhit(?:\b)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b)" 0; + "~*(?:\b)Googlebot(?:\b)" 0; + "~*(?:\b)Googlebot-Image(?:\b)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b)" 0; + "~*(?:\b)Googlebot-News(?:\b)" 0; + "~*(?:\b)Googlebot/Test(?:\b)" 0; + "~*(?:\b)Googlebot-Video(?:\b)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; + "~*(?:\b)Gravityscan(?:\b)" 0; + "~*(?:\b)gsa-crawler(?:\b)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b)" 0; + "~*(?:\b)Kraken/0.1(?:\b)" 0; + "~*(?:\b)LinkedInBot(?:\b)" 0; + "~*(?:\b)Mediapartners-Google(?:\b)" 0; + "~*(?:\b)msnbot(?:\b)" 0; + "~*(?:\b)msnbot-media(?:\b)" 0; + "~*(?:\b)SAMSUNG(?:\b)" 0; + "~*(?:\b)Slackbot(?:\b)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 0; + "~*(?:\b)slurp(?:\b)" 0; + "~*(?:\b)teoma(?:\b)" 0; + "~*(?:\b)TwitterBot(?:\b)" 0; + "~*(?:\b)Wordpress(?:\b)" 0; + "~*(?:\b)yahoo(?:\b)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------------------- @@ -760,12 +760,12 @@ map $http_user_agent $bad_bot { #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)jetmon(?:\b|)" 1; - "~*(?:\b)libwww-perl(?:\b|)" 1; - "~*(?:\b)Lynx(?:\b|)" 1; - "~*(?:\b)munin(?:\b|)" 1; - "~*(?:\b)Presto(?:\b|)" 1; - "~*(?:\b)Wget/1.15(?:\b|)" 1; + "~*(?:\b)jetmon(?:\b)" 1; + "~*(?:\b)libwww-perl(?:\b)" 1; + "~*(?:\b)Lynx(?:\b)" 1; + "~*(?:\b)munin(?:\b)" 1; + "~*(?:\b)Presto(?:\b)" 1; + "~*(?:\b)Wget/1.15(?:\b)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ------------------------------------------------------- @@ -778,23 +778,23 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)Alexa(?:\b|)" 2; - "~*(?:\b)archive.org(?:\b|)" 2; - "~*(?:\b)Baidu(?:\b|)" 2; - "~*(?:\b)BUbiNG(?:\b|)" 2; - "~*(?:\b)FlipboardProxy(?:\b|)" 2; - "~*(?:\b)ia_archiver(?:\b|)" 2; - "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; - "~*(?:\b)Proximic(?:\b|)" 2; - "~*(?:\b)R6_CommentReader(?:\b|)" 2; - "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; - "~*(?:\b)RED/1(?:\b|)" 2; - "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; - "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; - "~*(?:\b)Spaidu(?:\b|)" 2; - "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; - "~*(?:\b)YandexBot(?:\b|)" 2; - "~*(?:\b)YandexImages(?:\b|)" 2; + "~*(?:\b)Alexa(?:\b)" 2; + "~*(?:\b)archive.org(?:\b)" 2; + "~*(?:\b)Baidu(?:\b)" 2; + "~*(?:\b)BUbiNG(?:\b)" 2; + "~*(?:\b)FlipboardProxy(?:\b)" 2; + "~*(?:\b)ia_archiver(?:\b)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b)" 2; + "~*(?:\b)Proximic(?:\b)" 2; + "~*(?:\b)R6_CommentReader(?:\b)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b)" 2; + "~*(?:\b)RED/1(?:\b)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b)" 2; + "~*(?:\b)Spaidu(?:\b)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b)" 2; + "~*(?:\b)YandexBot(?:\b)" 2; + "~*(?:\b)YandexImages(?:\b)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index fceaee33a..5bddc3b6e 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1704 -### Updated: Sat Jun 29 21:12:06 SAST 2019 +### Version: V4.2019.07.1705 +### Updated: Mon Jul 1 08:42:45 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index ca4fcdf54..deb2992bb 100644 --- a/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.193.14.140 0; +35.202.145.110 0; diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index fceaee33a..5bddc3b6e 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1704 -### Updated: Sat Jun 29 21:12:06 SAST 2019 +### Version: V4.2019.07.1705 +### Updated: Mon Jul 1 08:42:45 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 2e2965ffc..12f18b91b 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.188.73.34 1; +35.202.145.110 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index d3d331ed4..c3c284efc 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.188.73.34 0; +35.202.145.110 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index fceaee33a..5bddc3b6e 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1704 -### Updated: Sat Jun 29 21:12:06 SAST 2019 +### Version: V4.2019.07.1705 +### Updated: Mon Jul 1 08:42:45 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 2e2965ffc..12f18b91b 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.188.73.34 1; +35.202.145.110 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index d3d331ed4..c3c284efc 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.188.73.34 0; +35.202.145.110 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index fceaee33a..5bddc3b6e 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1704 -### Updated: Sat Jun 29 21:12:06 SAST 2019 +### Version: V4.2019.07.1705 +### Updated: Mon Jul 1 08:42:45 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 2e2965ffc..12f18b91b 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.188.73.34 1; +35.202.145.110 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index d3d331ed4..c3c284efc 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.188.73.34 0; +35.202.145.110 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index fceaee33a..5bddc3b6e 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1704 -### Updated: Sat Jun 29 21:12:06 SAST 2019 +### Version: V4.2019.07.1705 +### Updated: Mon Jul 1 08:42:45 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 04ec378ff..6ff35261d 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.68.144.114 0; -35.188.73.34 0; +35.192.85.2 0; +35.193.7.13 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index f7c10b46f..85e523c84 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1704 -### Updated: Sat Jun 29 21:12:06 SAST 2019 +### Version: V4.2019.07.1705 +### Updated: Mon Jul 1 08:42:45 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index 008443014..ecdc0d878 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -35.192.187.174 0; -104.154.113.151 0; -35.188.73.34 0; +35.192.85.2 0; +35.224.112.202 0; +35.184.96.71 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index fceaee33a..5bddc3b6e 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1704 -### Updated: Sat Jun 29 21:12:06 SAST 2019 +### Version: V4.2019.07.1705 +### Updated: Mon Jul 1 08:42:45 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index 1f5f8e206..16f37cf31 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -34.68.144.114 0; -35.193.14.140 0; -104.198.131.58 0; +35.193.7.13 0; +35.192.85.2 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index fceaee33a..5bddc3b6e 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.06.1704 -### Updated: Sat Jun 29 21:12:06 SAST 2019 +### Version: V4.2019.07.1705 +### Updated: Mon Jul 1 08:42:45 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 2e2965ffc..12f18b91b 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.188.73.34 1; +35.202.145.110 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 5b214868e..9c485c1fa 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -BDFetch -GT::WWW -Larbin -mediawords -Offline Navigator -SEOstats -Snoopy -True_Robot -Whack -WWWOFFLE +Anarchie +Barkrowler +Brandwatch +Copier +DomainAppender +FrontPage +RealDownload +SocialRankIOBot +Webalta +Xenu diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index c4110b796..6ee370cbd 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -404checker +404enemy 80legs -Aboundex +Acunetix ADmantX AfD-Verbotsverfahren -AIBOT -AiHitBot -Alexibot -archive.org_bot -BackDoorBot -Backlink-Ceck +Aipbot +Alligator +AllSubmitter +AlphaBot +Anarchie +Apexoo +Attach +backlink-check BacklinkCrawler BackStreet Battleztar Bazinga BBBike +BDCbot BetaBot -Bigfoot -Bitacle Blackboard Black Hole +BlackWidow BLEXBot -Boardreader -BotALot -Brandwatch -Buddy -BuiltBotTough -BuiltWith -Bullseye +Brandprotect BunnySlippers BuzzSumo -CheTeam -ChinaClaw +CATExplorador +CazoodleBot +CCBot +Cegbfeieh +CheeseBot +CherryPicker +Claritybot Cloud mapping -coccocbot-web Collector com.plumanalytics -CopyRightCheck -crawler.feedback +crawler4j +crawl.sogou.com CrazyWebCrawler -CSHttp -DBLBot +DataCha0s Demon Deusu -Devil Digincore -DigitalPebble DIIbot Dirbuster Disco -Discobot Discoverybot DomainAppender DomainSigmaCrawler DomainStatsBot -Download Wonder +Dragonfly +DSearch EasyDL -Ebingbong eCatch +Ecxi EirGrabber EMail Siphon -EMail Wolf evc-batch Evil -Extreme Picture Finder -FHscan +Express WebPictures +ExtLinksBot +Extractor +ExtractorPro +Ezooms +FemtosearchBot +Fimap +Firefox/7.0 FlashGet +Flunky +Foobot FyberSpider -GalaxyBot Genieo GermCrawler Getintent -GetRight -GetWeb Gigablast -Gigabot +G-i-g-a-b-o-t +Gotit +GoZilla Go!Zilla GrabNet -GrapeFX +Grafula GT::WWW -Haansoft -Harvest -Hloader -HTMLparser +HaosouSpider +HEADMasterSEO +HTTP::Lite HTTrack -IDBot -Id-search +Iblog IlseBot -Image Fetch -IndeedBot -InfoTekies +Indy Library +InfoNaviRobot +instabid Intelliseek +InterGET Internet Ninja InternetSeer -ips-agent +Iria IRLbot Iskanie IstellaBot JikeSpider JOC Web Spider -Joomla -Jyxobot -Kozmosbot Lanshanbot -LeechGet -LexiBot -Linkdexbot +LeechFTP +Libwhisker +Likse LinkextractorPro LinkpadBot +LinkScan LinksManager -LinqiaMetadataDownloaderBot -LinqiaRSSBot -Lipperhey -Ltx71 +LinqiaScrapeBot +Lmspider +LNSpiderguy lwp-request Magnet -Mail.RU_Bot +Mag-Net +magpie-crawler +Majestic12 Majestic SEO -MarkWatch -Masscan -Mass Downloader +Mata Hari +meanpathbot +Meanpathbot Metauri -MFC_Tear_Sample Microsoft URL Control -MIDown tool MIIxpc Mister PiX Mojeek +Morfeus Fucking Scanner Mr.4x3 +MSFrontPage Msrabot -Musobot -Nameprotect -NearSite +Navroad +Needle NetAnts Netcraft +netEstate NE Crawler NetLyzer -NetSpider -Nettrack -Net Vampire NetZIP NextGenSearchBot -Nibbler NICErsPRO -Nimbostratus -Nmap -Nutch -Octopus -Offline Navigator -OpenLinkProfiler +Nikto +NimbleCrawler +Ninja +NPbot +OnCrawl +Openfind Openvas -OutfoxBot -Papa Foto -pcBrowser +OpenVAS +OrangeBot +OutclicksBot +Page Analyzer +PageGrabber +page scorer PeoplePal PHPCrawl Picscout -PictureFinder -Pockey +Picsearch +Pimonster +Pixray POE-Component-Client-HTTP ProPowerBot -ProWebWalker -Psbot Pump -Quick-Crawler +PxBroker +PyCurl +QueryN Metasearch +RankActive RankActiveLinkBot -RankingBot Rankivabot RankurBot -RealDownload Reaper -RebelMouse Recorder -RedesScrapy -RepoMonkey -SalesIntelligent -SBIder +ReGet +Ripper +Rogerbot +s1z.ru +ScanAlert +Scanbot scan.lol -Scrapy -SearchmetricsBot -Semrush -SemrushBot -SEOkicks -serpstatbot +SEOlyticsCrawler +seoscanners +SEOstats sexsearcher -SISTRIX -SiteSnagger +SiteLockSpider +SiteSucker Site Sucker -Sitevigil +SlySearch SmartDownload -Snake Sociscraper +sogouspider Sogou web spider Sosospider -Sottopop -SpaceBison -SpankBot -Spbot +Spammen Spinn3r -SputnikBot -spyfu +Sqlmap +Sqworm +Steeler +Stripper Sucker SuperBot +SurveyBot Swiftbot -Teleport +sysscan +Szukacz TeleportPro +Telesoft Telesphorep -The Intraformant -Titan +TightTwatBot Tracemyfile -trendiction.com -trendiction.de -Turingos +Trendiction Turnitin +TurnitinBot +TwengaBot Twice -URLy Warning -URLy.Warning +Typhoeus +Vacuum Vagabondo -VCI +VB Project Voltron -WBSearchBot -Web Auto +WASALive-Bot +Webalta WebCollage Web Collage -WebCopier -WEBDAV -Web Fetch +WebFuck Web Fuck WebGo IS WebImageCollector -WebPix -Web Pix +WebLeacher +WebmasterWorldForumBot Web Sauger Webshag -WebsiteQuester +Website Quester +Webster +WebSucker WebZIP Whack -Whacker -Who.is Bot +Whatweb Widow WinHTTrack -Woobot -Wotbox Wprecon -WWWOFFLE +WPScan +WWW::Mechanize x09Mozilla x22Mozilla -Zeus -zgrab +Xaldon_WebSpider +Xenu +Zauba +zauba.io +Zermelo Zitebot ZmEu ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index a722ed84b..59cfc2d69 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ 360Spider -80legs -Aboundex +404checker +404enemy +Abonti Acunetix ADmantX -AfD-Verbotsverfahren -AhrefsBot +AiHitBot Alexibot -Alligator AllSubmitter AlphaBot +Anarchie +Apexoo ASPSeek -Attach autoemailspider +BackDoorBot +Backlink-Ceck backlink-check BacklinkCrawler -Badass -Barkrowler -BatchFTP -Battleztar Bazinga -BDFetch +Bandit +BDCbot +Bigfoot Bitacle -Blackboard -Black Hole -BlackWidow -BlowFish -Boardreader -BotALot -Brandprotect Buddy BuiltWith BunnySlippers +BuzzSumo Calculon -CATExplorador CCBot -CheTeam +ChinaClaw +Chlooe Claritybot +Cliqzbot coccocbot-web -cognitiveseo com.plumanalytics Copier -Copyscape Cosmos -crawler4j +Craftbot crawler.feedback crawl.sogou.com CrazyWebCrawler +Crescent CrunchBot -CSHttp -Custo +Curious +DataCha0s +DBLBot +Demon Deusu -Discoverybot +Devil +Digincore +Dirbuster +Discobot +Dispatch DnyzBot +DomainAppender DomainCrawler -DomainSigmaCrawler -DomainStatsBot Dotbot Download Wonder Dragonfly Drip DSearch +DTS Agent eCatch -Ecxi +ECCP/1.0 EMail Siphon -EroCrawler -evc-batch -Express WebPictures -Extractor +Exabot ExtractorPro +Extreme Picture Finder EyeNetIE -facebookscraper -FlashGet -Freeuploader +FemtosearchBot +Firefox/7.0 +Flunky +Foobot +FrontPage +FyberSpider GalaxyBot -GermCrawler -Getintent +Genieo +GetRight +GetWeb Gigabot G-i-g-a-b-o-t GoZilla Grabber Grafula +GrapeFX GrapeshotCrawler -GridBot -Harvest -HMView +GT::WWW +Haansoft +HaosouSpider +Havij HTMLparser -HTTP::Lite -HTTrack Humanlinks +HybridBot IDBot -IlseBot -Image Fetch -Image Sucker -IndeedBot Indy Library -InfoTekies -Intelliseek Internet Ninja -InternetSeer Iria +IRLbot +JamesBOT JennyBot -JetCar Jetty JikeSpider -Jyxobot -Keyword Density +Joomla +JustView Lanshanbot -Larbin -LeechGet -Lftp +LexiBot Lightspeedsystems Linkdexbot LinkextractorPro LinksManager LinkWalker -Litemage_walker -Lmspider -LNSpiderguy -Ltx71 -LWP::Simple +LinqiaMetadataDownloaderBot +LinqiaScrapeBot +Lipperhey Spider +lwp-request +lwp-trivial +Mail.RU_Bot Majestic12 -MarkMonitor +MarkWatch Masscan -Mass Downloader -MauiBot -meanpathbot -Meanpathbot -MegaIndex.ru -MFC_Tear_Sample +Mata Hari +MeanPath Bot +Mediatoolkitbot +Metauri Microsoft Data Access -MJ12bot -MSFrontPage +MIDown tool +MIIxpc +Mojeek +Morfeus Fucking Scanner +MS Web Services Client Protocol +muhstik-scan Musobot +Nameprotect +NearSite +Needle +Nessus Netcraft -NetMechanic -NextGenSearchBot -Nikto +netEstate NE Crawler +NetSpider +Nettrack +Net Vampire +NetZIP +NICErsPRO NimbleCrawler Ninja Nmap -Nutch -OpenLinkProfiler -OutclicksBot +oBot +Octopus +Offline Explorer +Offline Navigator +OpenVAS PageAnalyzer -Pandalytics -Papa Foto +PageGrabber +page scorer +PageScorer Pavuk -PeoplePal +PECL::HTTP PHPCrawl Picscout +Picsearch +Pi-Monster PleaseCrawl plumanalytics -POE-Component-Client-HTTP -ProWebWalker +Probethenet +ProPowerBot Pump PxBroker QueryN Metasearch -RankActiveLinkBot +RankActive RankingBot -RankurBot -RedesScrapy -Rogerbot -SalesIntelligent +RealDownload +Reaper +RebelMouse +ReGet +Ripper +RocketCrawler ScanAlert -Scanbot +ScoutJet Scrapy SearchmetricsBot +SemrushBot SEOkicks -SEOkicks-Robot -SEOlyticsCrawler -SEOprofiler -SEOstats +Seomoz sexsearcher -Siphon -SISTRIX -SiteExplorer -SiteLockSpider +Sitebeam +Siteimprove SiteSnagger +SiteSucker Site Sucker -Sitevigil -SmartDownload +SlySearch SMTBot -Snake -Sociscraper -sogouspider +Snoopy Sogou web spider Sottopop -SpaceBison +Spammen +sp_auditbot Spbot -SputnikBot Sqlmap -SuperHTTP +Steeler +Stripper +Sucuri +SuperBot Surfbot -SurveyBot Suzuran -sysscan -T8Abot +Szukacz +T0PHackTeam tAkeOut -Telesoft +Teleport +Telesphoreo Telesphorep The Intraformant -TheNomad TightTwatBot -Titan +Toata +Toweyabot Tracemyfile -Trendiction trendiction.com -Typhoeus -URLy Warning -Vacuum +trendiction.de +True_Robot +UnisterBot +Upflow +URLy.Warning +VB Project VCI -VoidEYE +Virusdie Voil -Wallpapers/3.0 +WallpapersHD WBSearchBot -WebAuto -WebBandit -WebCollage WebEnhancer -Web Enhancer Web Fetch -Web Fuck +WebGo IS +WebLeacher WebmasterWorldForumBot -webmeup-crawler -WebPix -WebSauger +WebReaper Web Sauger -Webshag WebsiteExtractor -Website Quester Web Sucker WebWhacker -WebZIP +Whacker Whatweb +Who.is Bot +Widow Wonderbot -Woobot -Wprecon +WWW-Collector-E WWW-Mechanize -WWW::Mechanize -WWWOFFLE -Xaldon WebSpider -Xaldon_WebSpider -Xenu -xpymep1.exe -Zade +x22Mozilla +YoudaoBot Zauba Zeus zgrab -ZmEu -ZumBot +Zitebot +ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index cb1d0418e..fe39b5a44 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -addbin.men -allblogroll.com -cleanmypc.com -copblock.org -coprofam.org -notebook-pro.ru -stairliftsarea.com -truebeauty.cc -tversvet.ru -vitanail.ru +adloads.com +continent-e.tv +findclan.org +game300.ru +photorepair.ru +puteshestvennik.com +spamnuker.com +visa-china.ru +xtube.com +zmoda.hostreo.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 05e290919..2b9fcd8be 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -100dollars-seo.com -108shot.com -12bet.com +0n-line.tv 1314dh.com -1kdailyprofit.me -2000k.ru -202ch.com -23kw.ru -256bit.by -2drittel.de 2kata.ru -300richmond.co.nz -5i2.net -5kstore.com -7zap.com +2pxg8bcf.top +51.la +5-steps-to-start-business.com +6hopping.com 991mostfm.co.id -aa8b68101d388c446389283820863176e7.com -absolutelyanalog.com -absolute-s.ru -academiacsmendoza.org -addblueoff.com.ua -adf.ly -adhome.biz -adsloads.com -adultwebhosting.info -advancedcleaner.com -agecheckadult.com -akusherok.ru -alfabot.xyz -alfa-medosmotr.ru -alldezservice.kz -alldownload.pw -allesohnegirls.net -allnews.md -alyeskaresort.com -andpolice.com -angkortours.vn -apioasisspacenet-a.akamaihd.net -apiportalmorecom-a.akamaihd.net -apple.com-cleaner.systems -arcteryxsale.online -arewater.com -asiavirtualsolutions.com -atmagroup.ru -atovh.local-finders.com +aaa38852e886ac4af1a3cff9b47cab6272.com +abbanreddy.com +abcguru.xyz +adconscious.com +addbin.men +adloads.net +adman.se +admitad.com +ad-words.ru +adzpower.com +aghanyna.com +aibolita.com +alldubai.biz +alta-realestate.com +alvtank.se +amateurgalls.com +amazon-adsystem.com +anal-acrobats.com +androids-store.com +angry-fermi-7633.arukascloud.io +anime.dougasouko.com +antons-transporte.de +aosexkontakte.net +apiqualitinknet-a.akamaihd.net +apiwebwebgetcom-a.akamaihd.net +arendas.net +arenda-yeisk.ru +artdestshop.eu +asdfz.pro +ausergrubhof.info autoblogger24.info -auto-moto-elektronika.cz -autoua.top -autowebmarket.com.ua -avtocredit-legko.ru -azazaporn.com -bag77.ru +autogrep.ru +autorn.ru +availit.weebly.com +bad-stars.net balitouroffice.com -banki76.ru -basedecor.ru -bashtime.ru -bazaronline24.ru +bamo.xsl.pt +bankcrediti.ru +barbourjackets.online bedcapdealers.com -beget.tech -belstaffstore.online -berdasovivan.ru -bestserials.com -bestsexyblog.com -bet-prognoz.com -betwinservice.com -bharatdefencekavach.com -bioca.org -bioscorp.ru +bestwaystogetpaid.us +bingoporno.com bio.trade-jp.net -bird1.ru -biteg.xyz +birzha-truda.eu +bkns.vn +blackhatworth.com blog.f00kclan.de -blog-hits.com -bocoarchives.org +bloggedporn.com +bloggerads.net +blpmovies.com boltalko.xyz -boobsimge.com -breget74.ru -brendbutik.ru -brg8.com -bristolhotel.com.ua +bongiornos.info +boostmyppc.com +brandedleadgeneration.com +brk-rti.ru brus.city -btc4u.ru -burn-fat.ga -bus-offer.com -buyantiviralwp.com -callawaygolfoutlet.online -callcafe.info -calstaterealty.us -cartechnic.ru -cartierbraceletsreplica.pw -casablancamanor.co.za -cbcseward.com -cdnnetwok.xyz +buigas.00it.com +buyessaynow.biz +buzzurl.jp +bvlgariring.xyz +bytimedance.ru +cablecar.us +cam-kontakte.org +candytiner.com +canoncdriverq3.pen.io +carina-sy.de +carsdined.org +carsplate.com +cartierlovestore.xyz +cartierwatch.xyz cenoval.ru -centre-indigo.org.ua -cerev.info -cfjump.com +charmstroy.info +chastnoeporno.com chcu.net -chiblackhawks-jerseys.com -chinaelitecheapjerseys.com -cialgenisrx.com -cialisndbrx.com -cialis-samples.com -cityadspix.com -clean-start.net -clevernt.com -clickhype.com -clipartnew.com -clknsee.com -cognitiveseo.com -collegerentals.com +children-learningreading.info +chimiver.info +chinajerseyswholesalecoupons.com +cialischstgerts.com +cibpenonptib.flu.cc +cineacademy.ru +ci.ua +clarithromycin500mg.com +cleanmypc.com +clicksor.net +clmforexeu.com +colehaanoutlet.store +compliance-alex.top +compliance-alex.xyz compliance-donald.xyz -comsysnet.com -contextualyield.com -coolbar.pro -coop-gamers.ru -corta.co -cyse.tk -dame-ns.kz -daneshetabiat.com -danschawbel.com -date-withme.com -db.speedup.gdn -delitime.info -detmebel.su -devochki.top +compliance-elena.top +compliance-mary.top +cool-wedding.net +coslab.club +counter.yadro.ru +crazyprotocol.com +creams.makeforum.eu +creativehutindia.com +csgo4.win +cupday.com +custodieva.ru +dandiyabeats.in +daptravel.com +daretodonate.co +data.vtc.pw +dcj-nn.ru +dellalimov.com +dentalpearls.com.au +depositfiles-porn.ga +derevesendeco.com +descargar-musica-gratis.net +deshevo-nedorogo.ru +detskie-zabavi.ru +de.zapmeta.com +dgfitness.co dieswaene.com -divku.ru -dnepropetrovsk.mistr-x.org -dnsrsearch.com -doctissimo.fr -domainroam.win -dominterior.org -dotmass.top -droidlook.net -duplicashapp.com -dwomlink.info -dyshagi.ru -eas-seo.com -ed-shop01.ru -elektir.ru -elektrischeziga.livejournal.com -elektrischezigarette2.devhub.com -elektrozigarettekaufen2.blogse.nl -elvenmachine.com -encodable.com -eonpal.com +dikqlyremy.info +discovertreasurenow.com +divatraffic.com +divci-hry.info +documentsite.net +dodlive.mil +dogclothing.store +domaincdn.xyz +domcran.net +doska-vsem.ru +dosugrostov.site +doublepimp.com +downloadeer.net +download-wallpaper.net +draniki.org +drunkmoms.net +easytuningshop.ru +e-buyeasy.com +econom.co +eduinfosite.com +eduserver.net +eldiariodeguadalajara.com +elektrozigaretten1.postbit.com +enbersoft.com +energydiet-info.ru erotik-kostenlos.net -escort-russian.com -esnm.ru -esopini.com -ewebarticle.info -eyes-on-you.ga -eywords-monitoring-your-success.com -ezigarettekaufen.myblog.de -ezigarettenkaufen1.deviantart.com -ezigarettenkaufen2.dreamwidth.org -ezigaretteshop.webs.com -f1nder.org +eyessurgery.ru +ezigarettekaufen1.hpage.com +family1st.ca +familyphysician.ru fashion-stickers.ru -fba-mexico.com -fdzone.org -fefo.gdn +fertilitetsradgivningen.se +fetlifeblog.com fickblock18.com -financetip.eu -fireads.men -fishtauto.ru -fitnesspiks.com -fitness-video.net -flavors.me -flprog.com -footbalive.org -forex21.ru -forum-engineering.ru +fickenbumsen.net +fifa-coins.online +filesmonster.porn +firstdrugmall.ru +folowsite.com +forexgb.ru +forex-indextop20.ru +formulaantiuban.com +fortunejack.com fotosfotos.eu -foto-weinberger.at fotoxxxru.com -franecki.net -free-fbook-traffic.com -free-traffic.xyz +frameimage.org +freeseedsonline.com fullgirl.ru -fungamelands.com -funponsel.com -funtoonez.com -games.kolossale.ru -game-top.su -ge0ip.com -get-free-traffic-now.com -getlaid-xxxhookupdirect.com -ghernnqr.skyrock.com +fusoradio.info +g7m.pl +galaxyflowers.ru +gamedayhouse.com +gazoblok.net.ua +getdot.ru +getyourimage.club gigablast.com -girls-ufa.ru -glastecfilms.com.my -goldadpremium.com -goldandcard.ru -golmau.host.sk -googlemare.com -gpirate.com -gratis-sexkontakte.com -gratuitxblcodes.com -greenzaim.ru -gta-top.ru -guod.me -h2monline.com -hahashka.ru -hdwalls.xyz -healgastro.com -heartofpayne.xyz -hexpilot.com -heygidday.biz -highland-homes.com +gok-kasten.net +goodwinmetals.co +gotomontenegro.net +greenidesign.co +gribkovye-zabolevaniya.com +groupmoney.ru +hasshe.com +hausfrauensex18.com +hesteel.pl +high-speed1.net himgaws.pw -hledejvshopech.cz -hobbyhuren-datenbank.com -homemature.net -hotxnights.info -howtoclean.club -howtowhitenteethfast.xyz -hplaserjetpdriver8y.pen.io -humanorightswatch.org -idc.com.ua -ikearugs.xyz -ilikevitaly.com -ilovevitaly.ru -imagefinder.site -images-free.net +homeandhealth.ru +homeart.space +honyaku.yahoofs.jp +host-protection.com +hotblog.top +hotenergy.ru +housekuba.org +hspline.com +hut1.ru +hyhj.info +iboss.com +ideoworld.org +igrovyeavtomaty777.ru +ilmexico.com +img.wallpaperstock.net +inbabes.sexushost.com +inclk.com +indiasourcemart.in innoslicon.com -integritylandscapeservices.com -intim-uslugi.info -iscblog.info -itis4you.com -itsdp3.com -japan-bearings.ru -jasonpartington.com -jav-fetish.site -javrip.net -je7.us -joessmogtestonly.com -juliadiets.com -jyvopys.com -kabbalah-red-bracelets.com -kakadu-interior.com.ua -kambasoft.com -karusel-market.ru -kerwinandcariza.com -keywordsuggest.org -kinopolet.net -kiwi237au.tk -kleine-titten.biz -knowyournextmove.com -kolotiloff.ru -kunstaktien.info -kvartiry-remont.ucoz.ru -lankarns.com -laptop-4-less.com -law-check-eight.xyz -ledpolice.ru -legalrc.biz -limads.men -linkbux.ru -lion.conboy.us -loadingpages.me -locustdesign.co -longlifelomilomi.info -louisvuittonoutletstore.net -lumb.co -m0r0zk0-krava.ru -macfix.life -mac-shield.com -magnetic-bracelets.ru -malls.com -mapquestz.us -marsgatan.com +insomniagamingfestival.com +in-tandem.co +investsuccess.org +investyb.com +iqupdatetmz.win +iwantmyfreecash.com +jagg.info +javlibrary.cc +javxxx18.com +journeydownthescale.info +justbcause.com +justdating.online +jwss.cc +kamen-e.ru +keyword-suggestions.com +keywordsuggests.com +kinobest.pl +kinocccp.net +kino-ecran.ru +kinohit1.ru +kit-opt.ru +klikbonus.com +komp-pomosch.ru +krasnodar.zrus.org +lernur.net +letsart.ru +liran-locks.com +liveinternet.ru +liver-chintai.org +llastbuy.ru +localflirtbuddies.com +low-format.ru +macnewtech.com +macrotek.ru +magazin-pics.ru +magazintiande.ru +magicplayer-s.acestream.net +makemoneyonline.com +marketingtechniques.info +marwer.info masthopehomes.com -max-eclat.men -maxthon.com -mazda-roadsters.com -mega-bony2017.pl -meget.co.za -menetie.ru -metalonly.info -microsoftportal.net -midst.eu -millionare.com -mil-stak.com -mindeyegames.com -misandesign.se -mister-shop.com -mixx.com -moi-glazki.ru -money-for-placing-articles.com -moneytop.ru -monitorwebsitespeed.com -moscow-region.ru -mountainstream.ms -mrpornogratis.xxx -mustat.com -mylida.org -na15.ru +matpre.top +mature.free-websites.com +medicineseasybuy.com +meduza-consult.ru +meendo-free-traffic.ga +metaxalonevstizanidine.blogspot.com +milblueprint.com +minharevisao.com +more-letom.ru +moskva.nodup.ru +mostantikor.ru +mozello.ru +mp3downloadhq.com +musezone.ru +musicvidz.ru +muz-baza.net +muztops.ru +mydoctorok.ru +myonigroup.com +naperehresti.info +narkologiya-peterburg.ru +narutonaruto.ru neks.info -netallergy.ru -networkad.net -new-post.tk -newyorkhotelsmotels.info -nfljerseyscheapchinabiz.com -noumeda.com +netanalytics.xyz +newtechspb.ru +nextlnk12.com +nfljerseys.online +nikkiewart.ru +noclegonline.info november-lax.com -novosibirsk.xrus.org -npoet.ru -okonich.com.ua -old-rock.com -onclickpredictiv.com -onemactrckr.com -online-templatestore.com -o-o-11-o-o.com -openstat.com -ororodnik.goodbb.ru -ostroike.org +nvformula.ru +oddamzadarmo.eu +offer.camp +olvanto.ru +onlywoman.org +openfrost.com +open-odyssey.org +optitrade24.com +orgasmatrix.com +ossmalta.com otbelivanie-zubov.com -ovirus.ru -pandroid.co +owohho.com +paclitor.com +park.above.com +paydayloanslocal.com p-business.ru -pdns.cz -photo.houseofgaga.ru -photosaga.info -picmoonco.pw -pierrehardysale.online -pinsdaddy.com -pintattoos.com -pinwallpaper.xyz -pl-vouchers.com +pelfind.me +pensplan4u.com +perfection-pleasure.ru +physfunc.ru +piccdata.com +piluli.info +pinstake.com +plaff-go.ru pochtovyi-index.ru -pomoc-drogowa.cba.pl +podshipniki-ntn.ru +polybuild.ru pontiacsolstice.info -pony-business.com -popunder.ru -pornhubforum.tk -pornohd1080.online -pornoplen.com -pornosexrolik.com -porno.simple-image.com.ua +popcash.net +popugauka.ru +popugaychiki.com +porn555.com +pornodojd.ru +pornogratisdiario.com +pornolook.net +pornonik.com +porno-raskazy.ru pornowarp.info -pospr.waw.pl -predmety.in.ua -prizesbook.online -profitwithalex.info -promodj.com -prosmibank.ru -pssucai.info +pornoxxx.com.mx +pornstartits.xblog.in +pravoholding.ru +prefersurvey.net +preg.marketingvici.com +preparevideosafesystem4unow.space +princeadvantagesales.com +printie.com +privacyassistant.net +privatbank46.ru +producm.ru +proekt-mos.ru +profitkode.com +profitsport.club +promgirldresses.xyz +prostitutki-yaroslavlya.org pts163.ru -qld10000.net -qor360.com -quickchange.cc -quick-offer.com -qwarckoine.com -ragecash.com -rankia.com -redhotfreebies.co.uk -reeyanaturopathy.com -refererx.com -relax.ru -remorcicomerciale.ro -rentaremotecomputer.com -rock-cafe.info -ros-ctm.ru -rospromtest.ru -rubbed.us -russia-tao.ru -rvi.biz -salmonfishingsacramentoriver.com -salut-camp.ru +pulseonclick.com +pushdata.sendpulse.com +puzzleweb.ru +rabot.host.sk +randalljhoward.com +randki-sex.com +rankings-analytics.com +ranksignals.com +r-control.ru +realitykings.com +reckonstat.info +remont-fridge-tv.ru +renecaovilla.online +replica-watch.ru +rfid-locker.co +ringporno.com +ritlweb.com +rp9.ru +runofilms.ru +sabaapress.com +sale-japan.com +sa-live.com sampleletters.net -sandhillsonline.com -sbprabooks.com -scanner-alex.top -scanner-mark.top -scanner-mary.top +santechnik.jimdo.com +sape.top +sbwealthsolutions.ca +scanmarine.info +scanmyphones.com +scanner-don.top +scanner-george.top +scanner-john.top screentoolkit.com -sdelatmebel.ru -searchaddis.com -searchimpression.com -seasaltwithfood.com -selectads.men -semaltmedia.com -seo-2-0.com -seocdvig.ru -seoexperimenty.ru +secret.xn--oogle-wmc.com +seemoreresultshu-a.akamaihd.net +sell-fb-group-here.com +seminarygeorgia59.ga serpstat.com -serptehnika.ru -sexfreepornoxxx.com -sexkrasivo.net -sexpartygirls.net -sextracker.com -sfj-ror.no -sherlock.se -shmetall.com.ua -shoesonlinebuy.cn -shoplvlv.us +sexflirtbook.com +sexs-foto.com +sex-tracker.de +sexyali.com +shillyourcoins.com +sibecoprom.ru +sideeffectsoftizanidine.blogspot.com siha.de -silverage.ru -site-analyzer.com -smart-scripts.com -softlinesolutions.me -songplanet.ru -space2019.top -spidtest.org -sprttrack.com -spyfu.com -spywarebegone.com +sisiynas.ru +site3.free-share-buttons.com +smartpet.ru +smokewithrabbits.com +sms2x2.ru +smsactivator.ru +social-s-hhh.xyz +soft-terminal.ru +solicita.info +spaceshipad.com +spamnuker.com squidoo.com -srgwebmail.nl -steelmaster.lv -stevemonsen.com -stroilka.info -students-cheapskate.ml -studiofmp.com -success-seo.com -sugarlyflex.pw -superkanpo.com -tabakur77.com -tattoo33.ru +staff.prairiesouth.ca +star61.de +stefanbakosab.se +sterva.cc +stillmiracle.com +stroyhelp-dv.ru +stroymonolit.su +styro.ru +sudexpert66.ru +suggest-keywords.com +superinterstitial.com +svetlotorg.ru +svetodiodoff.ru +sygraem.com +symphonyintegratedhealthcare.com +syvertsen-da.no +takeprofitsystem.com +td-l-market.ru techart24.com -teenbbw.yopoint.in -telegramdownload10.com -templates.franklinfire.co -tfxiq.com -thebestweightlosspills.ovh -thedownloadfreeonlinegames.blogspot.com -theguardlan.com -thetattoohut.com -the-torrent-tracker.blogspot.com -thruport.com -timeallnews.ru -tizanidinehcl2mgsideeffects.blogspot.com -tizanidinerecreationaluse.blogspot.com -toon-families.com -topmira.com -torrentdownloadhub.com -totu.info -toursmaps.com -track112.site -tracklead.net -tracksurf.daooda.com -trubywriting.com +thaisamkok.com +thebestphotos.eu +thecounter.com +thin.me.pn +tiens2010.ru +tizanidine4mgstreetprice.blogspot.com +tizanidinedosage.blogspot.com +tizanidinedosageforsleep.blogspot.com +tizanidinehcl2mgtablet.blogspot.com +tizanidineoral.blogspot.com +tizanidineorflexeril.blogspot.com +tjkckpytpnje.com +tmm-kurs.ru +tootoo.to +top10-online-games.com +topsiteminecraft.com +torrents.cd +tracksz.co +trafficgenius.xyz +triumf-realty.ru truck-land.life -trustaffs.com -trustedmaccleaner.com -turbabitload.weebly.com -twinderbella.com -twitlinks.com -twittruth.com -uchil.net -ucoz.ru -ugogo.info -underthesite.com +ttsq.fr +tuberkulezanet.ru +unblocksit.es +unrealcommander.org +uptime-delta.net urlopener.com -vacances-voyages.info -veles.shop -vellings.info -venerologiya.com -venta-prom.ru -viagra.pp.ua -video-chat.cn -videochat.mx -vkontaktemusic.ru -vod.com.ua +urzedowski.eu +ustion.ru +validdomain.xyz +vesnatehno.ru +vezuviy.su +videochat.world +video-woman.com +vinsit.ru +vintontech.info +visa-pasport.ru +vitanail.ru vodkoved.ru vonradio.com -wahicbefa31.soup.io -walkme.com -wallpapers-best.com -webads.co.nz -webjam.com -webnode.me +voprosotvet24.ru +vostoktrade.info +vozbujdenie.com +vrnelectro.ru +vseuznaem.com +vvon.co.uk +vykup-avto-krasnodar.ru +w7s.ru +watch-movies.ru +watchmyfb.pl +webalan.ru +webenlace.com.ar +webmonetizer.net +webpromotion.ae +webshoppermac.com website-analyzer.info -wesharepics.com -whatsappdownload10.com +websites-reviews.com wheelchairliftsarea.com -wnhjavlhezp.gq -world-mmo.com -xbaboon.com -xfire.com -xgames-04.com -xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai -xn--q1a.xn--b1aube0e.xn--c1acygb.xn--p1ai -xolodremont.ru -xvideosbay.com -xwatt.ru -xz618.com -yaminecraft.ru -yopoint.in +wholesalejerseychinaoutlet.com +witherrom55.eklablog.fr +wnoz.de +wordkeyhelper.com +wpsecurity.website +wpthemedetector.co.uk +writingservices17.blogspot.ru +www.timer4web.com +x5market.ru +x-lime.com +xmronta.com +xn------7cdbapdecfd4ak1bn0amjffj7afu3y.xn--p1ai +xn----7sbahjd3btneuw1joc.xn--p1ai +xn--80aeahghtf8ac5i.xn--p1ai +xn--80aikhbrhr.net +xportvusbdriver8i.snack.ws +xxxtube69.com +yeartwit.com +youandcredit.ru +youradexchange.com +youraticles.pl +yourothersite.com yoursearch.me -yurgorod.ru -zahvat.ru -zajm-pod-zalog-nedvizhimosti.ru -zeroredirect7.com -zetmaster.ru -zigarettenonlinekaufen1.bloog.pl -zigarettenonlinekaufen2.drupalgardens.com -znaniyapolza.ru +youthreaders.com +yuweng.info +zakon-ob-obrazovanii.ru +za-music.mymobiplanet.com +zarepta.com +zrelaya.pw +zs2vm.top +zzlgxh.com diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index 53acca8c5..114c440b8 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -1000n1.ru -101flag.ru -76brighton.co.uk -7fon.ru -98oi.ru -aaa38852e886ac4af1a3cff9b47cab6272.com -adtech.us -aktobe.xkaz.org -albuteroli.com -alfa9.com -alfa-medosmotr.ru +101lesbian.xyz +2x2fan.ru +aa8b68101d388c446389283820863176e7.com +abbanreddy.com +adinterax.com +adultfriendfinder.com +aero2.ru +aihelen.net +albamargroup.com +alborzan.com +alert-hgd.xyz algerianembassy.co.in -alldownload.pw -alltheviews.com -amateurgalls.com -amazingpic.net -angry-fermi-7633.arukascloud.io -animal-drawings.com -aproposde.com -arabseyes.com -architecturebest.com -artdestshop.eu -arvut.org -ascat.porn -aviav.eu -avtochehli.by -bablonow.ru -bestcurs.org -bestfortraders.com -bestofpicture.com -betterscooter.com +allsilver925.co.il +alot.com +altamayoztourism.com +animali.deagostinipassion.it +approved.su +arendas.net +asdfg.pro +asmxsatadriverin.aircus.com +autotop.com.ua +autotrends.today +azbukadiets.ru +baladur.ru +bala.getenjoyment.net +bazaronline24.ru +besthomemadepornsites.com +bestwebsitesawards.com bezlimitko.xyz -bloglag.com -bloke.com -brg8.com -bum.com.ru -canzoni.ru -carabela.com.do -cartiertrinity.xyz -cbb1smartlist12.click -cdnnetwok.xyz -cheapmarmot.online -chee-by.biz -christianlouboutinreplica.win -cialischstgerts.com -citizenclsdriveri7.pen.io -clickintext.net -colobit.biz -cometorussia.net -conciergegroup.org -contextualyield.com -danceuniverse.ru -darkbooks.org -dentalpearls.com.au -designdevise.com -dichvuvesinhhanoi.com -diegolopezcastan.com -div.as -dtm-spain.com -earn-from-articles.com -earthmagic.info -easytuningshop.ru -ecommerce-seo.org -elektrischezigarette2.onsugar.com -elektrozigarette2.wordpressy.pl -emailaccountlogin.co -encodable.com +binomo.kz +bio-japan.net +bitcoins-live.ru +bizlist.com.de +blackcurranthumidifiers.site +blog.koorg.ru +bluerobot.info +bmwhighperformers.com +board.f00d.de +bonzybuddy.com +boostmyppc.com +bristolhotel.com +buyk.host.sk +buytizanidineonline.blogspot.com +cakesplus.com.au +casablancamanor.co.za +cercacamion.it +chocoslim.pro +cjs.com.ru +cleanersoft.com +com-onlinesupport.host +companies-catalog.com +compliance-jess.top +compliance-josh.top +counciltally.com +customerpromos-a.akamaihd.net +cyprusbuyproperties.com +daisye.top +darodar.com +davidovic.info +dokumentalkino.net +dood.live +drunkenstepfather.com +dvrlists.com +edelstahlschornstein-123.de +efnor-ac.com +elektronischezigarettekaufen1.myblog.de +elektrozigaretten1.postbit.com +elektrozigaretten1.tumblr.com enews.tech -eralph.tk -esnm.ru -essay-writing.work -eugenevaultstorage.com -ezigarettenkaufen1.pagina.gr -fermersovet.ru -focalink.com -funnypica.com -fz139.ttk.ru -gazoblok.net.ua -gilsonchiro.xyz -go2jump.org -gobongo.info -goodhumor24.com -goodnightjournal.com -gsasearchenginerankerdiscount.com -gsasearchengineranker.xyz -hanwei.us -hawaiielectriclight.com +enginebay.ru +etotupo.ru +europages.com.ru +exlarseva.webblog.es +expertblog.info +facebook-mobile.xyz +famix.xyz +filesclub.net +forum-engineering.ru +frbizlist.com +fr.netlog.com +front.to +fyxabomiw.ru +gaming-journal.com +gaytube.com +geckoandfly.com +getmiro.com +getridofstretchmarks.org +ghazel.ru +girlsatgames.ru +glasof.es +gnuetella.com +go2mike.ru +gonextmedia.com +guigyverpo.cf +havepussy.com +hcate.com +hd-film.pl healbio.ru -heliko.no -helvetia.com.ua -hermesreplica.pw -hifidesign.ru -highstairs-a.akamaihd.net -histock.info -hop.clickbank.net -hoverboard360.es -howtoclean.club -ictizanidinehcl4mg.blogspot.com -id-forex.com -iflycapetown.co.za -insomniagamingfestival.com -investingclub.ru -investyb.com -istizanidineanarcoticdrug.blogspot.com -justprofit.xyz -kabbalah-red-bracelets.com -kashubadesign.ru -keyword-suggestions.com -kidskunst.info -kmgamex.cf -krasnodar.zrus.org -kretpicf.pw -lapitec.eu -leadn.pl +hildinghr.se +hollywoodactress.info +hyiphunter.org +icaseclub.ru +ilte.info +imagefinder.site +inboxdollars.com +indetiske.ya.ru +internetartfair.com +ip-guide.com +jagg.info +javxxx18.com +jwss.cc +kamorel.com +klin3952.ru +krasnodar.xrus.org +laexotic.com +larger.io +law-check-seven.xyz +lennatin.info +letmacworkfaster.world +letslowbefast.site life.biz.ua -light.ifmo.ru -lol-smurfs.com -longadventure.com -lsp-awak-perikanan.com -luckyshop.net.ua -makedo.ru -mexicotravelnet.com -mhtr.be -mlf.hordo.win -mobstarr.com -moetomnenie.com -money-for-placing-articles.com -mototsikl.org -mug-na-chas-moscow.ru -my-aladin.com -mydirtyhobby.com -mydownlodablefiles.com -myfreecams.com -narkologiya-belgorod.ru -november-lax.com -nvssf.com -o00.in -offtime.ru +lincolntheatre.com +lirunet.ru +locustdesign.co +manimpotence.com +med-zdorovie.com.ua +meltwater.com +memberty.com +menetie.ru +mil-stak.com +moesonce.com +mokrayakiska.com +monclercheap.xyz +mostantikor.ru +musezone.su +mxgetcode.com +mymercy.info +myperiod.club +myyour.eu +nalogovyykodeks.ru +new-post.tk +nicefloor.co.uk +nordvpn.com +nzfilecloud.weebly.com +obuv-kupit.ru +odesproperty.com +one-gear.com +onemactrckr.com +onescreen.cc onlainbesplatno.ru -online-hd.pl -onlineserialy.ru -parlament.biz -pb-dv.ru -pekori.to -photorepair.ru -pinwallpaper.xyz -pliks.pl -pokemongooo.ml -porno-transsexuals.ru -pozdravleniya-c.ru -proftests.net -pronekut.com -qexyfu.bugs3.com -qwarckoine.com -rangoman.date +oren-cats.ru +ourville.info +panchro.co.uk +paphoselectricianandplumber.com +peekyou.com +pensplan4u.com +petrovka-online.com +picsforkeywordsuggestion.com +picswe.com +pkr1hand.com +poppen-nw.net +popugaychiki.com +pornowarp.info +prize44.com +prodvigator.ua +profitkode.com +quality-traffic.com +rankingchart.de rankings-analytics.com -rapidgator-porn.ga -refads.pro -relax.ru -renecaovillasale.online -revistaindustria.com -rockprogblog.com -rogervivierforsale.com -rusoft-zone.ru -samo-soznanie.ru -sardinie.us -savememoney.co.za -scalerite.co.za -sdrescher.net -seinterface.com -seksotur.ru -seo-2-0.com -sergiorossistore.online -servethis.com -sexad.net -sexiporno.net -sharpchallenge.com -shikiso.info -shillyourcoins.com -ship-marvel.co.ua -shohanb.com -shoppingjequiti.com.br -shymkent.xkaz.org -slowmac.tech -social-fun.ru -sotkal.lark.ru -special-porn.com -spidtest.org -spravka130.ru -sprttrack.com +ranksonic.info +replicaclub.ru +ribieiendom.no +ric.info +rome2rio.com +rostov.xrus.org +royalcar-ufa.ru +rus-pornuha.com +saddiechoua.com +saneyes.com +saugatuck.com +scanner-barak.top +scanner-viktor.top +scrapinghub.com +searchlock.com +searchtooknow-a.akamaihd.net +seocheki.net +shakhtar-doneck.ru +sharebutton.org +share-buttons-for-free.com +siteimprove.com +sitevaluation.com +sivs.ru +soda.media +sohoindia.net +sortthemesitesby.com spymac.net -srgwebmail.nl -stefanbakosab.se -sugvant.ru -sukarame.net -summerlinhomes411.info -thebluenoodle.com -ticketsys.inetwd.com -timdreby.com -tizanidinehcl2mgtablet.blogspot.com -tizanidineinfo.blogspot.com -tizanidineingredients.blogspot.com -tizanidinemusclerelaxant.blogspot.com -tizanidinepills.blogspot.com -tizanidinesideeffectsweightloss.blogspot.com -topads.men -topbestgames.com -topkarkas.com -topshef.ru -torrentgamer.net -torrent-to-magnet.com -touchmods.fr -traffic100.com -traffictrade.life -trubywriting.com -trustedmaccleaner.com -turbabitload.weebly.com -uac.net.au -uhdtv.website -ultimateclassicrock.com -urll.eu -uroffer.link -vandrie-ict.nl -vapomnoncri.tk -video-chat.cn -videtubs.pl -vietimgy.pw -vitanail.ru -vksaver-all.ru +srdrvp.com +starpages.net +statoutlook.info +stevemonsen.com +stillmiracle.com +studiofaca.com +sureone.pro +tds-advert002.info +tfxiq.com +thedownloadfromwarez.blogspot.com +theendivechronicles.com +titan-ads.life +titan-cloud.life +tizanidinehcl4mgtablet.blogspot.com +tpu.ru +trafaret74.ru +traffmonster.info +traffpartners.com +tula.mdverey.ru +umg-stroy.ru +ussearche.cf +veerotech.com +videosbox.ru +vip-file.com +vkgaleria.com +vkonche.com +vote-up.ru vut.com.ru -vvon.co.uk vzube.com -warning.or.kr -watsonrealtycorp.com -wdfdocando.com -westen-v.life -whatistizanidinehclusedfor.blogspot.com -wineitudes.wordpress.com -woodyguthrie.se -worldmusicfests.com -www.kabbalah-red-bracelets.com -xn----7sbaphztdjeboffeiof6c.xn--p1ai -xn--e1afanlbnfckd7c3d.xn--p1ai -yeartwit.com -yourads.website -ytmnd.com -zaloadi.ru +w3data.co +watchmygf.net +wdrake.com +webpromotion.ae +website-speed-checker.site +wechatdownload10.com +welcomeauto.ru +westsextube.com +wieseversa.no +wikes.20fr.com +wleuaprpxuvr.ga +wordpress-crew.net +worldhistory.biz +wowcasinoonline.ooo +ww1943.ru +x5market.ru +xn--h1ahbi.com.ua +xn--l1aengat.xn--p1ai +xtraffic.plus +xxlargepop.com +yoursite.com +zajm-zalog-krasnodar.ru zapatosenventa.info -zeroredirect2.com -zeroredirect8.com +zastenchivosti.net +zx6.ru diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index a7215fb74..525a76fd6 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -5,13 +5,6 @@