diff --git a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf index 83e853f14..36e36114a 100644 --- a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf @@ -33,4 +33,4 @@ # 111.111.111.111 0; -35.224.112.202 0; +35.192.136.167 0; diff --git a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf index 60944cc49..d39f16e78 100644 --- a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1540 -### Updated: Sun Jun 23 15:20:43 SAST 2019 +### Version: V3.2019.06.1541 +### Updated: Sun Jun 23 16:11:00 SAST 2019 ### Bad Referrer Count: 6693 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf index 83e853f14..36e36114a 100644 --- a/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf @@ -33,4 +33,4 @@ # 111.111.111.111 0; -35.224.112.202 0; +35.192.136.167 0; diff --git a/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf index 18e5b9d75..a9e0ee294 100644 --- a/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf @@ -10,8 +10,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1540 -### Updated: Sun Jun 23 15:20:44 SAST 2019 +### Version: V3.2019.06.1541 +### Updated: Sun Jun 23 16:11:01 SAST 2019 ### Bad Referrer Count: 6693 ### Bad Bot Count: 556 ################################################### @@ -137,562 +137,562 @@ map $http_user_agent $bad_bot { # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b)" 3; - "~*(?:\b)404checker(?:\b)" 3; - "~*(?:\b)404enemy(?:\b)" 3; - "~*(?:\b)80legs(?:\b)" 3; - "~*(?:\b)Abonti(?:\b)" 3; - "~*(?:\b)Aboundex(?:\b)" 3; - "~*(?:\b)Aboundexbot(?:\b)" 3; - "~*(?:\b)Acunetix(?:\b)" 3; - "~*(?:\b)ADmantX(?:\b)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; - "~*(?:\b)AhrefsBot(?:\b)" 3; - "~*(?:\b)AIBOT(?:\b)" 3; - "~*(?:\b)AiHitBot(?:\b)" 3; - "~*(?:\b)Aipbot(?:\b)" 3; - "~*(?:\b)Alexibot(?:\b)" 3; - "~*(?:\b)Alligator(?:\b)" 3; - "~*(?:\b)AllSubmitter(?:\b)" 3; - "~*(?:\b)AlphaBot(?:\b)" 3; - "~*(?:\b)Anarchie(?:\b)" 3; - "~*(?:\b)Apexoo(?:\b)" 3; - "~*(?:\b)archive.org_bot(?:\b)" 3; - "~*(?:\b)ASPSeek(?:\b)" 3; - "~*(?:\b)Asterias(?:\b)" 3; - "~*(?:\b)Attach(?:\b)" 3; - "~*(?:\b)autoemailspider(?:\b)" 3; - "~*(?:\b)BackDoorBot(?:\b)" 3; - "~*(?:\b)Backlink-Ceck(?:\b)" 3; - "~*(?:\b)backlink-check(?:\b)" 3; - "~*(?:\b)BacklinkCrawler(?:\b)" 3; - "~*(?:\b)BackStreet(?:\b)" 3; - "~*(?:\b)BackWeb(?:\b)" 3; - "~*(?:\b)Badass(?:\b)" 3; - "~*(?:\b)Bandit(?:\b)" 3; - "~*(?:\b)Barkrowler(?:\b)" 3; - "~*(?:\b)BatchFTP(?:\b)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; - "~*(?:\b)BBBike(?:\b)" 3; - "~*(?:\b)BDCbot(?:\b)" 3; - "~*(?:\b)BDFetch(?:\b)" 3; - "~*(?:\b)BetaBot(?:\b)" 3; - "~*(?:\b)Bigfoot(?:\b)" 3; - "~*(?:\b)Bitacle(?:\b)" 3; - "~*(?:\b)Blackboard(?:\b)" 3; - "~*(?:\b)Black\ Hole(?:\b)" 3; - "~*(?:\b)BlackWidow(?:\b)" 3; - "~*(?:\b)BLEXBot(?:\b)" 3; - "~*(?:\b)Blow(?:\b)" 3; - "~*(?:\b)BlowFish(?:\b)" 3; - "~*(?:\b)Boardreader(?:\b)" 3; - "~*(?:\b)Bolt(?:\b)" 3; - "~*(?:\b)BotALot(?:\b)" 3; - "~*(?:\b)Brandprotect(?:\b)" 3; - "~*(?:\b)Brandwatch(?:\b)" 3; - "~*(?:\b)Buddy(?:\b)" 3; - "~*(?:\b)BuiltBotTough(?:\b)" 3; - "~*(?:\b)BuiltWith(?:\b)" 3; - "~*(?:\b)Bullseye(?:\b)" 3; - "~*(?:\b)BunnySlippers(?:\b)" 3; - "~*(?:\b)BuzzSumo(?:\b)" 3; - "~*(?:\b)Calculon(?:\b)" 3; - "~*(?:\b)CATExplorador(?:\b)" 3; - "~*(?:\b)CazoodleBot(?:\b)" 3; - "~*(?:\b)CCBot(?:\b)" 3; - "~*(?:\b)Cegbfeieh(?:\b)" 3; - "~*(?:\b)CheeseBot(?:\b)" 3; - "~*(?:\b)CherryPicker(?:\b)" 3; - "~*(?:\b)CheTeam(?:\b)" 3; - "~*(?:\b)ChinaClaw(?:\b)" 3; - "~*(?:\b)Chlooe(?:\b)" 3; - "~*(?:\b)Claritybot(?:\b)" 3; - "~*(?:\b)Cliqzbot(?:\b)" 3; - "~*(?:\b)Cloud\ mapping(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; - "~*(?:\b)Cogentbot(?:\b)" 3; - "~*(?:\b)cognitiveseo(?:\b)" 3; - "~*(?:\b)Collector(?:\b)" 3; - "~*(?:\b)com.plumanalytics(?:\b)" 3; - "~*(?:\b)Copier(?:\b)" 3; - "~*(?:\b)CopyRightCheck(?:\b)" 3; - "~*(?:\b)Copyscape(?:\b)" 3; - "~*(?:\b)Cosmos(?:\b)" 3; - "~*(?:\b)Craftbot(?:\b)" 3; - "~*(?:\b)crawler4j(?:\b)" 3; - "~*(?:\b)crawler.feedback(?:\b)" 3; - "~*(?:\b)crawl.sogou.com(?:\b)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b)" 3; - "~*(?:\b)Crescent(?:\b)" 3; - "~*(?:\b)CrunchBot(?:\b)" 3; - "~*(?:\b)CSHttp(?:\b)" 3; - "~*(?:\b)Curious(?:\b)" 3; - "~*(?:\b)Custo(?:\b)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; - "~*(?:\b)DataCha0s(?:\b)" 3; - "~*(?:\b)DBLBot(?:\b)" 3; - "~*(?:\b)demandbase-bot(?:\b)" 3; - "~*(?:\b)Demon(?:\b)" 3; - "~*(?:\b)Deusu(?:\b)" 3; - "~*(?:\b)Devil(?:\b)" 3; - "~*(?:\b)Digincore(?:\b)" 3; - "~*(?:\b)DigitalPebble(?:\b)" 3; - "~*(?:\b)DIIbot(?:\b)" 3; - "~*(?:\b)Dirbuster(?:\b)" 3; - "~*(?:\b)Disco(?:\b)" 3; - "~*(?:\b)Discobot(?:\b)" 3; - "~*(?:\b)Discoverybot(?:\b)" 3; - "~*(?:\b)Dispatch(?:\b)" 3; - "~*(?:\b)DittoSpyder(?:\b)" 3; - "~*(?:\b)DnyzBot(?:\b)" 3; - "~*(?:\b)DomainAppender(?:\b)" 3; - "~*(?:\b)DomainCrawler(?:\b)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; - "~*(?:\b)DomainStatsBot(?:\b)" 3; - "~*(?:\b)Dotbot(?:\b)" 3; - "~*(?:\b)Download\ Wonder(?:\b)" 3; - "~*(?:\b)Dragonfly(?:\b)" 3; - "~*(?:\b)Drip(?:\b)" 3; - "~*(?:\b)DSearch(?:\b)" 3; - "~*(?:\b)DTS\ Agent(?:\b)" 3; - "~*(?:\b)EasyDL(?:\b)" 3; - "~*(?:\b)Ebingbong(?:\b)" 3; - "~*(?:\b)eCatch(?:\b)" 3; - "~*(?:\b)ECCP/1.0(?:\b)" 3; - "~*(?:\b)Ecxi(?:\b)" 3; - "~*(?:\b)EirGrabber(?:\b)" 3; - "~*(?:\b)EMail\ Siphon(?:\b)" 3; - "~*(?:\b)EMail\ Wolf(?:\b)" 3; - "~*(?:\b)EroCrawler(?:\b)" 3; - "~*(?:\b)evc-batch(?:\b)" 3; - "~*(?:\b)Evil(?:\b)" 3; - "~*(?:\b)Exabot(?:\b)" 3; - "~*(?:\b)Express\ WebPictures(?:\b)" 3; - "~*(?:\b)ExtLinksBot(?:\b)" 3; - "~*(?:\b)Extractor(?:\b)" 3; - "~*(?:\b)ExtractorPro(?:\b)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; - "~*(?:\b)EyeNetIE(?:\b)" 3; - "~*(?:\b)Ezooms(?:\b)" 3; - "~*(?:\b)facebookscraper(?:\b)" 3; - "~*(?:\b)FDM(?:\b)" 3; - "~*(?:\b)FemtosearchBot(?:\b)" 3; - "~*(?:\b)FHscan(?:\b)" 3; - "~*(?:\b)Fimap(?:\b)" 3; - "~*(?:\b)Firefox/7.0(?:\b)" 3; - "~*(?:\b)FlashGet(?:\b)" 3; - "~*(?:\b)Flunky(?:\b)" 3; - "~*(?:\b)Foobot(?:\b)" 3; - "~*(?:\b)Freeuploader(?:\b)" 3; - "~*(?:\b)FrontPage(?:\b)" 3; - "~*(?:\b)FyberSpider(?:\b)" 3; - "~*(?:\b)Fyrebot(?:\b)" 3; - "~*(?:\b)GalaxyBot(?:\b)" 3; - "~*(?:\b)Genieo(?:\b)" 3; - "~*(?:\b)GermCrawler(?:\b)" 3; - "~*(?:\b)Getintent(?:\b)" 3; - "~*(?:\b)GetRight(?:\b)" 3; - "~*(?:\b)GetWeb(?:\b)" 3; - "~*(?:\b)Gigablast(?:\b)" 3; - "~*(?:\b)Gigabot(?:\b)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; - "~*(?:\b)Gotit(?:\b)" 3; - "~*(?:\b)GoZilla(?:\b)" 3; - "~*(?:\b)Go!Zilla(?:\b)" 3; - "~*(?:\b)Grabber(?:\b)" 3; - "~*(?:\b)GrabNet(?:\b)" 3; - "~*(?:\b)Grafula(?:\b)" 3; - "~*(?:\b)GrapeFX(?:\b)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b)" 3; - "~*(?:\b)GridBot(?:\b)" 3; - "~*(?:\b)GT::WWW(?:\b)" 3; - "~*(?:\b)Haansoft(?:\b)" 3; - "~*(?:\b)HaosouSpider(?:\b)" 3; - "~*(?:\b)Harvest(?:\b)" 3; - "~*(?:\b)Havij(?:\b)" 3; - "~*(?:\b)HEADMasterSEO(?:\b)" 3; - "~*(?:\b)Heritrix(?:\b)" 3; - "~*(?:\b)Hloader(?:\b)" 3; - "~*(?:\b)HMView(?:\b)" 3; - "~*(?:\b)HTMLparser(?:\b)" 3; - "~*(?:\b)HTTP::Lite(?:\b)" 3; - "~*(?:\b)HTTrack(?:\b)" 3; - "~*(?:\b)Humanlinks(?:\b)" 3; - "~*(?:\b)HybridBot(?:\b)" 3; - "~*(?:\b)Iblog(?:\b)" 3; - "~*(?:\b)IDBot(?:\b)" 3; - "~*(?:\b)Id-search(?:\b)" 3; - "~*(?:\b)IlseBot(?:\b)" 3; - "~*(?:\b)Image\ Fetch(?:\b)" 3; - "~*(?:\b)Image\ Sucker(?:\b)" 3; - "~*(?:\b)IndeedBot(?:\b)" 3; - "~*(?:\b)Indy\ Library(?:\b)" 3; - "~*(?:\b)InfoNaviRobot(?:\b)" 3; - "~*(?:\b)InfoTekies(?:\b)" 3; - "~*(?:\b)instabid(?:\b)" 3; - "~*(?:\b)Intelliseek(?:\b)" 3; - "~*(?:\b)InterGET(?:\b)" 3; - "~*(?:\b)Internet\ Ninja(?:\b)" 3; - "~*(?:\b)InternetSeer(?:\b)" 3; - "~*(?:\b)internetVista\ monitor(?:\b)" 3; - "~*(?:\b)ips-agent(?:\b)" 3; - "~*(?:\b)Iria(?:\b)" 3; - "~*(?:\b)IRLbot(?:\b)" 3; - "~*(?:\b)Iskanie(?:\b)" 3; - "~*(?:\b)IstellaBot(?:\b)" 3; - "~*(?:\b)JamesBOT(?:\b)" 3; - "~*(?:\b)Jbrofuzz(?:\b)" 3; - "~*(?:\b)JennyBot(?:\b)" 3; - "~*(?:\b)JetCar(?:\b)" 3; - "~*(?:\b)Jetty(?:\b)" 3; - "~*(?:\b)JikeSpider(?:\b)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; - "~*(?:\b)Joomla(?:\b)" 3; - "~*(?:\b)Jorgee(?:\b)" 3; - "~*(?:\b)JustView(?:\b)" 3; - "~*(?:\b)Jyxobot(?:\b)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b)" 3; - "~*(?:\b)Keyword\ Density(?:\b)" 3; - "~*(?:\b)Kozmosbot(?:\b)" 3; - "~*(?:\b)Lanshanbot(?:\b)" 3; - "~*(?:\b)Larbin(?:\b)" 3; - "~*(?:\b)LeechFTP(?:\b)" 3; - "~*(?:\b)LeechGet(?:\b)" 3; - "~*(?:\b)LexiBot(?:\b)" 3; - "~*(?:\b)Lftp(?:\b)" 3; - "~*(?:\b)LibWeb(?:\b)" 3; - "~*(?:\b)Libwhisker(?:\b)" 3; - "~*(?:\b)Lightspeedsystems(?:\b)" 3; - "~*(?:\b)Likse(?:\b)" 3; - "~*(?:\b)Linkdexbot(?:\b)" 3; - "~*(?:\b)LinkextractorPro(?:\b)" 3; - "~*(?:\b)LinkpadBot(?:\b)" 3; - "~*(?:\b)LinkScan(?:\b)" 3; - "~*(?:\b)LinksManager(?:\b)" 3; - "~*(?:\b)LinkWalker(?:\b)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; - "~*(?:\b)Lipperhey(?:\b)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; - "~*(?:\b)Litemage_walker(?:\b)" 3; - "~*(?:\b)Lmspider(?:\b)" 3; - "~*(?:\b)LNSpiderguy(?:\b)" 3; - "~*(?:\b)Ltx71(?:\b)" 3; - "~*(?:\b)lwp-request(?:\b)" 3; - "~*(?:\b)LWP::Simple(?:\b)" 3; - "~*(?:\b)lwp-trivial(?:\b)" 3; - "~*(?:\b)Magnet(?:\b)" 3; - "~*(?:\b)Mag-Net(?:\b)" 3; - "~*(?:\b)magpie-crawler(?:\b)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b)" 3; - "~*(?:\b)Majestic12(?:\b)" 3; - "~*(?:\b)Majestic-SEO(?:\b)" 3; - "~*(?:\b)Majestic\ SEO(?:\b)" 3; - "~*(?:\b)MarkMonitor(?:\b)" 3; - "~*(?:\b)MarkWatch(?:\b)" 3; - "~*(?:\b)Masscan(?:\b)" 3; - "~*(?:\b)Mass\ Downloader(?:\b)" 3; - "~*(?:\b)Mata\ Hari(?:\b)" 3; - "~*(?:\b)MauiBot(?:\b)" 3; - "~*(?:\b)meanpathbot(?:\b)" 3; - "~*(?:\b)Meanpathbot(?:\b)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b)" 3; - "~*(?:\b)mediawords(?:\b)" 3; - "~*(?:\b)MegaIndex.ru(?:\b)" 3; - "~*(?:\b)Metauri(?:\b)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; - "~*(?:\b)MIDown\ tool(?:\b)" 3; - "~*(?:\b)MIIxpc(?:\b)" 3; - "~*(?:\b)Mister\ PiX(?:\b)" 3; - "~*(?:\b)MJ12bot(?:\b)" 3; - "~*(?:\b)Mojeek(?:\b)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; - "~*(?:\b)Mr.4x3(?:\b)" 3; - "~*(?:\b)MSFrontPage(?:\b)" 3; - "~*(?:\b)MSIECrawler(?:\b)" 3; - "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; - "~*(?:\b)muhstik-scan(?:\b)" 3; - "~*(?:\b)Musobot(?:\b)" 3; - "~*(?:\b)Name\ Intelligence(?:\b)" 3; - "~*(?:\b)Nameprotect(?:\b)" 3; - "~*(?:\b)Navroad(?:\b)" 3; - "~*(?:\b)NearSite(?:\b)" 3; - "~*(?:\b)Needle(?:\b)" 3; - "~*(?:\b)Nessus(?:\b)" 3; - "~*(?:\b)NetAnts(?:\b)" 3; - "~*(?:\b)Netcraft(?:\b)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; - "~*(?:\b)NetLyzer(?:\b)" 3; - "~*(?:\b)NetMechanic(?:\b)" 3; - "~*(?:\b)NetSpider(?:\b)" 3; - "~*(?:\b)Nettrack(?:\b)" 3; - "~*(?:\b)Net\ Vampire(?:\b)" 3; - "~*(?:\b)Netvibes(?:\b)" 3; - "~*(?:\b)NetZIP(?:\b)" 3; - "~*(?:\b)NextGenSearchBot(?:\b)" 3; - "~*(?:\b)Nibbler(?:\b)" 3; - "~*(?:\b)NICErsPRO(?:\b)" 3; - "~*(?:\b)Niki-bot(?:\b)" 3; - "~*(?:\b)Nikto(?:\b)" 3; - "~*(?:\b)NimbleCrawler(?:\b)" 3; - "~*(?:\b)Nimbostratus(?:\b)" 3; - "~*(?:\b)Ninja(?:\b)" 3; - "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)NPbot(?:\b)" 3; - "~*(?:\b)Nutch(?:\b)" 3; - "~*(?:\b)oBot(?:\b)" 3; - "~*(?:\b)Octopus(?:\b)" 3; - "~*(?:\b)Offline\ Explorer(?:\b)" 3; - "~*(?:\b)Offline\ Navigator(?:\b)" 3; - "~*(?:\b)OnCrawl(?:\b)" 3; - "~*(?:\b)Openfind(?:\b)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b)" 3; - "~*(?:\b)Openvas(?:\b)" 3; - "~*(?:\b)OpenVAS(?:\b)" 3; - "~*(?:\b)OrangeBot(?:\b)" 3; - "~*(?:\b)OrangeSpider(?:\b)" 3; - "~*(?:\b)OutclicksBot(?:\b)" 3; - "~*(?:\b)OutfoxBot(?:\b)" 3; - "~*(?:\b)PageAnalyzer(?:\b)" 3; - "~*(?:\b)Page\ Analyzer(?:\b)" 3; - "~*(?:\b)PageGrabber(?:\b)" 3; - "~*(?:\b)page\ scorer(?:\b)" 3; - "~*(?:\b)PageScorer(?:\b)" 3; - "~*(?:\b)Pandalytics(?:\b)" 3; - "~*(?:\b)Panscient(?:\b)" 3; - "~*(?:\b)Papa\ Foto(?:\b)" 3; - "~*(?:\b)Pavuk(?:\b)" 3; - "~*(?:\b)pcBrowser(?:\b)" 3; - "~*(?:\b)PECL::HTTP(?:\b)" 3; - "~*(?:\b)PeoplePal(?:\b)" 3; - "~*(?:\b)PHPCrawl(?:\b)" 3; - "~*(?:\b)Picscout(?:\b)" 3; - "~*(?:\b)Picsearch(?:\b)" 3; - "~*(?:\b)PictureFinder(?:\b)" 3; - "~*(?:\b)Pimonster(?:\b)" 3; - "~*(?:\b)Pi-Monster(?:\b)" 3; - "~*(?:\b)Pixray(?:\b)" 3; - "~*(?:\b)PleaseCrawl(?:\b)" 3; - "~*(?:\b)plumanalytics(?:\b)" 3; - "~*(?:\b)Pockey(?:\b)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; - "~*(?:\b)Probethenet(?:\b)" 3; - "~*(?:\b)ProPowerBot(?:\b)" 3; - "~*(?:\b)ProWebWalker(?:\b)" 3; - "~*(?:\b)Psbot(?:\b)" 3; - "~*(?:\b)Pump(?:\b)" 3; - "~*(?:\b)PxBroker(?:\b)" 3; - "~*(?:\b)PyCurl(?:\b)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; - "~*(?:\b)Quick-Crawler(?:\b)" 3; - "~*(?:\b)RankActive(?:\b)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b)" 3; - "~*(?:\b)RankFlex(?:\b)" 3; - "~*(?:\b)RankingBot(?:\b)" 3; - "~*(?:\b)RankingBot2(?:\b)" 3; - "~*(?:\b)Rankivabot(?:\b)" 3; - "~*(?:\b)RankurBot(?:\b)" 3; - "~*(?:\b)RealDownload(?:\b)" 3; - "~*(?:\b)Reaper(?:\b)" 3; - "~*(?:\b)RebelMouse(?:\b)" 3; - "~*(?:\b)Recorder(?:\b)" 3; - "~*(?:\b)RedesScrapy(?:\b)" 3; - "~*(?:\b)ReGet(?:\b)" 3; - "~*(?:\b)RepoMonkey(?:\b)" 3; - "~*(?:\b)Ripper(?:\b)" 3; - "~*(?:\b)RocketCrawler(?:\b)" 3; - "~*(?:\b)Rogerbot(?:\b)" 3; - "~*(?:\b)s1z.ru(?:\b)" 3; - "~*(?:\b)SalesIntelligent(?:\b)" 3; - "~*(?:\b)SBIder(?:\b)" 3; - "~*(?:\b)ScanAlert(?:\b)" 3; - "~*(?:\b)Scanbot(?:\b)" 3; - "~*(?:\b)scan.lol(?:\b)" 3; - "~*(?:\b)ScoutJet(?:\b)" 3; - "~*(?:\b)Scrapy(?:\b)" 3; - "~*(?:\b)Screaming(?:\b)" 3; - "~*(?:\b)ScreenerBot(?:\b)" 3; - "~*(?:\b)Searchestate(?:\b)" 3; - "~*(?:\b)SearchmetricsBot(?:\b)" 3; - "~*(?:\b)Semrush(?:\b)" 3; - "~*(?:\b)SemrushBot(?:\b)" 3; - "~*(?:\b)SEOkicks(?:\b)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; - "~*(?:\b)Seomoz(?:\b)" 3; - "~*(?:\b)SEOprofiler(?:\b)" 3; - "~*(?:\b)seoscanners(?:\b)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b)" 3; - "~*(?:\b)SEOstats(?:\b)" 3; - "~*(?:\b)serpstatbot(?:\b)" 3; - "~*(?:\b)sexsearcher(?:\b)" 3; - "~*(?:\b)Shodan(?:\b)" 3; - "~*(?:\b)Siphon(?:\b)" 3; - "~*(?:\b)SISTRIX(?:\b)" 3; - "~*(?:\b)Sitebeam(?:\b)" 3; - "~*(?:\b)SiteExplorer(?:\b)" 3; - "~*(?:\b)Siteimprove(?:\b)" 3; - "~*(?:\b)SiteLockSpider(?:\b)" 3; - "~*(?:\b)SiteSnagger(?:\b)" 3; - "~*(?:\b)SiteSucker(?:\b)" 3; - "~*(?:\b)Site\ Sucker(?:\b)" 3; - "~*(?:\b)Sitevigil(?:\b)" 3; - "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 3; - "~*(?:\b)SlySearch(?:\b)" 3; - "~*(?:\b)SmartDownload(?:\b)" 3; - "~*(?:\b)SMTBot(?:\b)" 3; - "~*(?:\b)Snake(?:\b)" 3; - "~*(?:\b)Snapbot(?:\b)" 3; - "~*(?:\b)Snoopy(?:\b)" 3; - "~*(?:\b)SocialRankIOBot(?:\b)" 3; - "~*(?:\b)Sociscraper(?:\b)" 3; - "~*(?:\b)sogouspider(?:\b)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; - "~*(?:\b)Sosospider(?:\b)" 3; - "~*(?:\b)Sottopop(?:\b)" 3; - "~*(?:\b)SpaceBison(?:\b)" 3; - "~*(?:\b)Spammen(?:\b)" 3; - "~*(?:\b)SpankBot(?:\b)" 3; - "~*(?:\b)Spanner(?:\b)" 3; - "~*(?:\b)sp_auditbot(?:\b)" 3; - "~*(?:\b)Spbot(?:\b)" 3; - "~*(?:\b)Spinn3r(?:\b)" 3; - "~*(?:\b)SputnikBot(?:\b)" 3; - "~*(?:\b)spyfu(?:\b)" 3; - "~*(?:\b)Sqlmap(?:\b)" 3; - "~*(?:\b)Sqlworm(?:\b)" 3; - "~*(?:\b)Sqworm(?:\b)" 3; - "~*(?:\b)Steeler(?:\b)" 3; - "~*(?:\b)Stripper(?:\b)" 3; - "~*(?:\b)Sucker(?:\b)" 3; - "~*(?:\b)Sucuri(?:\b)" 3; - "~*(?:\b)SuperBot(?:\b)" 3; - "~*(?:\b)SuperHTTP(?:\b)" 3; - "~*(?:\b)Surfbot(?:\b)" 3; - "~*(?:\b)SurveyBot(?:\b)" 3; - "~*(?:\b)Suzuran(?:\b)" 3; - "~*(?:\b)Swiftbot(?:\b)" 3; - "~*(?:\b)sysscan(?:\b)" 3; - "~*(?:\b)Szukacz(?:\b)" 3; - "~*(?:\b)T0PHackTeam(?:\b)" 3; - "~*(?:\b)T8Abot(?:\b)" 3; - "~*(?:\b)tAkeOut(?:\b)" 3; - "~*(?:\b)Teleport(?:\b)" 3; - "~*(?:\b)TeleportPro(?:\b)" 3; - "~*(?:\b)Telesoft(?:\b)" 3; - "~*(?:\b)Telesphoreo(?:\b)" 3; - "~*(?:\b)Telesphorep(?:\b)" 3; - "~*(?:\b)The\ Intraformant(?:\b)" 3; - "~*(?:\b)TheNomad(?:\b)" 3; - "~*(?:\b)TightTwatBot(?:\b)" 3; - "~*(?:\b)Titan(?:\b)" 3; - "~*(?:\b)Toata(?:\b)" 3; - "~*(?:\b)Toweyabot(?:\b)" 3; - "~*(?:\b)Tracemyfile(?:\b)" 3; - "~*(?:\b)Trendiction(?:\b)" 3; - "~*(?:\b)Trendictionbot(?:\b)" 3; - "~*(?:\b)trendiction.com(?:\b)" 3; - "~*(?:\b)trendiction.de(?:\b)" 3; - "~*(?:\b)True_Robot(?:\b)" 3; - "~*(?:\b)Turingos(?:\b)" 3; - "~*(?:\b)Turnitin(?:\b)" 3; - "~*(?:\b)TurnitinBot(?:\b)" 3; - "~*(?:\b)TwengaBot(?:\b)" 3; - "~*(?:\b)Twice(?:\b)" 3; - "~*(?:\b)Typhoeus(?:\b)" 3; - "~*(?:\b)UnisterBot(?:\b)" 3; - "~*(?:\b)Upflow(?:\b)" 3; - "~*(?:\b)URLy.Warning(?:\b)" 3; - "~*(?:\b)URLy\ Warning(?:\b)" 3; - "~*(?:\b)Vacuum(?:\b)" 3; - "~*(?:\b)Vagabondo(?:\b)" 3; - "~*(?:\b)VB\ Project(?:\b)" 3; - "~*(?:\b)VCI(?:\b)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b)" 3; - "~*(?:\b)VidibleScraper(?:\b)" 3; - "~*(?:\b)Virusdie(?:\b)" 3; - "~*(?:\b)VoidEYE(?:\b)" 3; - "~*(?:\b)Voil(?:\b)" 3; - "~*(?:\b)Voltron(?:\b)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b)" 3; - "~*(?:\b)WallpapersHD(?:\b)" 3; - "~*(?:\b)WASALive-Bot(?:\b)" 3; - "~*(?:\b)WBSearchBot(?:\b)" 3; - "~*(?:\b)Webalta(?:\b)" 3; - "~*(?:\b)WebAuto(?:\b)" 3; - "~*(?:\b)Web\ Auto(?:\b)" 3; - "~*(?:\b)WebBandit(?:\b)" 3; - "~*(?:\b)WebCollage(?:\b)" 3; - "~*(?:\b)Web\ Collage(?:\b)" 3; - "~*(?:\b)WebCopier(?:\b)" 3; - "~*(?:\b)WEBDAV(?:\b)" 3; - "~*(?:\b)WebEnhancer(?:\b)" 3; - "~*(?:\b)Web\ Enhancer(?:\b)" 3; - "~*(?:\b)WebFetch(?:\b)" 3; - "~*(?:\b)Web\ Fetch(?:\b)" 3; - "~*(?:\b)WebFuck(?:\b)" 3; - "~*(?:\b)Web\ Fuck(?:\b)" 3; - "~*(?:\b)WebGo\ IS(?:\b)" 3; - "~*(?:\b)WebImageCollector(?:\b)" 3; - "~*(?:\b)WebLeacher(?:\b)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; - "~*(?:\b)webmeup-crawler(?:\b)" 3; - "~*(?:\b)WebPix(?:\b)" 3; - "~*(?:\b)Web\ Pix(?:\b)" 3; - "~*(?:\b)WebReaper(?:\b)" 3; - "~*(?:\b)WebSauger(?:\b)" 3; - "~*(?:\b)Web\ Sauger(?:\b)" 3; - "~*(?:\b)Webshag(?:\b)" 3; - "~*(?:\b)WebsiteExtractor(?:\b)" 3; - "~*(?:\b)WebsiteQuester(?:\b)" 3; - "~*(?:\b)Website\ Quester(?:\b)" 3; - "~*(?:\b)Webster(?:\b)" 3; - "~*(?:\b)WebStripper(?:\b)" 3; - "~*(?:\b)WebSucker(?:\b)" 3; - "~*(?:\b)Web\ Sucker(?:\b)" 3; - "~*(?:\b)WebWhacker(?:\b)" 3; - "~*(?:\b)WebZIP(?:\b)" 3; - "~*(?:\b)WeSEE(?:\b)" 3; - "~*(?:\b)Whack(?:\b)" 3; - "~*(?:\b)Whacker(?:\b)" 3; - "~*(?:\b)Whatweb(?:\b)" 3; - "~*(?:\b)Who.is\ Bot(?:\b)" 3; - "~*(?:\b)Widow(?:\b)" 3; - "~*(?:\b)WinHTTrack(?:\b)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; - "~*(?:\b)WISENutbot(?:\b)" 3; - "~*(?:\b)Wonderbot(?:\b)" 3; - "~*(?:\b)Woobot(?:\b)" 3; - "~*(?:\b)Wotbox(?:\b)" 3; - "~*(?:\b)Wprecon(?:\b)" 3; - "~*(?:\b)WPScan(?:\b)" 3; - "~*(?:\b)WWW-Collector-E(?:\b)" 3; - "~*(?:\b)WWW-Mechanize(?:\b)" 3; - "~*(?:\b)WWW::Mechanize(?:\b)" 3; - "~*(?:\b)WWWOFFLE(?:\b)" 3; - "~*(?:\b)x09Mozilla(?:\b)" 3; - "~*(?:\b)x22Mozilla(?:\b)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; - "~*(?:\b)Xenu(?:\b)" 3; - "~*(?:\b)xpymep1.exe(?:\b)" 3; - "~*(?:\b)YoudaoBot(?:\b)" 3; - "~*(?:\b)Zade(?:\b)" 3; - "~*(?:\b)Zauba(?:\b)" 3; - "~*(?:\b)zauba.io(?:\b)" 3; - "~*(?:\b)Zermelo(?:\b)" 3; - "~*(?:\b)Zeus(?:\b)" 3; - "~*(?:\b)zgrab(?:\b)" 3; - "~*(?:\b)Zitebot(?:\b)" 3; - "~*(?:\b)ZmEu(?:\b)" 3; - "~*(?:\b)ZumBot(?:\b)" 3; - "~*(?:\b)ZyBorg(?:\b)" 3; + "~*(?:\b)360Spider(?:\b|)" 3; + "~*(?:\b)404checker(?:\b|)" 3; + "~*(?:\b)404enemy(?:\b|)" 3; + "~*(?:\b)80legs(?:\b|)" 3; + "~*(?:\b)Abonti(?:\b|)" 3; + "~*(?:\b)Aboundex(?:\b|)" 3; + "~*(?:\b)Aboundexbot(?:\b|)" 3; + "~*(?:\b)Acunetix(?:\b|)" 3; + "~*(?:\b)ADmantX(?:\b|)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; + "~*(?:\b)AhrefsBot(?:\b|)" 3; + "~*(?:\b)AIBOT(?:\b|)" 3; + "~*(?:\b)AiHitBot(?:\b|)" 3; + "~*(?:\b)Aipbot(?:\b|)" 3; + "~*(?:\b)Alexibot(?:\b|)" 3; + "~*(?:\b)Alligator(?:\b|)" 3; + "~*(?:\b)AllSubmitter(?:\b|)" 3; + "~*(?:\b)AlphaBot(?:\b|)" 3; + "~*(?:\b)Anarchie(?:\b|)" 3; + "~*(?:\b)Apexoo(?:\b|)" 3; + "~*(?:\b)archive.org_bot(?:\b|)" 3; + "~*(?:\b)ASPSeek(?:\b|)" 3; + "~*(?:\b)Asterias(?:\b|)" 3; + "~*(?:\b)Attach(?:\b|)" 3; + "~*(?:\b)autoemailspider(?:\b|)" 3; + "~*(?:\b)BackDoorBot(?:\b|)" 3; + "~*(?:\b)Backlink-Ceck(?:\b|)" 3; + "~*(?:\b)backlink-check(?:\b|)" 3; + "~*(?:\b)BacklinkCrawler(?:\b|)" 3; + "~*(?:\b)BackStreet(?:\b|)" 3; + "~*(?:\b)BackWeb(?:\b|)" 3; + "~*(?:\b)Badass(?:\b|)" 3; + "~*(?:\b)Bandit(?:\b|)" 3; + "~*(?:\b)Barkrowler(?:\b|)" 3; + "~*(?:\b)BatchFTP(?:\b|)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; + "~*(?:\b)BBBike(?:\b|)" 3; + "~*(?:\b)BDCbot(?:\b|)" 3; + "~*(?:\b)BDFetch(?:\b|)" 3; + "~*(?:\b)BetaBot(?:\b|)" 3; + "~*(?:\b)Bigfoot(?:\b|)" 3; + "~*(?:\b)Bitacle(?:\b|)" 3; + "~*(?:\b)Blackboard(?:\b|)" 3; + "~*(?:\b)Black\ Hole(?:\b|)" 3; + "~*(?:\b)BlackWidow(?:\b|)" 3; + "~*(?:\b)BLEXBot(?:\b|)" 3; + "~*(?:\b)Blow(?:\b|)" 3; + "~*(?:\b)BlowFish(?:\b|)" 3; + "~*(?:\b)Boardreader(?:\b|)" 3; + "~*(?:\b)Bolt(?:\b|)" 3; + "~*(?:\b)BotALot(?:\b|)" 3; + "~*(?:\b)Brandprotect(?:\b|)" 3; + "~*(?:\b)Brandwatch(?:\b|)" 3; + "~*(?:\b)Buddy(?:\b|)" 3; + "~*(?:\b)BuiltBotTough(?:\b|)" 3; + "~*(?:\b)BuiltWith(?:\b|)" 3; + "~*(?:\b)Bullseye(?:\b|)" 3; + "~*(?:\b)BunnySlippers(?:\b|)" 3; + "~*(?:\b)BuzzSumo(?:\b|)" 3; + "~*(?:\b)Calculon(?:\b|)" 3; + "~*(?:\b)CATExplorador(?:\b|)" 3; + "~*(?:\b)CazoodleBot(?:\b|)" 3; + "~*(?:\b)CCBot(?:\b|)" 3; + "~*(?:\b)Cegbfeieh(?:\b|)" 3; + "~*(?:\b)CheeseBot(?:\b|)" 3; + "~*(?:\b)CherryPicker(?:\b|)" 3; + "~*(?:\b)CheTeam(?:\b|)" 3; + "~*(?:\b)ChinaClaw(?:\b|)" 3; + "~*(?:\b)Chlooe(?:\b|)" 3; + "~*(?:\b)Claritybot(?:\b|)" 3; + "~*(?:\b)Cliqzbot(?:\b|)" 3; + "~*(?:\b)Cloud\ mapping(?:\b|)" 3; + "~*(?:\b)coccocbot-web(?:\b|)" 3; + "~*(?:\b)Cogentbot(?:\b|)" 3; + "~*(?:\b)cognitiveseo(?:\b|)" 3; + "~*(?:\b)Collector(?:\b|)" 3; + "~*(?:\b)com.plumanalytics(?:\b|)" 3; + "~*(?:\b)Copier(?:\b|)" 3; + "~*(?:\b)CopyRightCheck(?:\b|)" 3; + "~*(?:\b)Copyscape(?:\b|)" 3; + "~*(?:\b)Cosmos(?:\b|)" 3; + "~*(?:\b)Craftbot(?:\b|)" 3; + "~*(?:\b)crawler4j(?:\b|)" 3; + "~*(?:\b)crawler.feedback(?:\b|)" 3; + "~*(?:\b)crawl.sogou.com(?:\b|)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; + "~*(?:\b)Crescent(?:\b|)" 3; + "~*(?:\b)CrunchBot(?:\b|)" 3; + "~*(?:\b)CSHttp(?:\b|)" 3; + "~*(?:\b)Curious(?:\b|)" 3; + "~*(?:\b)Custo(?:\b|)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; + "~*(?:\b)DataCha0s(?:\b|)" 3; + "~*(?:\b)DBLBot(?:\b|)" 3; + "~*(?:\b)demandbase-bot(?:\b|)" 3; + "~*(?:\b)Demon(?:\b|)" 3; + "~*(?:\b)Deusu(?:\b|)" 3; + "~*(?:\b)Devil(?:\b|)" 3; + "~*(?:\b)Digincore(?:\b|)" 3; + "~*(?:\b)DigitalPebble(?:\b|)" 3; + "~*(?:\b)DIIbot(?:\b|)" 3; + "~*(?:\b)Dirbuster(?:\b|)" 3; + "~*(?:\b)Disco(?:\b|)" 3; + "~*(?:\b)Discobot(?:\b|)" 3; + "~*(?:\b)Discoverybot(?:\b|)" 3; + "~*(?:\b)Dispatch(?:\b|)" 3; + "~*(?:\b)DittoSpyder(?:\b|)" 3; + "~*(?:\b)DnyzBot(?:\b|)" 3; + "~*(?:\b)DomainAppender(?:\b|)" 3; + "~*(?:\b)DomainCrawler(?:\b|)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; + "~*(?:\b)DomainStatsBot(?:\b|)" 3; + "~*(?:\b)Dotbot(?:\b|)" 3; + "~*(?:\b)Download\ Wonder(?:\b|)" 3; + "~*(?:\b)Dragonfly(?:\b|)" 3; + "~*(?:\b)Drip(?:\b|)" 3; + "~*(?:\b)DSearch(?:\b|)" 3; + "~*(?:\b)DTS\ Agent(?:\b|)" 3; + "~*(?:\b)EasyDL(?:\b|)" 3; + "~*(?:\b)Ebingbong(?:\b|)" 3; + "~*(?:\b)eCatch(?:\b|)" 3; + "~*(?:\b)ECCP/1.0(?:\b|)" 3; + "~*(?:\b)Ecxi(?:\b|)" 3; + "~*(?:\b)EirGrabber(?:\b|)" 3; + "~*(?:\b)EMail\ Siphon(?:\b|)" 3; + "~*(?:\b)EMail\ Wolf(?:\b|)" 3; + "~*(?:\b)EroCrawler(?:\b|)" 3; + "~*(?:\b)evc-batch(?:\b|)" 3; + "~*(?:\b)Evil(?:\b|)" 3; + "~*(?:\b)Exabot(?:\b|)" 3; + "~*(?:\b)Express\ WebPictures(?:\b|)" 3; + "~*(?:\b)ExtLinksBot(?:\b|)" 3; + "~*(?:\b)Extractor(?:\b|)" 3; + "~*(?:\b)ExtractorPro(?:\b|)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; + "~*(?:\b)EyeNetIE(?:\b|)" 3; + "~*(?:\b)Ezooms(?:\b|)" 3; + "~*(?:\b)facebookscraper(?:\b|)" 3; + "~*(?:\b)FDM(?:\b|)" 3; + "~*(?:\b)FemtosearchBot(?:\b|)" 3; + "~*(?:\b)FHscan(?:\b|)" 3; + "~*(?:\b)Fimap(?:\b|)" 3; + "~*(?:\b)Firefox/7.0(?:\b|)" 3; + "~*(?:\b)FlashGet(?:\b|)" 3; + "~*(?:\b)Flunky(?:\b|)" 3; + "~*(?:\b)Foobot(?:\b|)" 3; + "~*(?:\b)Freeuploader(?:\b|)" 3; + "~*(?:\b)FrontPage(?:\b|)" 3; + "~*(?:\b)FyberSpider(?:\b|)" 3; + "~*(?:\b)Fyrebot(?:\b|)" 3; + "~*(?:\b)GalaxyBot(?:\b|)" 3; + "~*(?:\b)Genieo(?:\b|)" 3; + "~*(?:\b)GermCrawler(?:\b|)" 3; + "~*(?:\b)Getintent(?:\b|)" 3; + "~*(?:\b)GetRight(?:\b|)" 3; + "~*(?:\b)GetWeb(?:\b|)" 3; + "~*(?:\b)Gigablast(?:\b|)" 3; + "~*(?:\b)Gigabot(?:\b|)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; + "~*(?:\b)Gotit(?:\b|)" 3; + "~*(?:\b)GoZilla(?:\b|)" 3; + "~*(?:\b)Go!Zilla(?:\b|)" 3; + "~*(?:\b)Grabber(?:\b|)" 3; + "~*(?:\b)GrabNet(?:\b|)" 3; + "~*(?:\b)Grafula(?:\b|)" 3; + "~*(?:\b)GrapeFX(?:\b|)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; + "~*(?:\b)GridBot(?:\b|)" 3; + "~*(?:\b)GT::WWW(?:\b|)" 3; + "~*(?:\b)Haansoft(?:\b|)" 3; + "~*(?:\b)HaosouSpider(?:\b|)" 3; + "~*(?:\b)Harvest(?:\b|)" 3; + "~*(?:\b)Havij(?:\b|)" 3; + "~*(?:\b)HEADMasterSEO(?:\b|)" 3; + "~*(?:\b)Heritrix(?:\b|)" 3; + "~*(?:\b)Hloader(?:\b|)" 3; + "~*(?:\b)HMView(?:\b|)" 3; + "~*(?:\b)HTMLparser(?:\b|)" 3; + "~*(?:\b)HTTP::Lite(?:\b|)" 3; + "~*(?:\b)HTTrack(?:\b|)" 3; + "~*(?:\b)Humanlinks(?:\b|)" 3; + "~*(?:\b)HybridBot(?:\b|)" 3; + "~*(?:\b)Iblog(?:\b|)" 3; + "~*(?:\b)IDBot(?:\b|)" 3; + "~*(?:\b)Id-search(?:\b|)" 3; + "~*(?:\b)IlseBot(?:\b|)" 3; + "~*(?:\b)Image\ Fetch(?:\b|)" 3; + "~*(?:\b)Image\ Sucker(?:\b|)" 3; + "~*(?:\b)IndeedBot(?:\b|)" 3; + "~*(?:\b)Indy\ Library(?:\b|)" 3; + "~*(?:\b)InfoNaviRobot(?:\b|)" 3; + "~*(?:\b)InfoTekies(?:\b|)" 3; + "~*(?:\b)instabid(?:\b|)" 3; + "~*(?:\b)Intelliseek(?:\b|)" 3; + "~*(?:\b)InterGET(?:\b|)" 3; + "~*(?:\b)Internet\ Ninja(?:\b|)" 3; + "~*(?:\b)InternetSeer(?:\b|)" 3; + "~*(?:\b)internetVista\ monitor(?:\b|)" 3; + "~*(?:\b)ips-agent(?:\b|)" 3; + "~*(?:\b)Iria(?:\b|)" 3; + "~*(?:\b)IRLbot(?:\b|)" 3; + "~*(?:\b)Iskanie(?:\b|)" 3; + "~*(?:\b)IstellaBot(?:\b|)" 3; + "~*(?:\b)JamesBOT(?:\b|)" 3; + "~*(?:\b)Jbrofuzz(?:\b|)" 3; + "~*(?:\b)JennyBot(?:\b|)" 3; + "~*(?:\b)JetCar(?:\b|)" 3; + "~*(?:\b)Jetty(?:\b|)" 3; + "~*(?:\b)JikeSpider(?:\b|)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; + "~*(?:\b)Joomla(?:\b|)" 3; + "~*(?:\b)Jorgee(?:\b|)" 3; + "~*(?:\b)JustView(?:\b|)" 3; + "~*(?:\b)Jyxobot(?:\b|)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; + "~*(?:\b)Keyword\ Density(?:\b|)" 3; + "~*(?:\b)Kozmosbot(?:\b|)" 3; + "~*(?:\b)Lanshanbot(?:\b|)" 3; + "~*(?:\b)Larbin(?:\b|)" 3; + "~*(?:\b)LeechFTP(?:\b|)" 3; + "~*(?:\b)LeechGet(?:\b|)" 3; + "~*(?:\b)LexiBot(?:\b|)" 3; + "~*(?:\b)Lftp(?:\b|)" 3; + "~*(?:\b)LibWeb(?:\b|)" 3; + "~*(?:\b)Libwhisker(?:\b|)" 3; + "~*(?:\b)Lightspeedsystems(?:\b|)" 3; + "~*(?:\b)Likse(?:\b|)" 3; + "~*(?:\b)Linkdexbot(?:\b|)" 3; + "~*(?:\b)LinkextractorPro(?:\b|)" 3; + "~*(?:\b)LinkpadBot(?:\b|)" 3; + "~*(?:\b)LinkScan(?:\b|)" 3; + "~*(?:\b)LinksManager(?:\b|)" 3; + "~*(?:\b)LinkWalker(?:\b|)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; + "~*(?:\b)Lipperhey(?:\b|)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; + "~*(?:\b)Litemage_walker(?:\b|)" 3; + "~*(?:\b)Lmspider(?:\b|)" 3; + "~*(?:\b)LNSpiderguy(?:\b|)" 3; + "~*(?:\b)Ltx71(?:\b|)" 3; + "~*(?:\b)lwp-request(?:\b|)" 3; + "~*(?:\b)LWP::Simple(?:\b|)" 3; + "~*(?:\b)lwp-trivial(?:\b|)" 3; + "~*(?:\b)Magnet(?:\b|)" 3; + "~*(?:\b)Mag-Net(?:\b|)" 3; + "~*(?:\b)magpie-crawler(?:\b|)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; + "~*(?:\b)Majestic12(?:\b|)" 3; + "~*(?:\b)Majestic-SEO(?:\b|)" 3; + "~*(?:\b)Majestic\ SEO(?:\b|)" 3; + "~*(?:\b)MarkMonitor(?:\b|)" 3; + "~*(?:\b)MarkWatch(?:\b|)" 3; + "~*(?:\b)Masscan(?:\b|)" 3; + "~*(?:\b)Mass\ Downloader(?:\b|)" 3; + "~*(?:\b)Mata\ Hari(?:\b|)" 3; + "~*(?:\b)MauiBot(?:\b|)" 3; + "~*(?:\b)meanpathbot(?:\b|)" 3; + "~*(?:\b)Meanpathbot(?:\b|)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; + "~*(?:\b)mediawords(?:\b|)" 3; + "~*(?:\b)MegaIndex.ru(?:\b|)" 3; + "~*(?:\b)Metauri(?:\b|)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; + "~*(?:\b)MIDown\ tool(?:\b|)" 3; + "~*(?:\b)MIIxpc(?:\b|)" 3; + "~*(?:\b)Mister\ PiX(?:\b|)" 3; + "~*(?:\b)MJ12bot(?:\b|)" 3; + "~*(?:\b)Mojeek(?:\b|)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; + "~*(?:\b)Mr.4x3(?:\b|)" 3; + "~*(?:\b)MSFrontPage(?:\b|)" 3; + "~*(?:\b)MSIECrawler(?:\b|)" 3; + "~*(?:\b)Msrabot(?:\b|)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; + "~*(?:\b)muhstik-scan(?:\b|)" 3; + "~*(?:\b)Musobot(?:\b|)" 3; + "~*(?:\b)Name\ Intelligence(?:\b|)" 3; + "~*(?:\b)Nameprotect(?:\b|)" 3; + "~*(?:\b)Navroad(?:\b|)" 3; + "~*(?:\b)NearSite(?:\b|)" 3; + "~*(?:\b)Needle(?:\b|)" 3; + "~*(?:\b)Nessus(?:\b|)" 3; + "~*(?:\b)NetAnts(?:\b|)" 3; + "~*(?:\b)Netcraft(?:\b|)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; + "~*(?:\b)NetLyzer(?:\b|)" 3; + "~*(?:\b)NetMechanic(?:\b|)" 3; + "~*(?:\b)NetSpider(?:\b|)" 3; + "~*(?:\b)Nettrack(?:\b|)" 3; + "~*(?:\b)Net\ Vampire(?:\b|)" 3; + "~*(?:\b)Netvibes(?:\b|)" 3; + "~*(?:\b)NetZIP(?:\b|)" 3; + "~*(?:\b)NextGenSearchBot(?:\b|)" 3; + "~*(?:\b)Nibbler(?:\b|)" 3; + "~*(?:\b)NICErsPRO(?:\b|)" 3; + "~*(?:\b)Niki-bot(?:\b|)" 3; + "~*(?:\b)Nikto(?:\b|)" 3; + "~*(?:\b)NimbleCrawler(?:\b|)" 3; + "~*(?:\b)Nimbostratus(?:\b|)" 3; + "~*(?:\b)Ninja(?:\b|)" 3; + "~*(?:\b)Nmap(?:\b|)" 3; + "~*(?:\b)NPbot(?:\b|)" 3; + "~*(?:\b)Nutch(?:\b|)" 3; + "~*(?:\b)oBot(?:\b|)" 3; + "~*(?:\b)Octopus(?:\b|)" 3; + "~*(?:\b)Offline\ Explorer(?:\b|)" 3; + "~*(?:\b)Offline\ Navigator(?:\b|)" 3; + "~*(?:\b)OnCrawl(?:\b|)" 3; + "~*(?:\b)Openfind(?:\b|)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; + "~*(?:\b)Openvas(?:\b|)" 3; + "~*(?:\b)OpenVAS(?:\b|)" 3; + "~*(?:\b)OrangeBot(?:\b|)" 3; + "~*(?:\b)OrangeSpider(?:\b|)" 3; + "~*(?:\b)OutclicksBot(?:\b|)" 3; + "~*(?:\b)OutfoxBot(?:\b|)" 3; + "~*(?:\b)PageAnalyzer(?:\b|)" 3; + "~*(?:\b)Page\ Analyzer(?:\b|)" 3; + "~*(?:\b)PageGrabber(?:\b|)" 3; + "~*(?:\b)page\ scorer(?:\b|)" 3; + "~*(?:\b)PageScorer(?:\b|)" 3; + "~*(?:\b)Pandalytics(?:\b|)" 3; + "~*(?:\b)Panscient(?:\b|)" 3; + "~*(?:\b)Papa\ Foto(?:\b|)" 3; + "~*(?:\b)Pavuk(?:\b|)" 3; + "~*(?:\b)pcBrowser(?:\b|)" 3; + "~*(?:\b)PECL::HTTP(?:\b|)" 3; + "~*(?:\b)PeoplePal(?:\b|)" 3; + "~*(?:\b)PHPCrawl(?:\b|)" 3; + "~*(?:\b)Picscout(?:\b|)" 3; + "~*(?:\b)Picsearch(?:\b|)" 3; + "~*(?:\b)PictureFinder(?:\b|)" 3; + "~*(?:\b)Pimonster(?:\b|)" 3; + "~*(?:\b)Pi-Monster(?:\b|)" 3; + "~*(?:\b)Pixray(?:\b|)" 3; + "~*(?:\b)PleaseCrawl(?:\b|)" 3; + "~*(?:\b)plumanalytics(?:\b|)" 3; + "~*(?:\b)Pockey(?:\b|)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; + "~*(?:\b)Probethenet(?:\b|)" 3; + "~*(?:\b)ProPowerBot(?:\b|)" 3; + "~*(?:\b)ProWebWalker(?:\b|)" 3; + "~*(?:\b)Psbot(?:\b|)" 3; + "~*(?:\b)Pump(?:\b|)" 3; + "~*(?:\b)PxBroker(?:\b|)" 3; + "~*(?:\b)PyCurl(?:\b|)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; + "~*(?:\b)Quick-Crawler(?:\b|)" 3; + "~*(?:\b)RankActive(?:\b|)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; + "~*(?:\b)RankFlex(?:\b|)" 3; + "~*(?:\b)RankingBot(?:\b|)" 3; + "~*(?:\b)RankingBot2(?:\b|)" 3; + "~*(?:\b)Rankivabot(?:\b|)" 3; + "~*(?:\b)RankurBot(?:\b|)" 3; + "~*(?:\b)RealDownload(?:\b|)" 3; + "~*(?:\b)Reaper(?:\b|)" 3; + "~*(?:\b)RebelMouse(?:\b|)" 3; + "~*(?:\b)Recorder(?:\b|)" 3; + "~*(?:\b)RedesScrapy(?:\b|)" 3; + "~*(?:\b)ReGet(?:\b|)" 3; + "~*(?:\b)RepoMonkey(?:\b|)" 3; + "~*(?:\b)Ripper(?:\b|)" 3; + "~*(?:\b)RocketCrawler(?:\b|)" 3; + "~*(?:\b)Rogerbot(?:\b|)" 3; + "~*(?:\b)s1z.ru(?:\b|)" 3; + "~*(?:\b)SalesIntelligent(?:\b|)" 3; + "~*(?:\b)SBIder(?:\b|)" 3; + "~*(?:\b)ScanAlert(?:\b|)" 3; + "~*(?:\b)Scanbot(?:\b|)" 3; + "~*(?:\b)scan.lol(?:\b|)" 3; + "~*(?:\b)ScoutJet(?:\b|)" 3; + "~*(?:\b)Scrapy(?:\b|)" 3; + "~*(?:\b)Screaming(?:\b|)" 3; + "~*(?:\b)ScreenerBot(?:\b|)" 3; + "~*(?:\b)Searchestate(?:\b|)" 3; + "~*(?:\b)SearchmetricsBot(?:\b|)" 3; + "~*(?:\b)Semrush(?:\b|)" 3; + "~*(?:\b)SemrushBot(?:\b|)" 3; + "~*(?:\b)SEOkicks(?:\b|)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; + "~*(?:\b)Seomoz(?:\b|)" 3; + "~*(?:\b)SEOprofiler(?:\b|)" 3; + "~*(?:\b)seoscanners(?:\b|)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; + "~*(?:\b)SEOstats(?:\b|)" 3; + "~*(?:\b)serpstatbot(?:\b|)" 3; + "~*(?:\b)sexsearcher(?:\b|)" 3; + "~*(?:\b)Shodan(?:\b|)" 3; + "~*(?:\b)Siphon(?:\b|)" 3; + "~*(?:\b)SISTRIX(?:\b|)" 3; + "~*(?:\b)Sitebeam(?:\b|)" 3; + "~*(?:\b)SiteExplorer(?:\b|)" 3; + "~*(?:\b)Siteimprove(?:\b|)" 3; + "~*(?:\b)SiteLockSpider(?:\b|)" 3; + "~*(?:\b)SiteSnagger(?:\b|)" 3; + "~*(?:\b)SiteSucker(?:\b|)" 3; + "~*(?:\b)Site\ Sucker(?:\b|)" 3; + "~*(?:\b)Sitevigil(?:\b|)" 3; + "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 3; + "~*(?:\b)SlySearch(?:\b|)" 3; + "~*(?:\b)SmartDownload(?:\b|)" 3; + "~*(?:\b)SMTBot(?:\b|)" 3; + "~*(?:\b)Snake(?:\b|)" 3; + "~*(?:\b)Snapbot(?:\b|)" 3; + "~*(?:\b)Snoopy(?:\b|)" 3; + "~*(?:\b)SocialRankIOBot(?:\b|)" 3; + "~*(?:\b)Sociscraper(?:\b|)" 3; + "~*(?:\b)sogouspider(?:\b|)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; + "~*(?:\b)Sosospider(?:\b|)" 3; + "~*(?:\b)Sottopop(?:\b|)" 3; + "~*(?:\b)SpaceBison(?:\b|)" 3; + "~*(?:\b)Spammen(?:\b|)" 3; + "~*(?:\b)SpankBot(?:\b|)" 3; + "~*(?:\b)Spanner(?:\b|)" 3; + "~*(?:\b)sp_auditbot(?:\b|)" 3; + "~*(?:\b)Spbot(?:\b|)" 3; + "~*(?:\b)Spinn3r(?:\b|)" 3; + "~*(?:\b)SputnikBot(?:\b|)" 3; + "~*(?:\b)spyfu(?:\b|)" 3; + "~*(?:\b)Sqlmap(?:\b|)" 3; + "~*(?:\b)Sqlworm(?:\b|)" 3; + "~*(?:\b)Sqworm(?:\b|)" 3; + "~*(?:\b)Steeler(?:\b|)" 3; + "~*(?:\b)Stripper(?:\b|)" 3; + "~*(?:\b)Sucker(?:\b|)" 3; + "~*(?:\b)Sucuri(?:\b|)" 3; + "~*(?:\b)SuperBot(?:\b|)" 3; + "~*(?:\b)SuperHTTP(?:\b|)" 3; + "~*(?:\b)Surfbot(?:\b|)" 3; + "~*(?:\b)SurveyBot(?:\b|)" 3; + "~*(?:\b)Suzuran(?:\b|)" 3; + "~*(?:\b)Swiftbot(?:\b|)" 3; + "~*(?:\b)sysscan(?:\b|)" 3; + "~*(?:\b)Szukacz(?:\b|)" 3; + "~*(?:\b)T0PHackTeam(?:\b|)" 3; + "~*(?:\b)T8Abot(?:\b|)" 3; + "~*(?:\b)tAkeOut(?:\b|)" 3; + "~*(?:\b)Teleport(?:\b|)" 3; + "~*(?:\b)TeleportPro(?:\b|)" 3; + "~*(?:\b)Telesoft(?:\b|)" 3; + "~*(?:\b)Telesphoreo(?:\b|)" 3; + "~*(?:\b)Telesphorep(?:\b|)" 3; + "~*(?:\b)The\ Intraformant(?:\b|)" 3; + "~*(?:\b)TheNomad(?:\b|)" 3; + "~*(?:\b)TightTwatBot(?:\b|)" 3; + "~*(?:\b)Titan(?:\b|)" 3; + "~*(?:\b)Toata(?:\b|)" 3; + "~*(?:\b)Toweyabot(?:\b|)" 3; + "~*(?:\b)Tracemyfile(?:\b|)" 3; + "~*(?:\b)Trendiction(?:\b|)" 3; + "~*(?:\b)Trendictionbot(?:\b|)" 3; + "~*(?:\b)trendiction.com(?:\b|)" 3; + "~*(?:\b)trendiction.de(?:\b|)" 3; + "~*(?:\b)True_Robot(?:\b|)" 3; + "~*(?:\b)Turingos(?:\b|)" 3; + "~*(?:\b)Turnitin(?:\b|)" 3; + "~*(?:\b)TurnitinBot(?:\b|)" 3; + "~*(?:\b)TwengaBot(?:\b|)" 3; + "~*(?:\b)Twice(?:\b|)" 3; + "~*(?:\b)Typhoeus(?:\b|)" 3; + "~*(?:\b)UnisterBot(?:\b|)" 3; + "~*(?:\b)Upflow(?:\b|)" 3; + "~*(?:\b)URLy.Warning(?:\b|)" 3; + "~*(?:\b)URLy\ Warning(?:\b|)" 3; + "~*(?:\b)Vacuum(?:\b|)" 3; + "~*(?:\b)Vagabondo(?:\b|)" 3; + "~*(?:\b)VB\ Project(?:\b|)" 3; + "~*(?:\b)VCI(?:\b|)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; + "~*(?:\b)VidibleScraper(?:\b|)" 3; + "~*(?:\b)Virusdie(?:\b|)" 3; + "~*(?:\b)VoidEYE(?:\b|)" 3; + "~*(?:\b)Voil(?:\b|)" 3; + "~*(?:\b)Voltron(?:\b|)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; + "~*(?:\b)WallpapersHD(?:\b|)" 3; + "~*(?:\b)WASALive-Bot(?:\b|)" 3; + "~*(?:\b)WBSearchBot(?:\b|)" 3; + "~*(?:\b)Webalta(?:\b|)" 3; + "~*(?:\b)WebAuto(?:\b|)" 3; + "~*(?:\b)Web\ Auto(?:\b|)" 3; + "~*(?:\b)WebBandit(?:\b|)" 3; + "~*(?:\b)WebCollage(?:\b|)" 3; + "~*(?:\b)Web\ Collage(?:\b|)" 3; + "~*(?:\b)WebCopier(?:\b|)" 3; + "~*(?:\b)WEBDAV(?:\b|)" 3; + "~*(?:\b)WebEnhancer(?:\b|)" 3; + "~*(?:\b)Web\ Enhancer(?:\b|)" 3; + "~*(?:\b)WebFetch(?:\b|)" 3; + "~*(?:\b)Web\ Fetch(?:\b|)" 3; + "~*(?:\b)WebFuck(?:\b|)" 3; + "~*(?:\b)Web\ Fuck(?:\b|)" 3; + "~*(?:\b)WebGo\ IS(?:\b|)" 3; + "~*(?:\b)WebImageCollector(?:\b|)" 3; + "~*(?:\b)WebLeacher(?:\b|)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; + "~*(?:\b)webmeup-crawler(?:\b|)" 3; + "~*(?:\b)WebPix(?:\b|)" 3; + "~*(?:\b)Web\ Pix(?:\b|)" 3; + "~*(?:\b)WebReaper(?:\b|)" 3; + "~*(?:\b)WebSauger(?:\b|)" 3; + "~*(?:\b)Web\ Sauger(?:\b|)" 3; + "~*(?:\b)Webshag(?:\b|)" 3; + "~*(?:\b)WebsiteExtractor(?:\b|)" 3; + "~*(?:\b)WebsiteQuester(?:\b|)" 3; + "~*(?:\b)Website\ Quester(?:\b|)" 3; + "~*(?:\b)Webster(?:\b|)" 3; + "~*(?:\b)WebStripper(?:\b|)" 3; + "~*(?:\b)WebSucker(?:\b|)" 3; + "~*(?:\b)Web\ Sucker(?:\b|)" 3; + "~*(?:\b)WebWhacker(?:\b|)" 3; + "~*(?:\b)WebZIP(?:\b|)" 3; + "~*(?:\b)WeSEE(?:\b|)" 3; + "~*(?:\b)Whack(?:\b|)" 3; + "~*(?:\b)Whacker(?:\b|)" 3; + "~*(?:\b)Whatweb(?:\b|)" 3; + "~*(?:\b)Who.is\ Bot(?:\b|)" 3; + "~*(?:\b)Widow(?:\b|)" 3; + "~*(?:\b)WinHTTrack(?:\b|)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; + "~*(?:\b)WISENutbot(?:\b|)" 3; + "~*(?:\b)Wonderbot(?:\b|)" 3; + "~*(?:\b)Woobot(?:\b|)" 3; + "~*(?:\b)Wotbox(?:\b|)" 3; + "~*(?:\b)Wprecon(?:\b|)" 3; + "~*(?:\b)WPScan(?:\b|)" 3; + "~*(?:\b)WWW-Collector-E(?:\b|)" 3; + "~*(?:\b)WWW-Mechanize(?:\b|)" 3; + "~*(?:\b)WWW::Mechanize(?:\b|)" 3; + "~*(?:\b)WWWOFFLE(?:\b|)" 3; + "~*(?:\b)x09Mozilla(?:\b|)" 3; + "~*(?:\b)x22Mozilla(?:\b|)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; + "~*(?:\b)Xenu(?:\b|)" 3; + "~*(?:\b)xpymep1.exe(?:\b|)" 3; + "~*(?:\b)YoudaoBot(?:\b|)" 3; + "~*(?:\b)Zade(?:\b|)" 3; + "~*(?:\b)Zauba(?:\b|)" 3; + "~*(?:\b)zauba.io(?:\b|)" 3; + "~*(?:\b)Zermelo(?:\b|)" 3; + "~*(?:\b)Zeus(?:\b|)" 3; + "~*(?:\b)zgrab(?:\b|)" 3; + "~*(?:\b)Zitebot(?:\b|)" 3; + "~*(?:\b)ZmEu(?:\b|)" 3; + "~*(?:\b)ZumBot(?:\b|)" 3; + "~*(?:\b)ZyBorg(?:\b|)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # *********************************************** diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index 18e5b9d75..a9e0ee294 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -10,8 +10,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1540 -### Updated: Sun Jun 23 15:20:44 SAST 2019 +### Version: V3.2019.06.1541 +### Updated: Sun Jun 23 16:11:01 SAST 2019 ### Bad Referrer Count: 6693 ### Bad Bot Count: 556 ################################################### @@ -137,562 +137,562 @@ map $http_user_agent $bad_bot { # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b)" 3; - "~*(?:\b)404checker(?:\b)" 3; - "~*(?:\b)404enemy(?:\b)" 3; - "~*(?:\b)80legs(?:\b)" 3; - "~*(?:\b)Abonti(?:\b)" 3; - "~*(?:\b)Aboundex(?:\b)" 3; - "~*(?:\b)Aboundexbot(?:\b)" 3; - "~*(?:\b)Acunetix(?:\b)" 3; - "~*(?:\b)ADmantX(?:\b)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; - "~*(?:\b)AhrefsBot(?:\b)" 3; - "~*(?:\b)AIBOT(?:\b)" 3; - "~*(?:\b)AiHitBot(?:\b)" 3; - "~*(?:\b)Aipbot(?:\b)" 3; - "~*(?:\b)Alexibot(?:\b)" 3; - "~*(?:\b)Alligator(?:\b)" 3; - "~*(?:\b)AllSubmitter(?:\b)" 3; - "~*(?:\b)AlphaBot(?:\b)" 3; - "~*(?:\b)Anarchie(?:\b)" 3; - "~*(?:\b)Apexoo(?:\b)" 3; - "~*(?:\b)archive.org_bot(?:\b)" 3; - "~*(?:\b)ASPSeek(?:\b)" 3; - "~*(?:\b)Asterias(?:\b)" 3; - "~*(?:\b)Attach(?:\b)" 3; - "~*(?:\b)autoemailspider(?:\b)" 3; - "~*(?:\b)BackDoorBot(?:\b)" 3; - "~*(?:\b)Backlink-Ceck(?:\b)" 3; - "~*(?:\b)backlink-check(?:\b)" 3; - "~*(?:\b)BacklinkCrawler(?:\b)" 3; - "~*(?:\b)BackStreet(?:\b)" 3; - "~*(?:\b)BackWeb(?:\b)" 3; - "~*(?:\b)Badass(?:\b)" 3; - "~*(?:\b)Bandit(?:\b)" 3; - "~*(?:\b)Barkrowler(?:\b)" 3; - "~*(?:\b)BatchFTP(?:\b)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; - "~*(?:\b)BBBike(?:\b)" 3; - "~*(?:\b)BDCbot(?:\b)" 3; - "~*(?:\b)BDFetch(?:\b)" 3; - "~*(?:\b)BetaBot(?:\b)" 3; - "~*(?:\b)Bigfoot(?:\b)" 3; - "~*(?:\b)Bitacle(?:\b)" 3; - "~*(?:\b)Blackboard(?:\b)" 3; - "~*(?:\b)Black\ Hole(?:\b)" 3; - "~*(?:\b)BlackWidow(?:\b)" 3; - "~*(?:\b)BLEXBot(?:\b)" 3; - "~*(?:\b)Blow(?:\b)" 3; - "~*(?:\b)BlowFish(?:\b)" 3; - "~*(?:\b)Boardreader(?:\b)" 3; - "~*(?:\b)Bolt(?:\b)" 3; - "~*(?:\b)BotALot(?:\b)" 3; - "~*(?:\b)Brandprotect(?:\b)" 3; - "~*(?:\b)Brandwatch(?:\b)" 3; - "~*(?:\b)Buddy(?:\b)" 3; - "~*(?:\b)BuiltBotTough(?:\b)" 3; - "~*(?:\b)BuiltWith(?:\b)" 3; - "~*(?:\b)Bullseye(?:\b)" 3; - "~*(?:\b)BunnySlippers(?:\b)" 3; - "~*(?:\b)BuzzSumo(?:\b)" 3; - "~*(?:\b)Calculon(?:\b)" 3; - "~*(?:\b)CATExplorador(?:\b)" 3; - "~*(?:\b)CazoodleBot(?:\b)" 3; - "~*(?:\b)CCBot(?:\b)" 3; - "~*(?:\b)Cegbfeieh(?:\b)" 3; - "~*(?:\b)CheeseBot(?:\b)" 3; - "~*(?:\b)CherryPicker(?:\b)" 3; - "~*(?:\b)CheTeam(?:\b)" 3; - "~*(?:\b)ChinaClaw(?:\b)" 3; - "~*(?:\b)Chlooe(?:\b)" 3; - "~*(?:\b)Claritybot(?:\b)" 3; - "~*(?:\b)Cliqzbot(?:\b)" 3; - "~*(?:\b)Cloud\ mapping(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; - "~*(?:\b)Cogentbot(?:\b)" 3; - "~*(?:\b)cognitiveseo(?:\b)" 3; - "~*(?:\b)Collector(?:\b)" 3; - "~*(?:\b)com.plumanalytics(?:\b)" 3; - "~*(?:\b)Copier(?:\b)" 3; - "~*(?:\b)CopyRightCheck(?:\b)" 3; - "~*(?:\b)Copyscape(?:\b)" 3; - "~*(?:\b)Cosmos(?:\b)" 3; - "~*(?:\b)Craftbot(?:\b)" 3; - "~*(?:\b)crawler4j(?:\b)" 3; - "~*(?:\b)crawler.feedback(?:\b)" 3; - "~*(?:\b)crawl.sogou.com(?:\b)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b)" 3; - "~*(?:\b)Crescent(?:\b)" 3; - "~*(?:\b)CrunchBot(?:\b)" 3; - "~*(?:\b)CSHttp(?:\b)" 3; - "~*(?:\b)Curious(?:\b)" 3; - "~*(?:\b)Custo(?:\b)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; - "~*(?:\b)DataCha0s(?:\b)" 3; - "~*(?:\b)DBLBot(?:\b)" 3; - "~*(?:\b)demandbase-bot(?:\b)" 3; - "~*(?:\b)Demon(?:\b)" 3; - "~*(?:\b)Deusu(?:\b)" 3; - "~*(?:\b)Devil(?:\b)" 3; - "~*(?:\b)Digincore(?:\b)" 3; - "~*(?:\b)DigitalPebble(?:\b)" 3; - "~*(?:\b)DIIbot(?:\b)" 3; - "~*(?:\b)Dirbuster(?:\b)" 3; - "~*(?:\b)Disco(?:\b)" 3; - "~*(?:\b)Discobot(?:\b)" 3; - "~*(?:\b)Discoverybot(?:\b)" 3; - "~*(?:\b)Dispatch(?:\b)" 3; - "~*(?:\b)DittoSpyder(?:\b)" 3; - "~*(?:\b)DnyzBot(?:\b)" 3; - "~*(?:\b)DomainAppender(?:\b)" 3; - "~*(?:\b)DomainCrawler(?:\b)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; - "~*(?:\b)DomainStatsBot(?:\b)" 3; - "~*(?:\b)Dotbot(?:\b)" 3; - "~*(?:\b)Download\ Wonder(?:\b)" 3; - "~*(?:\b)Dragonfly(?:\b)" 3; - "~*(?:\b)Drip(?:\b)" 3; - "~*(?:\b)DSearch(?:\b)" 3; - "~*(?:\b)DTS\ Agent(?:\b)" 3; - "~*(?:\b)EasyDL(?:\b)" 3; - "~*(?:\b)Ebingbong(?:\b)" 3; - "~*(?:\b)eCatch(?:\b)" 3; - "~*(?:\b)ECCP/1.0(?:\b)" 3; - "~*(?:\b)Ecxi(?:\b)" 3; - "~*(?:\b)EirGrabber(?:\b)" 3; - "~*(?:\b)EMail\ Siphon(?:\b)" 3; - "~*(?:\b)EMail\ Wolf(?:\b)" 3; - "~*(?:\b)EroCrawler(?:\b)" 3; - "~*(?:\b)evc-batch(?:\b)" 3; - "~*(?:\b)Evil(?:\b)" 3; - "~*(?:\b)Exabot(?:\b)" 3; - "~*(?:\b)Express\ WebPictures(?:\b)" 3; - "~*(?:\b)ExtLinksBot(?:\b)" 3; - "~*(?:\b)Extractor(?:\b)" 3; - "~*(?:\b)ExtractorPro(?:\b)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; - "~*(?:\b)EyeNetIE(?:\b)" 3; - "~*(?:\b)Ezooms(?:\b)" 3; - "~*(?:\b)facebookscraper(?:\b)" 3; - "~*(?:\b)FDM(?:\b)" 3; - "~*(?:\b)FemtosearchBot(?:\b)" 3; - "~*(?:\b)FHscan(?:\b)" 3; - "~*(?:\b)Fimap(?:\b)" 3; - "~*(?:\b)Firefox/7.0(?:\b)" 3; - "~*(?:\b)FlashGet(?:\b)" 3; - "~*(?:\b)Flunky(?:\b)" 3; - "~*(?:\b)Foobot(?:\b)" 3; - "~*(?:\b)Freeuploader(?:\b)" 3; - "~*(?:\b)FrontPage(?:\b)" 3; - "~*(?:\b)FyberSpider(?:\b)" 3; - "~*(?:\b)Fyrebot(?:\b)" 3; - "~*(?:\b)GalaxyBot(?:\b)" 3; - "~*(?:\b)Genieo(?:\b)" 3; - "~*(?:\b)GermCrawler(?:\b)" 3; - "~*(?:\b)Getintent(?:\b)" 3; - "~*(?:\b)GetRight(?:\b)" 3; - "~*(?:\b)GetWeb(?:\b)" 3; - "~*(?:\b)Gigablast(?:\b)" 3; - "~*(?:\b)Gigabot(?:\b)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; - "~*(?:\b)Gotit(?:\b)" 3; - "~*(?:\b)GoZilla(?:\b)" 3; - "~*(?:\b)Go!Zilla(?:\b)" 3; - "~*(?:\b)Grabber(?:\b)" 3; - "~*(?:\b)GrabNet(?:\b)" 3; - "~*(?:\b)Grafula(?:\b)" 3; - "~*(?:\b)GrapeFX(?:\b)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b)" 3; - "~*(?:\b)GridBot(?:\b)" 3; - "~*(?:\b)GT::WWW(?:\b)" 3; - "~*(?:\b)Haansoft(?:\b)" 3; - "~*(?:\b)HaosouSpider(?:\b)" 3; - "~*(?:\b)Harvest(?:\b)" 3; - "~*(?:\b)Havij(?:\b)" 3; - "~*(?:\b)HEADMasterSEO(?:\b)" 3; - "~*(?:\b)Heritrix(?:\b)" 3; - "~*(?:\b)Hloader(?:\b)" 3; - "~*(?:\b)HMView(?:\b)" 3; - "~*(?:\b)HTMLparser(?:\b)" 3; - "~*(?:\b)HTTP::Lite(?:\b)" 3; - "~*(?:\b)HTTrack(?:\b)" 3; - "~*(?:\b)Humanlinks(?:\b)" 3; - "~*(?:\b)HybridBot(?:\b)" 3; - "~*(?:\b)Iblog(?:\b)" 3; - "~*(?:\b)IDBot(?:\b)" 3; - "~*(?:\b)Id-search(?:\b)" 3; - "~*(?:\b)IlseBot(?:\b)" 3; - "~*(?:\b)Image\ Fetch(?:\b)" 3; - "~*(?:\b)Image\ Sucker(?:\b)" 3; - "~*(?:\b)IndeedBot(?:\b)" 3; - "~*(?:\b)Indy\ Library(?:\b)" 3; - "~*(?:\b)InfoNaviRobot(?:\b)" 3; - "~*(?:\b)InfoTekies(?:\b)" 3; - "~*(?:\b)instabid(?:\b)" 3; - "~*(?:\b)Intelliseek(?:\b)" 3; - "~*(?:\b)InterGET(?:\b)" 3; - "~*(?:\b)Internet\ Ninja(?:\b)" 3; - "~*(?:\b)InternetSeer(?:\b)" 3; - "~*(?:\b)internetVista\ monitor(?:\b)" 3; - "~*(?:\b)ips-agent(?:\b)" 3; - "~*(?:\b)Iria(?:\b)" 3; - "~*(?:\b)IRLbot(?:\b)" 3; - "~*(?:\b)Iskanie(?:\b)" 3; - "~*(?:\b)IstellaBot(?:\b)" 3; - "~*(?:\b)JamesBOT(?:\b)" 3; - "~*(?:\b)Jbrofuzz(?:\b)" 3; - "~*(?:\b)JennyBot(?:\b)" 3; - "~*(?:\b)JetCar(?:\b)" 3; - "~*(?:\b)Jetty(?:\b)" 3; - "~*(?:\b)JikeSpider(?:\b)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; - "~*(?:\b)Joomla(?:\b)" 3; - "~*(?:\b)Jorgee(?:\b)" 3; - "~*(?:\b)JustView(?:\b)" 3; - "~*(?:\b)Jyxobot(?:\b)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b)" 3; - "~*(?:\b)Keyword\ Density(?:\b)" 3; - "~*(?:\b)Kozmosbot(?:\b)" 3; - "~*(?:\b)Lanshanbot(?:\b)" 3; - "~*(?:\b)Larbin(?:\b)" 3; - "~*(?:\b)LeechFTP(?:\b)" 3; - "~*(?:\b)LeechGet(?:\b)" 3; - "~*(?:\b)LexiBot(?:\b)" 3; - "~*(?:\b)Lftp(?:\b)" 3; - "~*(?:\b)LibWeb(?:\b)" 3; - "~*(?:\b)Libwhisker(?:\b)" 3; - "~*(?:\b)Lightspeedsystems(?:\b)" 3; - "~*(?:\b)Likse(?:\b)" 3; - "~*(?:\b)Linkdexbot(?:\b)" 3; - "~*(?:\b)LinkextractorPro(?:\b)" 3; - "~*(?:\b)LinkpadBot(?:\b)" 3; - "~*(?:\b)LinkScan(?:\b)" 3; - "~*(?:\b)LinksManager(?:\b)" 3; - "~*(?:\b)LinkWalker(?:\b)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; - "~*(?:\b)Lipperhey(?:\b)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; - "~*(?:\b)Litemage_walker(?:\b)" 3; - "~*(?:\b)Lmspider(?:\b)" 3; - "~*(?:\b)LNSpiderguy(?:\b)" 3; - "~*(?:\b)Ltx71(?:\b)" 3; - "~*(?:\b)lwp-request(?:\b)" 3; - "~*(?:\b)LWP::Simple(?:\b)" 3; - "~*(?:\b)lwp-trivial(?:\b)" 3; - "~*(?:\b)Magnet(?:\b)" 3; - "~*(?:\b)Mag-Net(?:\b)" 3; - "~*(?:\b)magpie-crawler(?:\b)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b)" 3; - "~*(?:\b)Majestic12(?:\b)" 3; - "~*(?:\b)Majestic-SEO(?:\b)" 3; - "~*(?:\b)Majestic\ SEO(?:\b)" 3; - "~*(?:\b)MarkMonitor(?:\b)" 3; - "~*(?:\b)MarkWatch(?:\b)" 3; - "~*(?:\b)Masscan(?:\b)" 3; - "~*(?:\b)Mass\ Downloader(?:\b)" 3; - "~*(?:\b)Mata\ Hari(?:\b)" 3; - "~*(?:\b)MauiBot(?:\b)" 3; - "~*(?:\b)meanpathbot(?:\b)" 3; - "~*(?:\b)Meanpathbot(?:\b)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b)" 3; - "~*(?:\b)mediawords(?:\b)" 3; - "~*(?:\b)MegaIndex.ru(?:\b)" 3; - "~*(?:\b)Metauri(?:\b)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; - "~*(?:\b)MIDown\ tool(?:\b)" 3; - "~*(?:\b)MIIxpc(?:\b)" 3; - "~*(?:\b)Mister\ PiX(?:\b)" 3; - "~*(?:\b)MJ12bot(?:\b)" 3; - "~*(?:\b)Mojeek(?:\b)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; - "~*(?:\b)Mr.4x3(?:\b)" 3; - "~*(?:\b)MSFrontPage(?:\b)" 3; - "~*(?:\b)MSIECrawler(?:\b)" 3; - "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; - "~*(?:\b)muhstik-scan(?:\b)" 3; - "~*(?:\b)Musobot(?:\b)" 3; - "~*(?:\b)Name\ Intelligence(?:\b)" 3; - "~*(?:\b)Nameprotect(?:\b)" 3; - "~*(?:\b)Navroad(?:\b)" 3; - "~*(?:\b)NearSite(?:\b)" 3; - "~*(?:\b)Needle(?:\b)" 3; - "~*(?:\b)Nessus(?:\b)" 3; - "~*(?:\b)NetAnts(?:\b)" 3; - "~*(?:\b)Netcraft(?:\b)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; - "~*(?:\b)NetLyzer(?:\b)" 3; - "~*(?:\b)NetMechanic(?:\b)" 3; - "~*(?:\b)NetSpider(?:\b)" 3; - "~*(?:\b)Nettrack(?:\b)" 3; - "~*(?:\b)Net\ Vampire(?:\b)" 3; - "~*(?:\b)Netvibes(?:\b)" 3; - "~*(?:\b)NetZIP(?:\b)" 3; - "~*(?:\b)NextGenSearchBot(?:\b)" 3; - "~*(?:\b)Nibbler(?:\b)" 3; - "~*(?:\b)NICErsPRO(?:\b)" 3; - "~*(?:\b)Niki-bot(?:\b)" 3; - "~*(?:\b)Nikto(?:\b)" 3; - "~*(?:\b)NimbleCrawler(?:\b)" 3; - "~*(?:\b)Nimbostratus(?:\b)" 3; - "~*(?:\b)Ninja(?:\b)" 3; - "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)NPbot(?:\b)" 3; - "~*(?:\b)Nutch(?:\b)" 3; - "~*(?:\b)oBot(?:\b)" 3; - "~*(?:\b)Octopus(?:\b)" 3; - "~*(?:\b)Offline\ Explorer(?:\b)" 3; - "~*(?:\b)Offline\ Navigator(?:\b)" 3; - "~*(?:\b)OnCrawl(?:\b)" 3; - "~*(?:\b)Openfind(?:\b)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b)" 3; - "~*(?:\b)Openvas(?:\b)" 3; - "~*(?:\b)OpenVAS(?:\b)" 3; - "~*(?:\b)OrangeBot(?:\b)" 3; - "~*(?:\b)OrangeSpider(?:\b)" 3; - "~*(?:\b)OutclicksBot(?:\b)" 3; - "~*(?:\b)OutfoxBot(?:\b)" 3; - "~*(?:\b)PageAnalyzer(?:\b)" 3; - "~*(?:\b)Page\ Analyzer(?:\b)" 3; - "~*(?:\b)PageGrabber(?:\b)" 3; - "~*(?:\b)page\ scorer(?:\b)" 3; - "~*(?:\b)PageScorer(?:\b)" 3; - "~*(?:\b)Pandalytics(?:\b)" 3; - "~*(?:\b)Panscient(?:\b)" 3; - "~*(?:\b)Papa\ Foto(?:\b)" 3; - "~*(?:\b)Pavuk(?:\b)" 3; - "~*(?:\b)pcBrowser(?:\b)" 3; - "~*(?:\b)PECL::HTTP(?:\b)" 3; - "~*(?:\b)PeoplePal(?:\b)" 3; - "~*(?:\b)PHPCrawl(?:\b)" 3; - "~*(?:\b)Picscout(?:\b)" 3; - "~*(?:\b)Picsearch(?:\b)" 3; - "~*(?:\b)PictureFinder(?:\b)" 3; - "~*(?:\b)Pimonster(?:\b)" 3; - "~*(?:\b)Pi-Monster(?:\b)" 3; - "~*(?:\b)Pixray(?:\b)" 3; - "~*(?:\b)PleaseCrawl(?:\b)" 3; - "~*(?:\b)plumanalytics(?:\b)" 3; - "~*(?:\b)Pockey(?:\b)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; - "~*(?:\b)Probethenet(?:\b)" 3; - "~*(?:\b)ProPowerBot(?:\b)" 3; - "~*(?:\b)ProWebWalker(?:\b)" 3; - "~*(?:\b)Psbot(?:\b)" 3; - "~*(?:\b)Pump(?:\b)" 3; - "~*(?:\b)PxBroker(?:\b)" 3; - "~*(?:\b)PyCurl(?:\b)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; - "~*(?:\b)Quick-Crawler(?:\b)" 3; - "~*(?:\b)RankActive(?:\b)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b)" 3; - "~*(?:\b)RankFlex(?:\b)" 3; - "~*(?:\b)RankingBot(?:\b)" 3; - "~*(?:\b)RankingBot2(?:\b)" 3; - "~*(?:\b)Rankivabot(?:\b)" 3; - "~*(?:\b)RankurBot(?:\b)" 3; - "~*(?:\b)RealDownload(?:\b)" 3; - "~*(?:\b)Reaper(?:\b)" 3; - "~*(?:\b)RebelMouse(?:\b)" 3; - "~*(?:\b)Recorder(?:\b)" 3; - "~*(?:\b)RedesScrapy(?:\b)" 3; - "~*(?:\b)ReGet(?:\b)" 3; - "~*(?:\b)RepoMonkey(?:\b)" 3; - "~*(?:\b)Ripper(?:\b)" 3; - "~*(?:\b)RocketCrawler(?:\b)" 3; - "~*(?:\b)Rogerbot(?:\b)" 3; - "~*(?:\b)s1z.ru(?:\b)" 3; - "~*(?:\b)SalesIntelligent(?:\b)" 3; - "~*(?:\b)SBIder(?:\b)" 3; - "~*(?:\b)ScanAlert(?:\b)" 3; - "~*(?:\b)Scanbot(?:\b)" 3; - "~*(?:\b)scan.lol(?:\b)" 3; - "~*(?:\b)ScoutJet(?:\b)" 3; - "~*(?:\b)Scrapy(?:\b)" 3; - "~*(?:\b)Screaming(?:\b)" 3; - "~*(?:\b)ScreenerBot(?:\b)" 3; - "~*(?:\b)Searchestate(?:\b)" 3; - "~*(?:\b)SearchmetricsBot(?:\b)" 3; - "~*(?:\b)Semrush(?:\b)" 3; - "~*(?:\b)SemrushBot(?:\b)" 3; - "~*(?:\b)SEOkicks(?:\b)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; - "~*(?:\b)Seomoz(?:\b)" 3; - "~*(?:\b)SEOprofiler(?:\b)" 3; - "~*(?:\b)seoscanners(?:\b)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b)" 3; - "~*(?:\b)SEOstats(?:\b)" 3; - "~*(?:\b)serpstatbot(?:\b)" 3; - "~*(?:\b)sexsearcher(?:\b)" 3; - "~*(?:\b)Shodan(?:\b)" 3; - "~*(?:\b)Siphon(?:\b)" 3; - "~*(?:\b)SISTRIX(?:\b)" 3; - "~*(?:\b)Sitebeam(?:\b)" 3; - "~*(?:\b)SiteExplorer(?:\b)" 3; - "~*(?:\b)Siteimprove(?:\b)" 3; - "~*(?:\b)SiteLockSpider(?:\b)" 3; - "~*(?:\b)SiteSnagger(?:\b)" 3; - "~*(?:\b)SiteSucker(?:\b)" 3; - "~*(?:\b)Site\ Sucker(?:\b)" 3; - "~*(?:\b)Sitevigil(?:\b)" 3; - "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 3; - "~*(?:\b)SlySearch(?:\b)" 3; - "~*(?:\b)SmartDownload(?:\b)" 3; - "~*(?:\b)SMTBot(?:\b)" 3; - "~*(?:\b)Snake(?:\b)" 3; - "~*(?:\b)Snapbot(?:\b)" 3; - "~*(?:\b)Snoopy(?:\b)" 3; - "~*(?:\b)SocialRankIOBot(?:\b)" 3; - "~*(?:\b)Sociscraper(?:\b)" 3; - "~*(?:\b)sogouspider(?:\b)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; - "~*(?:\b)Sosospider(?:\b)" 3; - "~*(?:\b)Sottopop(?:\b)" 3; - "~*(?:\b)SpaceBison(?:\b)" 3; - "~*(?:\b)Spammen(?:\b)" 3; - "~*(?:\b)SpankBot(?:\b)" 3; - "~*(?:\b)Spanner(?:\b)" 3; - "~*(?:\b)sp_auditbot(?:\b)" 3; - "~*(?:\b)Spbot(?:\b)" 3; - "~*(?:\b)Spinn3r(?:\b)" 3; - "~*(?:\b)SputnikBot(?:\b)" 3; - "~*(?:\b)spyfu(?:\b)" 3; - "~*(?:\b)Sqlmap(?:\b)" 3; - "~*(?:\b)Sqlworm(?:\b)" 3; - "~*(?:\b)Sqworm(?:\b)" 3; - "~*(?:\b)Steeler(?:\b)" 3; - "~*(?:\b)Stripper(?:\b)" 3; - "~*(?:\b)Sucker(?:\b)" 3; - "~*(?:\b)Sucuri(?:\b)" 3; - "~*(?:\b)SuperBot(?:\b)" 3; - "~*(?:\b)SuperHTTP(?:\b)" 3; - "~*(?:\b)Surfbot(?:\b)" 3; - "~*(?:\b)SurveyBot(?:\b)" 3; - "~*(?:\b)Suzuran(?:\b)" 3; - "~*(?:\b)Swiftbot(?:\b)" 3; - "~*(?:\b)sysscan(?:\b)" 3; - "~*(?:\b)Szukacz(?:\b)" 3; - "~*(?:\b)T0PHackTeam(?:\b)" 3; - "~*(?:\b)T8Abot(?:\b)" 3; - "~*(?:\b)tAkeOut(?:\b)" 3; - "~*(?:\b)Teleport(?:\b)" 3; - "~*(?:\b)TeleportPro(?:\b)" 3; - "~*(?:\b)Telesoft(?:\b)" 3; - "~*(?:\b)Telesphoreo(?:\b)" 3; - "~*(?:\b)Telesphorep(?:\b)" 3; - "~*(?:\b)The\ Intraformant(?:\b)" 3; - "~*(?:\b)TheNomad(?:\b)" 3; - "~*(?:\b)TightTwatBot(?:\b)" 3; - "~*(?:\b)Titan(?:\b)" 3; - "~*(?:\b)Toata(?:\b)" 3; - "~*(?:\b)Toweyabot(?:\b)" 3; - "~*(?:\b)Tracemyfile(?:\b)" 3; - "~*(?:\b)Trendiction(?:\b)" 3; - "~*(?:\b)Trendictionbot(?:\b)" 3; - "~*(?:\b)trendiction.com(?:\b)" 3; - "~*(?:\b)trendiction.de(?:\b)" 3; - "~*(?:\b)True_Robot(?:\b)" 3; - "~*(?:\b)Turingos(?:\b)" 3; - "~*(?:\b)Turnitin(?:\b)" 3; - "~*(?:\b)TurnitinBot(?:\b)" 3; - "~*(?:\b)TwengaBot(?:\b)" 3; - "~*(?:\b)Twice(?:\b)" 3; - "~*(?:\b)Typhoeus(?:\b)" 3; - "~*(?:\b)UnisterBot(?:\b)" 3; - "~*(?:\b)Upflow(?:\b)" 3; - "~*(?:\b)URLy.Warning(?:\b)" 3; - "~*(?:\b)URLy\ Warning(?:\b)" 3; - "~*(?:\b)Vacuum(?:\b)" 3; - "~*(?:\b)Vagabondo(?:\b)" 3; - "~*(?:\b)VB\ Project(?:\b)" 3; - "~*(?:\b)VCI(?:\b)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b)" 3; - "~*(?:\b)VidibleScraper(?:\b)" 3; - "~*(?:\b)Virusdie(?:\b)" 3; - "~*(?:\b)VoidEYE(?:\b)" 3; - "~*(?:\b)Voil(?:\b)" 3; - "~*(?:\b)Voltron(?:\b)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b)" 3; - "~*(?:\b)WallpapersHD(?:\b)" 3; - "~*(?:\b)WASALive-Bot(?:\b)" 3; - "~*(?:\b)WBSearchBot(?:\b)" 3; - "~*(?:\b)Webalta(?:\b)" 3; - "~*(?:\b)WebAuto(?:\b)" 3; - "~*(?:\b)Web\ Auto(?:\b)" 3; - "~*(?:\b)WebBandit(?:\b)" 3; - "~*(?:\b)WebCollage(?:\b)" 3; - "~*(?:\b)Web\ Collage(?:\b)" 3; - "~*(?:\b)WebCopier(?:\b)" 3; - "~*(?:\b)WEBDAV(?:\b)" 3; - "~*(?:\b)WebEnhancer(?:\b)" 3; - "~*(?:\b)Web\ Enhancer(?:\b)" 3; - "~*(?:\b)WebFetch(?:\b)" 3; - "~*(?:\b)Web\ Fetch(?:\b)" 3; - "~*(?:\b)WebFuck(?:\b)" 3; - "~*(?:\b)Web\ Fuck(?:\b)" 3; - "~*(?:\b)WebGo\ IS(?:\b)" 3; - "~*(?:\b)WebImageCollector(?:\b)" 3; - "~*(?:\b)WebLeacher(?:\b)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; - "~*(?:\b)webmeup-crawler(?:\b)" 3; - "~*(?:\b)WebPix(?:\b)" 3; - "~*(?:\b)Web\ Pix(?:\b)" 3; - "~*(?:\b)WebReaper(?:\b)" 3; - "~*(?:\b)WebSauger(?:\b)" 3; - "~*(?:\b)Web\ Sauger(?:\b)" 3; - "~*(?:\b)Webshag(?:\b)" 3; - "~*(?:\b)WebsiteExtractor(?:\b)" 3; - "~*(?:\b)WebsiteQuester(?:\b)" 3; - "~*(?:\b)Website\ Quester(?:\b)" 3; - "~*(?:\b)Webster(?:\b)" 3; - "~*(?:\b)WebStripper(?:\b)" 3; - "~*(?:\b)WebSucker(?:\b)" 3; - "~*(?:\b)Web\ Sucker(?:\b)" 3; - "~*(?:\b)WebWhacker(?:\b)" 3; - "~*(?:\b)WebZIP(?:\b)" 3; - "~*(?:\b)WeSEE(?:\b)" 3; - "~*(?:\b)Whack(?:\b)" 3; - "~*(?:\b)Whacker(?:\b)" 3; - "~*(?:\b)Whatweb(?:\b)" 3; - "~*(?:\b)Who.is\ Bot(?:\b)" 3; - "~*(?:\b)Widow(?:\b)" 3; - "~*(?:\b)WinHTTrack(?:\b)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; - "~*(?:\b)WISENutbot(?:\b)" 3; - "~*(?:\b)Wonderbot(?:\b)" 3; - "~*(?:\b)Woobot(?:\b)" 3; - "~*(?:\b)Wotbox(?:\b)" 3; - "~*(?:\b)Wprecon(?:\b)" 3; - "~*(?:\b)WPScan(?:\b)" 3; - "~*(?:\b)WWW-Collector-E(?:\b)" 3; - "~*(?:\b)WWW-Mechanize(?:\b)" 3; - "~*(?:\b)WWW::Mechanize(?:\b)" 3; - "~*(?:\b)WWWOFFLE(?:\b)" 3; - "~*(?:\b)x09Mozilla(?:\b)" 3; - "~*(?:\b)x22Mozilla(?:\b)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; - "~*(?:\b)Xenu(?:\b)" 3; - "~*(?:\b)xpymep1.exe(?:\b)" 3; - "~*(?:\b)YoudaoBot(?:\b)" 3; - "~*(?:\b)Zade(?:\b)" 3; - "~*(?:\b)Zauba(?:\b)" 3; - "~*(?:\b)zauba.io(?:\b)" 3; - "~*(?:\b)Zermelo(?:\b)" 3; - "~*(?:\b)Zeus(?:\b)" 3; - "~*(?:\b)zgrab(?:\b)" 3; - "~*(?:\b)Zitebot(?:\b)" 3; - "~*(?:\b)ZmEu(?:\b)" 3; - "~*(?:\b)ZumBot(?:\b)" 3; - "~*(?:\b)ZyBorg(?:\b)" 3; + "~*(?:\b)360Spider(?:\b|)" 3; + "~*(?:\b)404checker(?:\b|)" 3; + "~*(?:\b)404enemy(?:\b|)" 3; + "~*(?:\b)80legs(?:\b|)" 3; + "~*(?:\b)Abonti(?:\b|)" 3; + "~*(?:\b)Aboundex(?:\b|)" 3; + "~*(?:\b)Aboundexbot(?:\b|)" 3; + "~*(?:\b)Acunetix(?:\b|)" 3; + "~*(?:\b)ADmantX(?:\b|)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; + "~*(?:\b)AhrefsBot(?:\b|)" 3; + "~*(?:\b)AIBOT(?:\b|)" 3; + "~*(?:\b)AiHitBot(?:\b|)" 3; + "~*(?:\b)Aipbot(?:\b|)" 3; + "~*(?:\b)Alexibot(?:\b|)" 3; + "~*(?:\b)Alligator(?:\b|)" 3; + "~*(?:\b)AllSubmitter(?:\b|)" 3; + "~*(?:\b)AlphaBot(?:\b|)" 3; + "~*(?:\b)Anarchie(?:\b|)" 3; + "~*(?:\b)Apexoo(?:\b|)" 3; + "~*(?:\b)archive.org_bot(?:\b|)" 3; + "~*(?:\b)ASPSeek(?:\b|)" 3; + "~*(?:\b)Asterias(?:\b|)" 3; + "~*(?:\b)Attach(?:\b|)" 3; + "~*(?:\b)autoemailspider(?:\b|)" 3; + "~*(?:\b)BackDoorBot(?:\b|)" 3; + "~*(?:\b)Backlink-Ceck(?:\b|)" 3; + "~*(?:\b)backlink-check(?:\b|)" 3; + "~*(?:\b)BacklinkCrawler(?:\b|)" 3; + "~*(?:\b)BackStreet(?:\b|)" 3; + "~*(?:\b)BackWeb(?:\b|)" 3; + "~*(?:\b)Badass(?:\b|)" 3; + "~*(?:\b)Bandit(?:\b|)" 3; + "~*(?:\b)Barkrowler(?:\b|)" 3; + "~*(?:\b)BatchFTP(?:\b|)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; + "~*(?:\b)BBBike(?:\b|)" 3; + "~*(?:\b)BDCbot(?:\b|)" 3; + "~*(?:\b)BDFetch(?:\b|)" 3; + "~*(?:\b)BetaBot(?:\b|)" 3; + "~*(?:\b)Bigfoot(?:\b|)" 3; + "~*(?:\b)Bitacle(?:\b|)" 3; + "~*(?:\b)Blackboard(?:\b|)" 3; + "~*(?:\b)Black\ Hole(?:\b|)" 3; + "~*(?:\b)BlackWidow(?:\b|)" 3; + "~*(?:\b)BLEXBot(?:\b|)" 3; + "~*(?:\b)Blow(?:\b|)" 3; + "~*(?:\b)BlowFish(?:\b|)" 3; + "~*(?:\b)Boardreader(?:\b|)" 3; + "~*(?:\b)Bolt(?:\b|)" 3; + "~*(?:\b)BotALot(?:\b|)" 3; + "~*(?:\b)Brandprotect(?:\b|)" 3; + "~*(?:\b)Brandwatch(?:\b|)" 3; + "~*(?:\b)Buddy(?:\b|)" 3; + "~*(?:\b)BuiltBotTough(?:\b|)" 3; + "~*(?:\b)BuiltWith(?:\b|)" 3; + "~*(?:\b)Bullseye(?:\b|)" 3; + "~*(?:\b)BunnySlippers(?:\b|)" 3; + "~*(?:\b)BuzzSumo(?:\b|)" 3; + "~*(?:\b)Calculon(?:\b|)" 3; + "~*(?:\b)CATExplorador(?:\b|)" 3; + "~*(?:\b)CazoodleBot(?:\b|)" 3; + "~*(?:\b)CCBot(?:\b|)" 3; + "~*(?:\b)Cegbfeieh(?:\b|)" 3; + "~*(?:\b)CheeseBot(?:\b|)" 3; + "~*(?:\b)CherryPicker(?:\b|)" 3; + "~*(?:\b)CheTeam(?:\b|)" 3; + "~*(?:\b)ChinaClaw(?:\b|)" 3; + "~*(?:\b)Chlooe(?:\b|)" 3; + "~*(?:\b)Claritybot(?:\b|)" 3; + "~*(?:\b)Cliqzbot(?:\b|)" 3; + "~*(?:\b)Cloud\ mapping(?:\b|)" 3; + "~*(?:\b)coccocbot-web(?:\b|)" 3; + "~*(?:\b)Cogentbot(?:\b|)" 3; + "~*(?:\b)cognitiveseo(?:\b|)" 3; + "~*(?:\b)Collector(?:\b|)" 3; + "~*(?:\b)com.plumanalytics(?:\b|)" 3; + "~*(?:\b)Copier(?:\b|)" 3; + "~*(?:\b)CopyRightCheck(?:\b|)" 3; + "~*(?:\b)Copyscape(?:\b|)" 3; + "~*(?:\b)Cosmos(?:\b|)" 3; + "~*(?:\b)Craftbot(?:\b|)" 3; + "~*(?:\b)crawler4j(?:\b|)" 3; + "~*(?:\b)crawler.feedback(?:\b|)" 3; + "~*(?:\b)crawl.sogou.com(?:\b|)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; + "~*(?:\b)Crescent(?:\b|)" 3; + "~*(?:\b)CrunchBot(?:\b|)" 3; + "~*(?:\b)CSHttp(?:\b|)" 3; + "~*(?:\b)Curious(?:\b|)" 3; + "~*(?:\b)Custo(?:\b|)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; + "~*(?:\b)DataCha0s(?:\b|)" 3; + "~*(?:\b)DBLBot(?:\b|)" 3; + "~*(?:\b)demandbase-bot(?:\b|)" 3; + "~*(?:\b)Demon(?:\b|)" 3; + "~*(?:\b)Deusu(?:\b|)" 3; + "~*(?:\b)Devil(?:\b|)" 3; + "~*(?:\b)Digincore(?:\b|)" 3; + "~*(?:\b)DigitalPebble(?:\b|)" 3; + "~*(?:\b)DIIbot(?:\b|)" 3; + "~*(?:\b)Dirbuster(?:\b|)" 3; + "~*(?:\b)Disco(?:\b|)" 3; + "~*(?:\b)Discobot(?:\b|)" 3; + "~*(?:\b)Discoverybot(?:\b|)" 3; + "~*(?:\b)Dispatch(?:\b|)" 3; + "~*(?:\b)DittoSpyder(?:\b|)" 3; + "~*(?:\b)DnyzBot(?:\b|)" 3; + "~*(?:\b)DomainAppender(?:\b|)" 3; + "~*(?:\b)DomainCrawler(?:\b|)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; + "~*(?:\b)DomainStatsBot(?:\b|)" 3; + "~*(?:\b)Dotbot(?:\b|)" 3; + "~*(?:\b)Download\ Wonder(?:\b|)" 3; + "~*(?:\b)Dragonfly(?:\b|)" 3; + "~*(?:\b)Drip(?:\b|)" 3; + "~*(?:\b)DSearch(?:\b|)" 3; + "~*(?:\b)DTS\ Agent(?:\b|)" 3; + "~*(?:\b)EasyDL(?:\b|)" 3; + "~*(?:\b)Ebingbong(?:\b|)" 3; + "~*(?:\b)eCatch(?:\b|)" 3; + "~*(?:\b)ECCP/1.0(?:\b|)" 3; + "~*(?:\b)Ecxi(?:\b|)" 3; + "~*(?:\b)EirGrabber(?:\b|)" 3; + "~*(?:\b)EMail\ Siphon(?:\b|)" 3; + "~*(?:\b)EMail\ Wolf(?:\b|)" 3; + "~*(?:\b)EroCrawler(?:\b|)" 3; + "~*(?:\b)evc-batch(?:\b|)" 3; + "~*(?:\b)Evil(?:\b|)" 3; + "~*(?:\b)Exabot(?:\b|)" 3; + "~*(?:\b)Express\ WebPictures(?:\b|)" 3; + "~*(?:\b)ExtLinksBot(?:\b|)" 3; + "~*(?:\b)Extractor(?:\b|)" 3; + "~*(?:\b)ExtractorPro(?:\b|)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; + "~*(?:\b)EyeNetIE(?:\b|)" 3; + "~*(?:\b)Ezooms(?:\b|)" 3; + "~*(?:\b)facebookscraper(?:\b|)" 3; + "~*(?:\b)FDM(?:\b|)" 3; + "~*(?:\b)FemtosearchBot(?:\b|)" 3; + "~*(?:\b)FHscan(?:\b|)" 3; + "~*(?:\b)Fimap(?:\b|)" 3; + "~*(?:\b)Firefox/7.0(?:\b|)" 3; + "~*(?:\b)FlashGet(?:\b|)" 3; + "~*(?:\b)Flunky(?:\b|)" 3; + "~*(?:\b)Foobot(?:\b|)" 3; + "~*(?:\b)Freeuploader(?:\b|)" 3; + "~*(?:\b)FrontPage(?:\b|)" 3; + "~*(?:\b)FyberSpider(?:\b|)" 3; + "~*(?:\b)Fyrebot(?:\b|)" 3; + "~*(?:\b)GalaxyBot(?:\b|)" 3; + "~*(?:\b)Genieo(?:\b|)" 3; + "~*(?:\b)GermCrawler(?:\b|)" 3; + "~*(?:\b)Getintent(?:\b|)" 3; + "~*(?:\b)GetRight(?:\b|)" 3; + "~*(?:\b)GetWeb(?:\b|)" 3; + "~*(?:\b)Gigablast(?:\b|)" 3; + "~*(?:\b)Gigabot(?:\b|)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; + "~*(?:\b)Gotit(?:\b|)" 3; + "~*(?:\b)GoZilla(?:\b|)" 3; + "~*(?:\b)Go!Zilla(?:\b|)" 3; + "~*(?:\b)Grabber(?:\b|)" 3; + "~*(?:\b)GrabNet(?:\b|)" 3; + "~*(?:\b)Grafula(?:\b|)" 3; + "~*(?:\b)GrapeFX(?:\b|)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; + "~*(?:\b)GridBot(?:\b|)" 3; + "~*(?:\b)GT::WWW(?:\b|)" 3; + "~*(?:\b)Haansoft(?:\b|)" 3; + "~*(?:\b)HaosouSpider(?:\b|)" 3; + "~*(?:\b)Harvest(?:\b|)" 3; + "~*(?:\b)Havij(?:\b|)" 3; + "~*(?:\b)HEADMasterSEO(?:\b|)" 3; + "~*(?:\b)Heritrix(?:\b|)" 3; + "~*(?:\b)Hloader(?:\b|)" 3; + "~*(?:\b)HMView(?:\b|)" 3; + "~*(?:\b)HTMLparser(?:\b|)" 3; + "~*(?:\b)HTTP::Lite(?:\b|)" 3; + "~*(?:\b)HTTrack(?:\b|)" 3; + "~*(?:\b)Humanlinks(?:\b|)" 3; + "~*(?:\b)HybridBot(?:\b|)" 3; + "~*(?:\b)Iblog(?:\b|)" 3; + "~*(?:\b)IDBot(?:\b|)" 3; + "~*(?:\b)Id-search(?:\b|)" 3; + "~*(?:\b)IlseBot(?:\b|)" 3; + "~*(?:\b)Image\ Fetch(?:\b|)" 3; + "~*(?:\b)Image\ Sucker(?:\b|)" 3; + "~*(?:\b)IndeedBot(?:\b|)" 3; + "~*(?:\b)Indy\ Library(?:\b|)" 3; + "~*(?:\b)InfoNaviRobot(?:\b|)" 3; + "~*(?:\b)InfoTekies(?:\b|)" 3; + "~*(?:\b)instabid(?:\b|)" 3; + "~*(?:\b)Intelliseek(?:\b|)" 3; + "~*(?:\b)InterGET(?:\b|)" 3; + "~*(?:\b)Internet\ Ninja(?:\b|)" 3; + "~*(?:\b)InternetSeer(?:\b|)" 3; + "~*(?:\b)internetVista\ monitor(?:\b|)" 3; + "~*(?:\b)ips-agent(?:\b|)" 3; + "~*(?:\b)Iria(?:\b|)" 3; + "~*(?:\b)IRLbot(?:\b|)" 3; + "~*(?:\b)Iskanie(?:\b|)" 3; + "~*(?:\b)IstellaBot(?:\b|)" 3; + "~*(?:\b)JamesBOT(?:\b|)" 3; + "~*(?:\b)Jbrofuzz(?:\b|)" 3; + "~*(?:\b)JennyBot(?:\b|)" 3; + "~*(?:\b)JetCar(?:\b|)" 3; + "~*(?:\b)Jetty(?:\b|)" 3; + "~*(?:\b)JikeSpider(?:\b|)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; + "~*(?:\b)Joomla(?:\b|)" 3; + "~*(?:\b)Jorgee(?:\b|)" 3; + "~*(?:\b)JustView(?:\b|)" 3; + "~*(?:\b)Jyxobot(?:\b|)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; + "~*(?:\b)Keyword\ Density(?:\b|)" 3; + "~*(?:\b)Kozmosbot(?:\b|)" 3; + "~*(?:\b)Lanshanbot(?:\b|)" 3; + "~*(?:\b)Larbin(?:\b|)" 3; + "~*(?:\b)LeechFTP(?:\b|)" 3; + "~*(?:\b)LeechGet(?:\b|)" 3; + "~*(?:\b)LexiBot(?:\b|)" 3; + "~*(?:\b)Lftp(?:\b|)" 3; + "~*(?:\b)LibWeb(?:\b|)" 3; + "~*(?:\b)Libwhisker(?:\b|)" 3; + "~*(?:\b)Lightspeedsystems(?:\b|)" 3; + "~*(?:\b)Likse(?:\b|)" 3; + "~*(?:\b)Linkdexbot(?:\b|)" 3; + "~*(?:\b)LinkextractorPro(?:\b|)" 3; + "~*(?:\b)LinkpadBot(?:\b|)" 3; + "~*(?:\b)LinkScan(?:\b|)" 3; + "~*(?:\b)LinksManager(?:\b|)" 3; + "~*(?:\b)LinkWalker(?:\b|)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; + "~*(?:\b)Lipperhey(?:\b|)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; + "~*(?:\b)Litemage_walker(?:\b|)" 3; + "~*(?:\b)Lmspider(?:\b|)" 3; + "~*(?:\b)LNSpiderguy(?:\b|)" 3; + "~*(?:\b)Ltx71(?:\b|)" 3; + "~*(?:\b)lwp-request(?:\b|)" 3; + "~*(?:\b)LWP::Simple(?:\b|)" 3; + "~*(?:\b)lwp-trivial(?:\b|)" 3; + "~*(?:\b)Magnet(?:\b|)" 3; + "~*(?:\b)Mag-Net(?:\b|)" 3; + "~*(?:\b)magpie-crawler(?:\b|)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; + "~*(?:\b)Majestic12(?:\b|)" 3; + "~*(?:\b)Majestic-SEO(?:\b|)" 3; + "~*(?:\b)Majestic\ SEO(?:\b|)" 3; + "~*(?:\b)MarkMonitor(?:\b|)" 3; + "~*(?:\b)MarkWatch(?:\b|)" 3; + "~*(?:\b)Masscan(?:\b|)" 3; + "~*(?:\b)Mass\ Downloader(?:\b|)" 3; + "~*(?:\b)Mata\ Hari(?:\b|)" 3; + "~*(?:\b)MauiBot(?:\b|)" 3; + "~*(?:\b)meanpathbot(?:\b|)" 3; + "~*(?:\b)Meanpathbot(?:\b|)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; + "~*(?:\b)mediawords(?:\b|)" 3; + "~*(?:\b)MegaIndex.ru(?:\b|)" 3; + "~*(?:\b)Metauri(?:\b|)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; + "~*(?:\b)MIDown\ tool(?:\b|)" 3; + "~*(?:\b)MIIxpc(?:\b|)" 3; + "~*(?:\b)Mister\ PiX(?:\b|)" 3; + "~*(?:\b)MJ12bot(?:\b|)" 3; + "~*(?:\b)Mojeek(?:\b|)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; + "~*(?:\b)Mr.4x3(?:\b|)" 3; + "~*(?:\b)MSFrontPage(?:\b|)" 3; + "~*(?:\b)MSIECrawler(?:\b|)" 3; + "~*(?:\b)Msrabot(?:\b|)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; + "~*(?:\b)muhstik-scan(?:\b|)" 3; + "~*(?:\b)Musobot(?:\b|)" 3; + "~*(?:\b)Name\ Intelligence(?:\b|)" 3; + "~*(?:\b)Nameprotect(?:\b|)" 3; + "~*(?:\b)Navroad(?:\b|)" 3; + "~*(?:\b)NearSite(?:\b|)" 3; + "~*(?:\b)Needle(?:\b|)" 3; + "~*(?:\b)Nessus(?:\b|)" 3; + "~*(?:\b)NetAnts(?:\b|)" 3; + "~*(?:\b)Netcraft(?:\b|)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; + "~*(?:\b)NetLyzer(?:\b|)" 3; + "~*(?:\b)NetMechanic(?:\b|)" 3; + "~*(?:\b)NetSpider(?:\b|)" 3; + "~*(?:\b)Nettrack(?:\b|)" 3; + "~*(?:\b)Net\ Vampire(?:\b|)" 3; + "~*(?:\b)Netvibes(?:\b|)" 3; + "~*(?:\b)NetZIP(?:\b|)" 3; + "~*(?:\b)NextGenSearchBot(?:\b|)" 3; + "~*(?:\b)Nibbler(?:\b|)" 3; + "~*(?:\b)NICErsPRO(?:\b|)" 3; + "~*(?:\b)Niki-bot(?:\b|)" 3; + "~*(?:\b)Nikto(?:\b|)" 3; + "~*(?:\b)NimbleCrawler(?:\b|)" 3; + "~*(?:\b)Nimbostratus(?:\b|)" 3; + "~*(?:\b)Ninja(?:\b|)" 3; + "~*(?:\b)Nmap(?:\b|)" 3; + "~*(?:\b)NPbot(?:\b|)" 3; + "~*(?:\b)Nutch(?:\b|)" 3; + "~*(?:\b)oBot(?:\b|)" 3; + "~*(?:\b)Octopus(?:\b|)" 3; + "~*(?:\b)Offline\ Explorer(?:\b|)" 3; + "~*(?:\b)Offline\ Navigator(?:\b|)" 3; + "~*(?:\b)OnCrawl(?:\b|)" 3; + "~*(?:\b)Openfind(?:\b|)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; + "~*(?:\b)Openvas(?:\b|)" 3; + "~*(?:\b)OpenVAS(?:\b|)" 3; + "~*(?:\b)OrangeBot(?:\b|)" 3; + "~*(?:\b)OrangeSpider(?:\b|)" 3; + "~*(?:\b)OutclicksBot(?:\b|)" 3; + "~*(?:\b)OutfoxBot(?:\b|)" 3; + "~*(?:\b)PageAnalyzer(?:\b|)" 3; + "~*(?:\b)Page\ Analyzer(?:\b|)" 3; + "~*(?:\b)PageGrabber(?:\b|)" 3; + "~*(?:\b)page\ scorer(?:\b|)" 3; + "~*(?:\b)PageScorer(?:\b|)" 3; + "~*(?:\b)Pandalytics(?:\b|)" 3; + "~*(?:\b)Panscient(?:\b|)" 3; + "~*(?:\b)Papa\ Foto(?:\b|)" 3; + "~*(?:\b)Pavuk(?:\b|)" 3; + "~*(?:\b)pcBrowser(?:\b|)" 3; + "~*(?:\b)PECL::HTTP(?:\b|)" 3; + "~*(?:\b)PeoplePal(?:\b|)" 3; + "~*(?:\b)PHPCrawl(?:\b|)" 3; + "~*(?:\b)Picscout(?:\b|)" 3; + "~*(?:\b)Picsearch(?:\b|)" 3; + "~*(?:\b)PictureFinder(?:\b|)" 3; + "~*(?:\b)Pimonster(?:\b|)" 3; + "~*(?:\b)Pi-Monster(?:\b|)" 3; + "~*(?:\b)Pixray(?:\b|)" 3; + "~*(?:\b)PleaseCrawl(?:\b|)" 3; + "~*(?:\b)plumanalytics(?:\b|)" 3; + "~*(?:\b)Pockey(?:\b|)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; + "~*(?:\b)Probethenet(?:\b|)" 3; + "~*(?:\b)ProPowerBot(?:\b|)" 3; + "~*(?:\b)ProWebWalker(?:\b|)" 3; + "~*(?:\b)Psbot(?:\b|)" 3; + "~*(?:\b)Pump(?:\b|)" 3; + "~*(?:\b)PxBroker(?:\b|)" 3; + "~*(?:\b)PyCurl(?:\b|)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; + "~*(?:\b)Quick-Crawler(?:\b|)" 3; + "~*(?:\b)RankActive(?:\b|)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; + "~*(?:\b)RankFlex(?:\b|)" 3; + "~*(?:\b)RankingBot(?:\b|)" 3; + "~*(?:\b)RankingBot2(?:\b|)" 3; + "~*(?:\b)Rankivabot(?:\b|)" 3; + "~*(?:\b)RankurBot(?:\b|)" 3; + "~*(?:\b)RealDownload(?:\b|)" 3; + "~*(?:\b)Reaper(?:\b|)" 3; + "~*(?:\b)RebelMouse(?:\b|)" 3; + "~*(?:\b)Recorder(?:\b|)" 3; + "~*(?:\b)RedesScrapy(?:\b|)" 3; + "~*(?:\b)ReGet(?:\b|)" 3; + "~*(?:\b)RepoMonkey(?:\b|)" 3; + "~*(?:\b)Ripper(?:\b|)" 3; + "~*(?:\b)RocketCrawler(?:\b|)" 3; + "~*(?:\b)Rogerbot(?:\b|)" 3; + "~*(?:\b)s1z.ru(?:\b|)" 3; + "~*(?:\b)SalesIntelligent(?:\b|)" 3; + "~*(?:\b)SBIder(?:\b|)" 3; + "~*(?:\b)ScanAlert(?:\b|)" 3; + "~*(?:\b)Scanbot(?:\b|)" 3; + "~*(?:\b)scan.lol(?:\b|)" 3; + "~*(?:\b)ScoutJet(?:\b|)" 3; + "~*(?:\b)Scrapy(?:\b|)" 3; + "~*(?:\b)Screaming(?:\b|)" 3; + "~*(?:\b)ScreenerBot(?:\b|)" 3; + "~*(?:\b)Searchestate(?:\b|)" 3; + "~*(?:\b)SearchmetricsBot(?:\b|)" 3; + "~*(?:\b)Semrush(?:\b|)" 3; + "~*(?:\b)SemrushBot(?:\b|)" 3; + "~*(?:\b)SEOkicks(?:\b|)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; + "~*(?:\b)Seomoz(?:\b|)" 3; + "~*(?:\b)SEOprofiler(?:\b|)" 3; + "~*(?:\b)seoscanners(?:\b|)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; + "~*(?:\b)SEOstats(?:\b|)" 3; + "~*(?:\b)serpstatbot(?:\b|)" 3; + "~*(?:\b)sexsearcher(?:\b|)" 3; + "~*(?:\b)Shodan(?:\b|)" 3; + "~*(?:\b)Siphon(?:\b|)" 3; + "~*(?:\b)SISTRIX(?:\b|)" 3; + "~*(?:\b)Sitebeam(?:\b|)" 3; + "~*(?:\b)SiteExplorer(?:\b|)" 3; + "~*(?:\b)Siteimprove(?:\b|)" 3; + "~*(?:\b)SiteLockSpider(?:\b|)" 3; + "~*(?:\b)SiteSnagger(?:\b|)" 3; + "~*(?:\b)SiteSucker(?:\b|)" 3; + "~*(?:\b)Site\ Sucker(?:\b|)" 3; + "~*(?:\b)Sitevigil(?:\b|)" 3; + "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 3; + "~*(?:\b)SlySearch(?:\b|)" 3; + "~*(?:\b)SmartDownload(?:\b|)" 3; + "~*(?:\b)SMTBot(?:\b|)" 3; + "~*(?:\b)Snake(?:\b|)" 3; + "~*(?:\b)Snapbot(?:\b|)" 3; + "~*(?:\b)Snoopy(?:\b|)" 3; + "~*(?:\b)SocialRankIOBot(?:\b|)" 3; + "~*(?:\b)Sociscraper(?:\b|)" 3; + "~*(?:\b)sogouspider(?:\b|)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; + "~*(?:\b)Sosospider(?:\b|)" 3; + "~*(?:\b)Sottopop(?:\b|)" 3; + "~*(?:\b)SpaceBison(?:\b|)" 3; + "~*(?:\b)Spammen(?:\b|)" 3; + "~*(?:\b)SpankBot(?:\b|)" 3; + "~*(?:\b)Spanner(?:\b|)" 3; + "~*(?:\b)sp_auditbot(?:\b|)" 3; + "~*(?:\b)Spbot(?:\b|)" 3; + "~*(?:\b)Spinn3r(?:\b|)" 3; + "~*(?:\b)SputnikBot(?:\b|)" 3; + "~*(?:\b)spyfu(?:\b|)" 3; + "~*(?:\b)Sqlmap(?:\b|)" 3; + "~*(?:\b)Sqlworm(?:\b|)" 3; + "~*(?:\b)Sqworm(?:\b|)" 3; + "~*(?:\b)Steeler(?:\b|)" 3; + "~*(?:\b)Stripper(?:\b|)" 3; + "~*(?:\b)Sucker(?:\b|)" 3; + "~*(?:\b)Sucuri(?:\b|)" 3; + "~*(?:\b)SuperBot(?:\b|)" 3; + "~*(?:\b)SuperHTTP(?:\b|)" 3; + "~*(?:\b)Surfbot(?:\b|)" 3; + "~*(?:\b)SurveyBot(?:\b|)" 3; + "~*(?:\b)Suzuran(?:\b|)" 3; + "~*(?:\b)Swiftbot(?:\b|)" 3; + "~*(?:\b)sysscan(?:\b|)" 3; + "~*(?:\b)Szukacz(?:\b|)" 3; + "~*(?:\b)T0PHackTeam(?:\b|)" 3; + "~*(?:\b)T8Abot(?:\b|)" 3; + "~*(?:\b)tAkeOut(?:\b|)" 3; + "~*(?:\b)Teleport(?:\b|)" 3; + "~*(?:\b)TeleportPro(?:\b|)" 3; + "~*(?:\b)Telesoft(?:\b|)" 3; + "~*(?:\b)Telesphoreo(?:\b|)" 3; + "~*(?:\b)Telesphorep(?:\b|)" 3; + "~*(?:\b)The\ Intraformant(?:\b|)" 3; + "~*(?:\b)TheNomad(?:\b|)" 3; + "~*(?:\b)TightTwatBot(?:\b|)" 3; + "~*(?:\b)Titan(?:\b|)" 3; + "~*(?:\b)Toata(?:\b|)" 3; + "~*(?:\b)Toweyabot(?:\b|)" 3; + "~*(?:\b)Tracemyfile(?:\b|)" 3; + "~*(?:\b)Trendiction(?:\b|)" 3; + "~*(?:\b)Trendictionbot(?:\b|)" 3; + "~*(?:\b)trendiction.com(?:\b|)" 3; + "~*(?:\b)trendiction.de(?:\b|)" 3; + "~*(?:\b)True_Robot(?:\b|)" 3; + "~*(?:\b)Turingos(?:\b|)" 3; + "~*(?:\b)Turnitin(?:\b|)" 3; + "~*(?:\b)TurnitinBot(?:\b|)" 3; + "~*(?:\b)TwengaBot(?:\b|)" 3; + "~*(?:\b)Twice(?:\b|)" 3; + "~*(?:\b)Typhoeus(?:\b|)" 3; + "~*(?:\b)UnisterBot(?:\b|)" 3; + "~*(?:\b)Upflow(?:\b|)" 3; + "~*(?:\b)URLy.Warning(?:\b|)" 3; + "~*(?:\b)URLy\ Warning(?:\b|)" 3; + "~*(?:\b)Vacuum(?:\b|)" 3; + "~*(?:\b)Vagabondo(?:\b|)" 3; + "~*(?:\b)VB\ Project(?:\b|)" 3; + "~*(?:\b)VCI(?:\b|)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; + "~*(?:\b)VidibleScraper(?:\b|)" 3; + "~*(?:\b)Virusdie(?:\b|)" 3; + "~*(?:\b)VoidEYE(?:\b|)" 3; + "~*(?:\b)Voil(?:\b|)" 3; + "~*(?:\b)Voltron(?:\b|)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; + "~*(?:\b)WallpapersHD(?:\b|)" 3; + "~*(?:\b)WASALive-Bot(?:\b|)" 3; + "~*(?:\b)WBSearchBot(?:\b|)" 3; + "~*(?:\b)Webalta(?:\b|)" 3; + "~*(?:\b)WebAuto(?:\b|)" 3; + "~*(?:\b)Web\ Auto(?:\b|)" 3; + "~*(?:\b)WebBandit(?:\b|)" 3; + "~*(?:\b)WebCollage(?:\b|)" 3; + "~*(?:\b)Web\ Collage(?:\b|)" 3; + "~*(?:\b)WebCopier(?:\b|)" 3; + "~*(?:\b)WEBDAV(?:\b|)" 3; + "~*(?:\b)WebEnhancer(?:\b|)" 3; + "~*(?:\b)Web\ Enhancer(?:\b|)" 3; + "~*(?:\b)WebFetch(?:\b|)" 3; + "~*(?:\b)Web\ Fetch(?:\b|)" 3; + "~*(?:\b)WebFuck(?:\b|)" 3; + "~*(?:\b)Web\ Fuck(?:\b|)" 3; + "~*(?:\b)WebGo\ IS(?:\b|)" 3; + "~*(?:\b)WebImageCollector(?:\b|)" 3; + "~*(?:\b)WebLeacher(?:\b|)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; + "~*(?:\b)webmeup-crawler(?:\b|)" 3; + "~*(?:\b)WebPix(?:\b|)" 3; + "~*(?:\b)Web\ Pix(?:\b|)" 3; + "~*(?:\b)WebReaper(?:\b|)" 3; + "~*(?:\b)WebSauger(?:\b|)" 3; + "~*(?:\b)Web\ Sauger(?:\b|)" 3; + "~*(?:\b)Webshag(?:\b|)" 3; + "~*(?:\b)WebsiteExtractor(?:\b|)" 3; + "~*(?:\b)WebsiteQuester(?:\b|)" 3; + "~*(?:\b)Website\ Quester(?:\b|)" 3; + "~*(?:\b)Webster(?:\b|)" 3; + "~*(?:\b)WebStripper(?:\b|)" 3; + "~*(?:\b)WebSucker(?:\b|)" 3; + "~*(?:\b)Web\ Sucker(?:\b|)" 3; + "~*(?:\b)WebWhacker(?:\b|)" 3; + "~*(?:\b)WebZIP(?:\b|)" 3; + "~*(?:\b)WeSEE(?:\b|)" 3; + "~*(?:\b)Whack(?:\b|)" 3; + "~*(?:\b)Whacker(?:\b|)" 3; + "~*(?:\b)Whatweb(?:\b|)" 3; + "~*(?:\b)Who.is\ Bot(?:\b|)" 3; + "~*(?:\b)Widow(?:\b|)" 3; + "~*(?:\b)WinHTTrack(?:\b|)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; + "~*(?:\b)WISENutbot(?:\b|)" 3; + "~*(?:\b)Wonderbot(?:\b|)" 3; + "~*(?:\b)Woobot(?:\b|)" 3; + "~*(?:\b)Wotbox(?:\b|)" 3; + "~*(?:\b)Wprecon(?:\b|)" 3; + "~*(?:\b)WPScan(?:\b|)" 3; + "~*(?:\b)WWW-Collector-E(?:\b|)" 3; + "~*(?:\b)WWW-Mechanize(?:\b|)" 3; + "~*(?:\b)WWW::Mechanize(?:\b|)" 3; + "~*(?:\b)WWWOFFLE(?:\b|)" 3; + "~*(?:\b)x09Mozilla(?:\b|)" 3; + "~*(?:\b)x22Mozilla(?:\b|)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; + "~*(?:\b)Xenu(?:\b|)" 3; + "~*(?:\b)xpymep1.exe(?:\b|)" 3; + "~*(?:\b)YoudaoBot(?:\b|)" 3; + "~*(?:\b)Zade(?:\b|)" 3; + "~*(?:\b)Zauba(?:\b|)" 3; + "~*(?:\b)zauba.io(?:\b|)" 3; + "~*(?:\b)Zermelo(?:\b|)" 3; + "~*(?:\b)Zeus(?:\b|)" 3; + "~*(?:\b)zgrab(?:\b|)" 3; + "~*(?:\b)Zitebot(?:\b|)" 3; + "~*(?:\b)ZmEu(?:\b|)" 3; + "~*(?:\b)ZumBot(?:\b|)" 3; + "~*(?:\b)ZyBorg(?:\b|)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # *********************************************** diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index 60944cc49..d39f16e78 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1540 -### Updated: Sun Jun 23 15:20:43 SAST 2019 +### Version: V3.2019.06.1541 +### Updated: Sun Jun 23 16:11:00 SAST 2019 ### Bad Referrer Count: 6693 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index 2bd9722a5..28986df05 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,100 +1,100 @@ -Acunetix -AiHitBot -Aipbot -ASPSeek -autoemailspider -BacklinkCrawler -Bandit -BatchFTP -Black Hole +80legs +Aboundex +Alexibot +Anarchie +BackStreet +Battleztar Bazinga +Bitacle BlackWidow Blow -BlowFish -BunnySlippers -CheTeam -Craftbot +Brandwatch +Bullseye +BuzzSumo +Calculon +ChinaClaw +Chlooe +CopyRightCheck +Copyscape +crawl.sogou.com Crescent -CrunchBot -Discobot -DomainStatsBot -eCatch +DatabaseDriverMysqli +Demon +Devil +DIIbot +EasyDL EirGrabber -EMail Siphon -Exabot -Extractor -ExtractorPro -Ezooms -Gigabot -Grafula -HTMLparser -HTTP::Lite -HTTrack -Id-search -InfoNaviRobot +FHscan +GermCrawler +GetRight +Grabber +Haansoft +HybridBot +IlseBot +IndeedBot instabid -JamesBOT +Jbrofuzz JetCar -Jorgee -Lanshanbot -Libwhisker -Lightspeedsystems -LinkpadBot +Joomla +Keyword Density +LinkScan LinksManager -LinkWalker -Lmspider -LWP::Simple -Mail.RU_Bot -Microsoft Data Access -MIIxpc +Litemage_walker +Ltx71 +Meanpathbot +MFC_Tear_Sample +MIDown tool Mister PiX -MJ12bot -Musobot -Nameprotect -Netvibes -Niki-bot -Ninja -Nutch -Offline Explorer -pcBrowser -PECL::HTTP +Mojeek +Needle +NetLyzer +NetSpider +Nimbostratus +OpenVAS +page scorer PeoplePal -Picsearch -RankActiveLinkBot -RankingBot -RepoMonkey -SalesIntelligent +Pimonster +Ripper ScanAlert -SEOprofiler -SeoSiteCheckup +SEOkicks +Seomoz +seoscanners +sexsearcher +SiteExplorer +SiteSnagger SiteSucker -SMTBot -Snapbot -Snoopy +Site Sucker +Sitevigil +sogouspider Sogou web spider -sp_auditbot -Spbot -SuperBot +Sottopop +Spanner +Spinn3r +Sqworm +SuperHTTP Surfbot -Telesphoreo -Tracemyfile -True_Robot +TeleportPro +The Intraformant +trendiction.com +TurnitinBot Twice -Typhoeus UnisterBot +Vacuum +VeriCiteCrawler +VidibleScraper +Voil WallpapersHD -WASALive-Bot -WBSearchBot WebBandit -WebEnhancer -Web Fuck -WebImageCollector -WebLeacher +WebCopier +WEBDAV +Web Enhancer +WebFuck +Webshag +Website Quester +WebZIP +Whack Who.is Bot -WISENutbot -Wprecon +Wonderbot WPScan WWW-Mechanize -Xenu -xpymep1.exe -YoudaoBot -Zeus +ZumBot +ZyBorg diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index 818705f46..16bfc9ed8 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,200 +1,200 @@ -1adult.com -83net.jp -aasoldes.fr -addblueoff.com.ua -adserver-e7.com -ads-seo.men -agecheckadult.com -alevinefesleri.com -allpdfmags.net -android4fun.org -androids-store.com -anjalika.co.in -apibetweenlinesn-a.akamaihd.net -apimossnetinfo-a.akamaihd.net -aramaicmedia.org -artdeko.info -artdestshop.eu -ausmepa.org.au -best-deals-products.com -bestdraws.com -binomo.com -biznesrealnost.ru -boeuklad.com -bristolhotel.com -burkesales.com -callcafe.info +1000n1.ru +2ads.co.uk +4ur.click +6128786.com +8si.ru +aa81bf391151884adfa3dd677e41f94be1.com +abcdeg.xyz +actionnooz.com +advokateg.xyz +afmuseum.com +a.frcls.fr +agahinameh.com +agapovdg.ru +akuhni.by +alborzan.com +apartment.ru +asrv-a.akamaihd.net +autobrennero.it +autonew.biz +avtolombard-krasnodar.com +ayanaspa.com +bad-stars.net +banan.tv +bannerads.de +bard-real.com.ua +betteroffers.review +bjorkbacken.nu +blacktwhite.com +blog.axant.it +bmwhighperformers.com +bombla.org +brewdom.ru +browsepulse-a.akamaihd.net +budilneg.xyz +bugof.gq +businescoop.men +bvlgariwallet.xyz +bytimedance.ru call-of-duty.info -capacitacionyaprendizaje.com -capture-room.com -cardul.ru -cdnnetwok.xyz -children-learningreading.info -chorus.terakeet.com -cienum.fr -cineacademy.ru -clickhype.com -cmd.kz -continent-e.tv -cookingmeat.ru -cupday.com -dalavia.ru -daooda.com -davebestdeals.com -dealighted.com -demian.kz -destinationrealestate.com -discountbarbour.online -djonwatch.ru -dms-sw.ru -dprtb.com -drhomes.biz -elektrozigarette2.webs.com -elektrozigarette2.wordpressy.pl -elite-sex-finders.com -enews.tech -erotiktreff24.info -etotupo.ru -fantasticpixcool.com -fastcrawl.com -filmetricsasia.com -findthe.pet -fordsonmajbor.cf +cardiosport.com.ua +cenoval.ru +cheapwebsitehoster.com +christianlouboutinreplica.win +ciproandtizanidine.blogspot.com +clickzzs.nl +compliance-julia.top +compliance-willy.top +compucelunlock.net +copblock.org +cottageofgrace.com +cougfan.info +custodieva.ru +custom-product-labels.com +cyclobenzaprinevstizanidine.blogspot.com +damedingel.ya.ru +daretodonate.co +darkages.info +date-withme.com +decorationspcs.com +deda-moroza-zakaz.ru +dengi-pod-zalog-nedvizhimosti.ru +domain-tracker.com +dominterior.org +ekobata.ru +ekspertmed.com +europeanwatches.ru +ez8motelseaworldsandiego.com +flashbannernow.com +floridahuntingfishingadventures.com +fok.nl foto-sisek.porngalleries.top -free-deals.faith -freejabs.com -freshdz.com -freza-sverlo.ru -fullgirl.ru -galaxyflowers.ru -geilehausfrauen.net -geile-lelly.eu -gettpromos.com -ghostvisitor.com -go2album.com -gorabagrata.ru -guitar-master.org -happy.new.yeartwit.com -health-medical-portal.info -hikesearch.net -hiwibyh.bugs3.com -host-tracker.com -howopen.ru -hyipmanager.in -iflycapetown.co.za -imgarcade.com -infogame.name -infowarcraft.ru +frequiry.com +frustrated-favorable.gq +gearsadspromo.club +glicol.kz +gogps.me +goldadpremium.com +gwagka.com +hacktougroup.ru +hamptonoaks.ca +haveinc.xyz +hd-film.pl +homedecoguide.info +homemature.net +horoshieokna.com +hotelcrocenzi.sm +houseofrose.com +hoverboardforsaledirect.com +huimin764128.com +hulfingtonpost.com +i4track.net +ideibiznesa2015.ru +import-sales.com +indiakino.net +infospot.pt +insider.pro internetartfair.com -javatex.co.id -javcoast.com -javitas.info -jbl-charge.info -jyvopys.com -kinoduh.ru -kiwi237au.tk -kmd-pto.ru -kndxbkdx.bloger.index.hr -lafriore.ru -ldrtrack.com -letolove.ru -letsart.ru -life.biz.ua -likesdesign.com -lombia.co -makedo.ru -masserect.com -medi-fitt.hu -meds-online24.com -message-warning.net -metaxalonevstizanidine.blogspot.com -mex-annushka.ru -mhtr.be -moinozhki.com -moncleroutletonline.pw -moyaterapiya.ru -msk-diplomat.com -musicdaddy.net -muztops.ru -mydownloadengine.com -mygameplus.ru -mypets.by -nacap.ru -naval.jislaaik.com -newsperuse.com -nfljerseyscheapchinabiz.com -niki-mlt.ru -nullrefer.com -o333o.com -onlinemeetingnow.com -onlinetvseries.me +investingclub.ru +isotoner.com +iwanttodeliver.com +izismile.com +izoll.ru +jewelryandfiligree.com +jmat.cn +justkillingti.me +jwss.cc +kinomaniatv.pl +kliksaya.com +konoplisemena.com +larose.jb2c.me +larutti.ru +libertybilisim.com +lider-zhaluzi.kiev.ua +lkbennettoutlet.online +lol-smurfs.com +longlifelomilomi.info +matb3aa.com +maxthon.com +mensandals.xyz +microstatic.pl +misswell.net +moneytop.ru +moonci.ru +mossmesi.com +most-kerch.org +mycaf.it +myonlinepayday.co +myseoconsultant.com +narco24.me +nardulan.com +officedocuments.net +onlinemegax.com ons-add.men -o-o-8-o-o.ru -paphoselectricianandplumber.com -pdns.cz -petrovka-online.com -pihl.se -pokemongooo.ml -pornpost.in -powc.r.ca.d.sendibm2.com -proxyelite.biz -rankexperience.com -refudiatethissarah.info -richinvestmonitor.com -ringtonepartner.com -ros-ctm.ru -rutor.group -santasgift.ml -seomarketings.online -seo-traffic-ranking.info -serpstat.com -sethrollins.net -sfd-chess.ru -signoredom.com -site5.com -sixcooler.de -skuteczna-dieta.co.pl -snts.shell-pmr.ru -songoo.wz.cz -sovinsteel.ru +ontargetseo.us +openmultipleurl.com +pacificair.com +pagesense.com +partner-stop.men +pavlodar.xkaz.org +percin.biz.ly +picphotos.net +pix24x7.com +podshipniki-ntn.ru +pornmania.pl +porno-play.net +pr0fit-b0x.com +raavidesigns.com +ranksonic.org +razleton.com +registrysweeper.com +rentehno.ru +ruf777.com +runtnc.net +samoiedo.it +samo-soznanie.ru +sape.top +sashagreyblog.ga +sawin.beth.webd.pl +sdsjweb.com +searchmywindow-a.akamaihd.net +sex-tracker.com +sexyteens.hol.es +sigmund-freud.co.uk +slavkokacunko.de special-porn.com -sstroy44.ru -staff.prairiesouth.ca -strag-invest.ru -stroiminsk.com -styro.ru -susanholtphotography.com -teesdaleflyballclub.co.uk -thewebsitetemplate.info -thiegs.reco.ws -tizanidineoral.blogspot.com +speechfoodie.com +startwp.org +stoki.ru +svetodiodoff.ru +swinginwithme.ru +terraclicks.com +tetracsaudi.com +thefotosgratis.eu tmearegion26.com -trudogolik.net -try-rx.com -uk-zheu20.ru +top1-seo-service.com +topdownloads.ru +top-l2.com +trafficgenius.xyz +tramadolandtizanidine.blogspot.com +treasuretrack-a.akamaihd.net +tuckermktg.com +twincitiescarservice.com +ugguk.online ural-buldozer.ru -urlcut.ru -usiad.net -vip-dom.in -volume-pills.biz -w3javascript.com -wait3sec.org -webscouter.net -weburlopener.com -wesharepics.com -wheelchairliftsarea.com -wineitudes.wordpress.com -wma-x.com -worldoffiles.ru -xn--80aafb2a.xn--p1ai -xn--80ahvj9e.xn--p1ai -xn--d1aifoe0a9a.top -xtraffic.plus -xxxrus.org -yaoguangdj.com -youandcredit.ru -zagadki.in.ua -zaloro.com -zambini.ru -zapatosenventa.info -zhorapankratov7.blogspot.com +vandrie-ict.nl +vykupavto-krasnodar.ru +web-betting.ru +websiteaccountant.de +weburok.com +wejdz-tu.pl +windowssearch-exp.com +wjgony.com +wowas31.ucoz.ru +www.arenda-yeisk.ru +xlolitka.com +xn----itbkqkfiq.xn--p1ai +x-rates.ru +yougotanewdomain.com +zaim-pod-zalog-krasnodar.ru +zetgie.com.pl diff --git a/.latest_release/bots.d.tar.gz b/.latest_release/bots.d.tar.gz index 2629c5286..999ce1833 100644 Binary files a/.latest_release/bots.d.tar.gz and b/.latest_release/bots.d.tar.gz differ diff --git a/.latest_release/conf.d.tar.gz b/.latest_release/conf.d.tar.gz index 72f334e8e..a4027c8e7 100644 Binary files a/.latest_release/conf.d.tar.gz and b/.latest_release/conf.d.tar.gz differ diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index e887a68df..6ed8693c7 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V3.2019.06.1540 +#### Version: V3.2019.06.1541 #### Bad Referrer Count: 6693 #### Bad Bot Count: 556 ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 93f201fc7..7470afd75 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V3.2019.06.1540 +#### Version: V3.2019.06.1541 #### Bad Referrer Count: 6693 #### Bad Bot Count: 556 ____________________ diff --git a/README.md b/README.md index bb19ec78d..f45f0c268 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V3.2019.06.1540 +#### Version: V3.2019.06.1541 #### Bad Referrer Count: 6693 #### Bad Bot Count: 556 ____________________ diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 88fc59ccd..0089e3ee7 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V3.2019.06.1540 +#### Version: V3.2019.06.1541 #### Bad Referrer Count: 6693 #### Bad Bot Count: 556 ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 60944cc49..d39f16e78 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1540 -### Updated: Sun Jun 23 15:20:43 SAST 2019 +### Version: V3.2019.06.1541 +### Updated: Sun Jun 23 16:11:00 SAST 2019 ### Bad Referrer Count: 6693 ### Bad Bot Count: 556 ################################################### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 18e5b9d75..a9e0ee294 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -10,8 +10,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1540 -### Updated: Sun Jun 23 15:20:44 SAST 2019 +### Version: V3.2019.06.1541 +### Updated: Sun Jun 23 16:11:01 SAST 2019 ### Bad Referrer Count: 6693 ### Bad Bot Count: 556 ################################################### @@ -137,562 +137,562 @@ map $http_user_agent $bad_bot { # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b)" 3; - "~*(?:\b)404checker(?:\b)" 3; - "~*(?:\b)404enemy(?:\b)" 3; - "~*(?:\b)80legs(?:\b)" 3; - "~*(?:\b)Abonti(?:\b)" 3; - "~*(?:\b)Aboundex(?:\b)" 3; - "~*(?:\b)Aboundexbot(?:\b)" 3; - "~*(?:\b)Acunetix(?:\b)" 3; - "~*(?:\b)ADmantX(?:\b)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; - "~*(?:\b)AhrefsBot(?:\b)" 3; - "~*(?:\b)AIBOT(?:\b)" 3; - "~*(?:\b)AiHitBot(?:\b)" 3; - "~*(?:\b)Aipbot(?:\b)" 3; - "~*(?:\b)Alexibot(?:\b)" 3; - "~*(?:\b)Alligator(?:\b)" 3; - "~*(?:\b)AllSubmitter(?:\b)" 3; - "~*(?:\b)AlphaBot(?:\b)" 3; - "~*(?:\b)Anarchie(?:\b)" 3; - "~*(?:\b)Apexoo(?:\b)" 3; - "~*(?:\b)archive.org_bot(?:\b)" 3; - "~*(?:\b)ASPSeek(?:\b)" 3; - "~*(?:\b)Asterias(?:\b)" 3; - "~*(?:\b)Attach(?:\b)" 3; - "~*(?:\b)autoemailspider(?:\b)" 3; - "~*(?:\b)BackDoorBot(?:\b)" 3; - "~*(?:\b)Backlink-Ceck(?:\b)" 3; - "~*(?:\b)backlink-check(?:\b)" 3; - "~*(?:\b)BacklinkCrawler(?:\b)" 3; - "~*(?:\b)BackStreet(?:\b)" 3; - "~*(?:\b)BackWeb(?:\b)" 3; - "~*(?:\b)Badass(?:\b)" 3; - "~*(?:\b)Bandit(?:\b)" 3; - "~*(?:\b)Barkrowler(?:\b)" 3; - "~*(?:\b)BatchFTP(?:\b)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; - "~*(?:\b)BBBike(?:\b)" 3; - "~*(?:\b)BDCbot(?:\b)" 3; - "~*(?:\b)BDFetch(?:\b)" 3; - "~*(?:\b)BetaBot(?:\b)" 3; - "~*(?:\b)Bigfoot(?:\b)" 3; - "~*(?:\b)Bitacle(?:\b)" 3; - "~*(?:\b)Blackboard(?:\b)" 3; - "~*(?:\b)Black\ Hole(?:\b)" 3; - "~*(?:\b)BlackWidow(?:\b)" 3; - "~*(?:\b)BLEXBot(?:\b)" 3; - "~*(?:\b)Blow(?:\b)" 3; - "~*(?:\b)BlowFish(?:\b)" 3; - "~*(?:\b)Boardreader(?:\b)" 3; - "~*(?:\b)Bolt(?:\b)" 3; - "~*(?:\b)BotALot(?:\b)" 3; - "~*(?:\b)Brandprotect(?:\b)" 3; - "~*(?:\b)Brandwatch(?:\b)" 3; - "~*(?:\b)Buddy(?:\b)" 3; - "~*(?:\b)BuiltBotTough(?:\b)" 3; - "~*(?:\b)BuiltWith(?:\b)" 3; - "~*(?:\b)Bullseye(?:\b)" 3; - "~*(?:\b)BunnySlippers(?:\b)" 3; - "~*(?:\b)BuzzSumo(?:\b)" 3; - "~*(?:\b)Calculon(?:\b)" 3; - "~*(?:\b)CATExplorador(?:\b)" 3; - "~*(?:\b)CazoodleBot(?:\b)" 3; - "~*(?:\b)CCBot(?:\b)" 3; - "~*(?:\b)Cegbfeieh(?:\b)" 3; - "~*(?:\b)CheeseBot(?:\b)" 3; - "~*(?:\b)CherryPicker(?:\b)" 3; - "~*(?:\b)CheTeam(?:\b)" 3; - "~*(?:\b)ChinaClaw(?:\b)" 3; - "~*(?:\b)Chlooe(?:\b)" 3; - "~*(?:\b)Claritybot(?:\b)" 3; - "~*(?:\b)Cliqzbot(?:\b)" 3; - "~*(?:\b)Cloud\ mapping(?:\b)" 3; - "~*(?:\b)coccocbot-web(?:\b)" 3; - "~*(?:\b)Cogentbot(?:\b)" 3; - "~*(?:\b)cognitiveseo(?:\b)" 3; - "~*(?:\b)Collector(?:\b)" 3; - "~*(?:\b)com.plumanalytics(?:\b)" 3; - "~*(?:\b)Copier(?:\b)" 3; - "~*(?:\b)CopyRightCheck(?:\b)" 3; - "~*(?:\b)Copyscape(?:\b)" 3; - "~*(?:\b)Cosmos(?:\b)" 3; - "~*(?:\b)Craftbot(?:\b)" 3; - "~*(?:\b)crawler4j(?:\b)" 3; - "~*(?:\b)crawler.feedback(?:\b)" 3; - "~*(?:\b)crawl.sogou.com(?:\b)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b)" 3; - "~*(?:\b)Crescent(?:\b)" 3; - "~*(?:\b)CrunchBot(?:\b)" 3; - "~*(?:\b)CSHttp(?:\b)" 3; - "~*(?:\b)Curious(?:\b)" 3; - "~*(?:\b)Custo(?:\b)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; - "~*(?:\b)DataCha0s(?:\b)" 3; - "~*(?:\b)DBLBot(?:\b)" 3; - "~*(?:\b)demandbase-bot(?:\b)" 3; - "~*(?:\b)Demon(?:\b)" 3; - "~*(?:\b)Deusu(?:\b)" 3; - "~*(?:\b)Devil(?:\b)" 3; - "~*(?:\b)Digincore(?:\b)" 3; - "~*(?:\b)DigitalPebble(?:\b)" 3; - "~*(?:\b)DIIbot(?:\b)" 3; - "~*(?:\b)Dirbuster(?:\b)" 3; - "~*(?:\b)Disco(?:\b)" 3; - "~*(?:\b)Discobot(?:\b)" 3; - "~*(?:\b)Discoverybot(?:\b)" 3; - "~*(?:\b)Dispatch(?:\b)" 3; - "~*(?:\b)DittoSpyder(?:\b)" 3; - "~*(?:\b)DnyzBot(?:\b)" 3; - "~*(?:\b)DomainAppender(?:\b)" 3; - "~*(?:\b)DomainCrawler(?:\b)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; - "~*(?:\b)DomainStatsBot(?:\b)" 3; - "~*(?:\b)Dotbot(?:\b)" 3; - "~*(?:\b)Download\ Wonder(?:\b)" 3; - "~*(?:\b)Dragonfly(?:\b)" 3; - "~*(?:\b)Drip(?:\b)" 3; - "~*(?:\b)DSearch(?:\b)" 3; - "~*(?:\b)DTS\ Agent(?:\b)" 3; - "~*(?:\b)EasyDL(?:\b)" 3; - "~*(?:\b)Ebingbong(?:\b)" 3; - "~*(?:\b)eCatch(?:\b)" 3; - "~*(?:\b)ECCP/1.0(?:\b)" 3; - "~*(?:\b)Ecxi(?:\b)" 3; - "~*(?:\b)EirGrabber(?:\b)" 3; - "~*(?:\b)EMail\ Siphon(?:\b)" 3; - "~*(?:\b)EMail\ Wolf(?:\b)" 3; - "~*(?:\b)EroCrawler(?:\b)" 3; - "~*(?:\b)evc-batch(?:\b)" 3; - "~*(?:\b)Evil(?:\b)" 3; - "~*(?:\b)Exabot(?:\b)" 3; - "~*(?:\b)Express\ WebPictures(?:\b)" 3; - "~*(?:\b)ExtLinksBot(?:\b)" 3; - "~*(?:\b)Extractor(?:\b)" 3; - "~*(?:\b)ExtractorPro(?:\b)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; - "~*(?:\b)EyeNetIE(?:\b)" 3; - "~*(?:\b)Ezooms(?:\b)" 3; - "~*(?:\b)facebookscraper(?:\b)" 3; - "~*(?:\b)FDM(?:\b)" 3; - "~*(?:\b)FemtosearchBot(?:\b)" 3; - "~*(?:\b)FHscan(?:\b)" 3; - "~*(?:\b)Fimap(?:\b)" 3; - "~*(?:\b)Firefox/7.0(?:\b)" 3; - "~*(?:\b)FlashGet(?:\b)" 3; - "~*(?:\b)Flunky(?:\b)" 3; - "~*(?:\b)Foobot(?:\b)" 3; - "~*(?:\b)Freeuploader(?:\b)" 3; - "~*(?:\b)FrontPage(?:\b)" 3; - "~*(?:\b)FyberSpider(?:\b)" 3; - "~*(?:\b)Fyrebot(?:\b)" 3; - "~*(?:\b)GalaxyBot(?:\b)" 3; - "~*(?:\b)Genieo(?:\b)" 3; - "~*(?:\b)GermCrawler(?:\b)" 3; - "~*(?:\b)Getintent(?:\b)" 3; - "~*(?:\b)GetRight(?:\b)" 3; - "~*(?:\b)GetWeb(?:\b)" 3; - "~*(?:\b)Gigablast(?:\b)" 3; - "~*(?:\b)Gigabot(?:\b)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; - "~*(?:\b)Gotit(?:\b)" 3; - "~*(?:\b)GoZilla(?:\b)" 3; - "~*(?:\b)Go!Zilla(?:\b)" 3; - "~*(?:\b)Grabber(?:\b)" 3; - "~*(?:\b)GrabNet(?:\b)" 3; - "~*(?:\b)Grafula(?:\b)" 3; - "~*(?:\b)GrapeFX(?:\b)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b)" 3; - "~*(?:\b)GridBot(?:\b)" 3; - "~*(?:\b)GT::WWW(?:\b)" 3; - "~*(?:\b)Haansoft(?:\b)" 3; - "~*(?:\b)HaosouSpider(?:\b)" 3; - "~*(?:\b)Harvest(?:\b)" 3; - "~*(?:\b)Havij(?:\b)" 3; - "~*(?:\b)HEADMasterSEO(?:\b)" 3; - "~*(?:\b)Heritrix(?:\b)" 3; - "~*(?:\b)Hloader(?:\b)" 3; - "~*(?:\b)HMView(?:\b)" 3; - "~*(?:\b)HTMLparser(?:\b)" 3; - "~*(?:\b)HTTP::Lite(?:\b)" 3; - "~*(?:\b)HTTrack(?:\b)" 3; - "~*(?:\b)Humanlinks(?:\b)" 3; - "~*(?:\b)HybridBot(?:\b)" 3; - "~*(?:\b)Iblog(?:\b)" 3; - "~*(?:\b)IDBot(?:\b)" 3; - "~*(?:\b)Id-search(?:\b)" 3; - "~*(?:\b)IlseBot(?:\b)" 3; - "~*(?:\b)Image\ Fetch(?:\b)" 3; - "~*(?:\b)Image\ Sucker(?:\b)" 3; - "~*(?:\b)IndeedBot(?:\b)" 3; - "~*(?:\b)Indy\ Library(?:\b)" 3; - "~*(?:\b)InfoNaviRobot(?:\b)" 3; - "~*(?:\b)InfoTekies(?:\b)" 3; - "~*(?:\b)instabid(?:\b)" 3; - "~*(?:\b)Intelliseek(?:\b)" 3; - "~*(?:\b)InterGET(?:\b)" 3; - "~*(?:\b)Internet\ Ninja(?:\b)" 3; - "~*(?:\b)InternetSeer(?:\b)" 3; - "~*(?:\b)internetVista\ monitor(?:\b)" 3; - "~*(?:\b)ips-agent(?:\b)" 3; - "~*(?:\b)Iria(?:\b)" 3; - "~*(?:\b)IRLbot(?:\b)" 3; - "~*(?:\b)Iskanie(?:\b)" 3; - "~*(?:\b)IstellaBot(?:\b)" 3; - "~*(?:\b)JamesBOT(?:\b)" 3; - "~*(?:\b)Jbrofuzz(?:\b)" 3; - "~*(?:\b)JennyBot(?:\b)" 3; - "~*(?:\b)JetCar(?:\b)" 3; - "~*(?:\b)Jetty(?:\b)" 3; - "~*(?:\b)JikeSpider(?:\b)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; - "~*(?:\b)Joomla(?:\b)" 3; - "~*(?:\b)Jorgee(?:\b)" 3; - "~*(?:\b)JustView(?:\b)" 3; - "~*(?:\b)Jyxobot(?:\b)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b)" 3; - "~*(?:\b)Keyword\ Density(?:\b)" 3; - "~*(?:\b)Kozmosbot(?:\b)" 3; - "~*(?:\b)Lanshanbot(?:\b)" 3; - "~*(?:\b)Larbin(?:\b)" 3; - "~*(?:\b)LeechFTP(?:\b)" 3; - "~*(?:\b)LeechGet(?:\b)" 3; - "~*(?:\b)LexiBot(?:\b)" 3; - "~*(?:\b)Lftp(?:\b)" 3; - "~*(?:\b)LibWeb(?:\b)" 3; - "~*(?:\b)Libwhisker(?:\b)" 3; - "~*(?:\b)Lightspeedsystems(?:\b)" 3; - "~*(?:\b)Likse(?:\b)" 3; - "~*(?:\b)Linkdexbot(?:\b)" 3; - "~*(?:\b)LinkextractorPro(?:\b)" 3; - "~*(?:\b)LinkpadBot(?:\b)" 3; - "~*(?:\b)LinkScan(?:\b)" 3; - "~*(?:\b)LinksManager(?:\b)" 3; - "~*(?:\b)LinkWalker(?:\b)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; - "~*(?:\b)Lipperhey(?:\b)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; - "~*(?:\b)Litemage_walker(?:\b)" 3; - "~*(?:\b)Lmspider(?:\b)" 3; - "~*(?:\b)LNSpiderguy(?:\b)" 3; - "~*(?:\b)Ltx71(?:\b)" 3; - "~*(?:\b)lwp-request(?:\b)" 3; - "~*(?:\b)LWP::Simple(?:\b)" 3; - "~*(?:\b)lwp-trivial(?:\b)" 3; - "~*(?:\b)Magnet(?:\b)" 3; - "~*(?:\b)Mag-Net(?:\b)" 3; - "~*(?:\b)magpie-crawler(?:\b)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b)" 3; - "~*(?:\b)Majestic12(?:\b)" 3; - "~*(?:\b)Majestic-SEO(?:\b)" 3; - "~*(?:\b)Majestic\ SEO(?:\b)" 3; - "~*(?:\b)MarkMonitor(?:\b)" 3; - "~*(?:\b)MarkWatch(?:\b)" 3; - "~*(?:\b)Masscan(?:\b)" 3; - "~*(?:\b)Mass\ Downloader(?:\b)" 3; - "~*(?:\b)Mata\ Hari(?:\b)" 3; - "~*(?:\b)MauiBot(?:\b)" 3; - "~*(?:\b)meanpathbot(?:\b)" 3; - "~*(?:\b)Meanpathbot(?:\b)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b)" 3; - "~*(?:\b)mediawords(?:\b)" 3; - "~*(?:\b)MegaIndex.ru(?:\b)" 3; - "~*(?:\b)Metauri(?:\b)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; - "~*(?:\b)MIDown\ tool(?:\b)" 3; - "~*(?:\b)MIIxpc(?:\b)" 3; - "~*(?:\b)Mister\ PiX(?:\b)" 3; - "~*(?:\b)MJ12bot(?:\b)" 3; - "~*(?:\b)Mojeek(?:\b)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; - "~*(?:\b)Mr.4x3(?:\b)" 3; - "~*(?:\b)MSFrontPage(?:\b)" 3; - "~*(?:\b)MSIECrawler(?:\b)" 3; - "~*(?:\b)Msrabot(?:\b)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; - "~*(?:\b)muhstik-scan(?:\b)" 3; - "~*(?:\b)Musobot(?:\b)" 3; - "~*(?:\b)Name\ Intelligence(?:\b)" 3; - "~*(?:\b)Nameprotect(?:\b)" 3; - "~*(?:\b)Navroad(?:\b)" 3; - "~*(?:\b)NearSite(?:\b)" 3; - "~*(?:\b)Needle(?:\b)" 3; - "~*(?:\b)Nessus(?:\b)" 3; - "~*(?:\b)NetAnts(?:\b)" 3; - "~*(?:\b)Netcraft(?:\b)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; - "~*(?:\b)NetLyzer(?:\b)" 3; - "~*(?:\b)NetMechanic(?:\b)" 3; - "~*(?:\b)NetSpider(?:\b)" 3; - "~*(?:\b)Nettrack(?:\b)" 3; - "~*(?:\b)Net\ Vampire(?:\b)" 3; - "~*(?:\b)Netvibes(?:\b)" 3; - "~*(?:\b)NetZIP(?:\b)" 3; - "~*(?:\b)NextGenSearchBot(?:\b)" 3; - "~*(?:\b)Nibbler(?:\b)" 3; - "~*(?:\b)NICErsPRO(?:\b)" 3; - "~*(?:\b)Niki-bot(?:\b)" 3; - "~*(?:\b)Nikto(?:\b)" 3; - "~*(?:\b)NimbleCrawler(?:\b)" 3; - "~*(?:\b)Nimbostratus(?:\b)" 3; - "~*(?:\b)Ninja(?:\b)" 3; - "~*(?:\b)Nmap(?:\b)" 3; - "~*(?:\b)NPbot(?:\b)" 3; - "~*(?:\b)Nutch(?:\b)" 3; - "~*(?:\b)oBot(?:\b)" 3; - "~*(?:\b)Octopus(?:\b)" 3; - "~*(?:\b)Offline\ Explorer(?:\b)" 3; - "~*(?:\b)Offline\ Navigator(?:\b)" 3; - "~*(?:\b)OnCrawl(?:\b)" 3; - "~*(?:\b)Openfind(?:\b)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b)" 3; - "~*(?:\b)Openvas(?:\b)" 3; - "~*(?:\b)OpenVAS(?:\b)" 3; - "~*(?:\b)OrangeBot(?:\b)" 3; - "~*(?:\b)OrangeSpider(?:\b)" 3; - "~*(?:\b)OutclicksBot(?:\b)" 3; - "~*(?:\b)OutfoxBot(?:\b)" 3; - "~*(?:\b)PageAnalyzer(?:\b)" 3; - "~*(?:\b)Page\ Analyzer(?:\b)" 3; - "~*(?:\b)PageGrabber(?:\b)" 3; - "~*(?:\b)page\ scorer(?:\b)" 3; - "~*(?:\b)PageScorer(?:\b)" 3; - "~*(?:\b)Pandalytics(?:\b)" 3; - "~*(?:\b)Panscient(?:\b)" 3; - "~*(?:\b)Papa\ Foto(?:\b)" 3; - "~*(?:\b)Pavuk(?:\b)" 3; - "~*(?:\b)pcBrowser(?:\b)" 3; - "~*(?:\b)PECL::HTTP(?:\b)" 3; - "~*(?:\b)PeoplePal(?:\b)" 3; - "~*(?:\b)PHPCrawl(?:\b)" 3; - "~*(?:\b)Picscout(?:\b)" 3; - "~*(?:\b)Picsearch(?:\b)" 3; - "~*(?:\b)PictureFinder(?:\b)" 3; - "~*(?:\b)Pimonster(?:\b)" 3; - "~*(?:\b)Pi-Monster(?:\b)" 3; - "~*(?:\b)Pixray(?:\b)" 3; - "~*(?:\b)PleaseCrawl(?:\b)" 3; - "~*(?:\b)plumanalytics(?:\b)" 3; - "~*(?:\b)Pockey(?:\b)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; - "~*(?:\b)Probethenet(?:\b)" 3; - "~*(?:\b)ProPowerBot(?:\b)" 3; - "~*(?:\b)ProWebWalker(?:\b)" 3; - "~*(?:\b)Psbot(?:\b)" 3; - "~*(?:\b)Pump(?:\b)" 3; - "~*(?:\b)PxBroker(?:\b)" 3; - "~*(?:\b)PyCurl(?:\b)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; - "~*(?:\b)Quick-Crawler(?:\b)" 3; - "~*(?:\b)RankActive(?:\b)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b)" 3; - "~*(?:\b)RankFlex(?:\b)" 3; - "~*(?:\b)RankingBot(?:\b)" 3; - "~*(?:\b)RankingBot2(?:\b)" 3; - "~*(?:\b)Rankivabot(?:\b)" 3; - "~*(?:\b)RankurBot(?:\b)" 3; - "~*(?:\b)RealDownload(?:\b)" 3; - "~*(?:\b)Reaper(?:\b)" 3; - "~*(?:\b)RebelMouse(?:\b)" 3; - "~*(?:\b)Recorder(?:\b)" 3; - "~*(?:\b)RedesScrapy(?:\b)" 3; - "~*(?:\b)ReGet(?:\b)" 3; - "~*(?:\b)RepoMonkey(?:\b)" 3; - "~*(?:\b)Ripper(?:\b)" 3; - "~*(?:\b)RocketCrawler(?:\b)" 3; - "~*(?:\b)Rogerbot(?:\b)" 3; - "~*(?:\b)s1z.ru(?:\b)" 3; - "~*(?:\b)SalesIntelligent(?:\b)" 3; - "~*(?:\b)SBIder(?:\b)" 3; - "~*(?:\b)ScanAlert(?:\b)" 3; - "~*(?:\b)Scanbot(?:\b)" 3; - "~*(?:\b)scan.lol(?:\b)" 3; - "~*(?:\b)ScoutJet(?:\b)" 3; - "~*(?:\b)Scrapy(?:\b)" 3; - "~*(?:\b)Screaming(?:\b)" 3; - "~*(?:\b)ScreenerBot(?:\b)" 3; - "~*(?:\b)Searchestate(?:\b)" 3; - "~*(?:\b)SearchmetricsBot(?:\b)" 3; - "~*(?:\b)Semrush(?:\b)" 3; - "~*(?:\b)SemrushBot(?:\b)" 3; - "~*(?:\b)SEOkicks(?:\b)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; - "~*(?:\b)Seomoz(?:\b)" 3; - "~*(?:\b)SEOprofiler(?:\b)" 3; - "~*(?:\b)seoscanners(?:\b)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b)" 3; - "~*(?:\b)SEOstats(?:\b)" 3; - "~*(?:\b)serpstatbot(?:\b)" 3; - "~*(?:\b)sexsearcher(?:\b)" 3; - "~*(?:\b)Shodan(?:\b)" 3; - "~*(?:\b)Siphon(?:\b)" 3; - "~*(?:\b)SISTRIX(?:\b)" 3; - "~*(?:\b)Sitebeam(?:\b)" 3; - "~*(?:\b)SiteExplorer(?:\b)" 3; - "~*(?:\b)Siteimprove(?:\b)" 3; - "~*(?:\b)SiteLockSpider(?:\b)" 3; - "~*(?:\b)SiteSnagger(?:\b)" 3; - "~*(?:\b)SiteSucker(?:\b)" 3; - "~*(?:\b)Site\ Sucker(?:\b)" 3; - "~*(?:\b)Sitevigil(?:\b)" 3; - "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 3; - "~*(?:\b)SlySearch(?:\b)" 3; - "~*(?:\b)SmartDownload(?:\b)" 3; - "~*(?:\b)SMTBot(?:\b)" 3; - "~*(?:\b)Snake(?:\b)" 3; - "~*(?:\b)Snapbot(?:\b)" 3; - "~*(?:\b)Snoopy(?:\b)" 3; - "~*(?:\b)SocialRankIOBot(?:\b)" 3; - "~*(?:\b)Sociscraper(?:\b)" 3; - "~*(?:\b)sogouspider(?:\b)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; - "~*(?:\b)Sosospider(?:\b)" 3; - "~*(?:\b)Sottopop(?:\b)" 3; - "~*(?:\b)SpaceBison(?:\b)" 3; - "~*(?:\b)Spammen(?:\b)" 3; - "~*(?:\b)SpankBot(?:\b)" 3; - "~*(?:\b)Spanner(?:\b)" 3; - "~*(?:\b)sp_auditbot(?:\b)" 3; - "~*(?:\b)Spbot(?:\b)" 3; - "~*(?:\b)Spinn3r(?:\b)" 3; - "~*(?:\b)SputnikBot(?:\b)" 3; - "~*(?:\b)spyfu(?:\b)" 3; - "~*(?:\b)Sqlmap(?:\b)" 3; - "~*(?:\b)Sqlworm(?:\b)" 3; - "~*(?:\b)Sqworm(?:\b)" 3; - "~*(?:\b)Steeler(?:\b)" 3; - "~*(?:\b)Stripper(?:\b)" 3; - "~*(?:\b)Sucker(?:\b)" 3; - "~*(?:\b)Sucuri(?:\b)" 3; - "~*(?:\b)SuperBot(?:\b)" 3; - "~*(?:\b)SuperHTTP(?:\b)" 3; - "~*(?:\b)Surfbot(?:\b)" 3; - "~*(?:\b)SurveyBot(?:\b)" 3; - "~*(?:\b)Suzuran(?:\b)" 3; - "~*(?:\b)Swiftbot(?:\b)" 3; - "~*(?:\b)sysscan(?:\b)" 3; - "~*(?:\b)Szukacz(?:\b)" 3; - "~*(?:\b)T0PHackTeam(?:\b)" 3; - "~*(?:\b)T8Abot(?:\b)" 3; - "~*(?:\b)tAkeOut(?:\b)" 3; - "~*(?:\b)Teleport(?:\b)" 3; - "~*(?:\b)TeleportPro(?:\b)" 3; - "~*(?:\b)Telesoft(?:\b)" 3; - "~*(?:\b)Telesphoreo(?:\b)" 3; - "~*(?:\b)Telesphorep(?:\b)" 3; - "~*(?:\b)The\ Intraformant(?:\b)" 3; - "~*(?:\b)TheNomad(?:\b)" 3; - "~*(?:\b)TightTwatBot(?:\b)" 3; - "~*(?:\b)Titan(?:\b)" 3; - "~*(?:\b)Toata(?:\b)" 3; - "~*(?:\b)Toweyabot(?:\b)" 3; - "~*(?:\b)Tracemyfile(?:\b)" 3; - "~*(?:\b)Trendiction(?:\b)" 3; - "~*(?:\b)Trendictionbot(?:\b)" 3; - "~*(?:\b)trendiction.com(?:\b)" 3; - "~*(?:\b)trendiction.de(?:\b)" 3; - "~*(?:\b)True_Robot(?:\b)" 3; - "~*(?:\b)Turingos(?:\b)" 3; - "~*(?:\b)Turnitin(?:\b)" 3; - "~*(?:\b)TurnitinBot(?:\b)" 3; - "~*(?:\b)TwengaBot(?:\b)" 3; - "~*(?:\b)Twice(?:\b)" 3; - "~*(?:\b)Typhoeus(?:\b)" 3; - "~*(?:\b)UnisterBot(?:\b)" 3; - "~*(?:\b)Upflow(?:\b)" 3; - "~*(?:\b)URLy.Warning(?:\b)" 3; - "~*(?:\b)URLy\ Warning(?:\b)" 3; - "~*(?:\b)Vacuum(?:\b)" 3; - "~*(?:\b)Vagabondo(?:\b)" 3; - "~*(?:\b)VB\ Project(?:\b)" 3; - "~*(?:\b)VCI(?:\b)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b)" 3; - "~*(?:\b)VidibleScraper(?:\b)" 3; - "~*(?:\b)Virusdie(?:\b)" 3; - "~*(?:\b)VoidEYE(?:\b)" 3; - "~*(?:\b)Voil(?:\b)" 3; - "~*(?:\b)Voltron(?:\b)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b)" 3; - "~*(?:\b)WallpapersHD(?:\b)" 3; - "~*(?:\b)WASALive-Bot(?:\b)" 3; - "~*(?:\b)WBSearchBot(?:\b)" 3; - "~*(?:\b)Webalta(?:\b)" 3; - "~*(?:\b)WebAuto(?:\b)" 3; - "~*(?:\b)Web\ Auto(?:\b)" 3; - "~*(?:\b)WebBandit(?:\b)" 3; - "~*(?:\b)WebCollage(?:\b)" 3; - "~*(?:\b)Web\ Collage(?:\b)" 3; - "~*(?:\b)WebCopier(?:\b)" 3; - "~*(?:\b)WEBDAV(?:\b)" 3; - "~*(?:\b)WebEnhancer(?:\b)" 3; - "~*(?:\b)Web\ Enhancer(?:\b)" 3; - "~*(?:\b)WebFetch(?:\b)" 3; - "~*(?:\b)Web\ Fetch(?:\b)" 3; - "~*(?:\b)WebFuck(?:\b)" 3; - "~*(?:\b)Web\ Fuck(?:\b)" 3; - "~*(?:\b)WebGo\ IS(?:\b)" 3; - "~*(?:\b)WebImageCollector(?:\b)" 3; - "~*(?:\b)WebLeacher(?:\b)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; - "~*(?:\b)webmeup-crawler(?:\b)" 3; - "~*(?:\b)WebPix(?:\b)" 3; - "~*(?:\b)Web\ Pix(?:\b)" 3; - "~*(?:\b)WebReaper(?:\b)" 3; - "~*(?:\b)WebSauger(?:\b)" 3; - "~*(?:\b)Web\ Sauger(?:\b)" 3; - "~*(?:\b)Webshag(?:\b)" 3; - "~*(?:\b)WebsiteExtractor(?:\b)" 3; - "~*(?:\b)WebsiteQuester(?:\b)" 3; - "~*(?:\b)Website\ Quester(?:\b)" 3; - "~*(?:\b)Webster(?:\b)" 3; - "~*(?:\b)WebStripper(?:\b)" 3; - "~*(?:\b)WebSucker(?:\b)" 3; - "~*(?:\b)Web\ Sucker(?:\b)" 3; - "~*(?:\b)WebWhacker(?:\b)" 3; - "~*(?:\b)WebZIP(?:\b)" 3; - "~*(?:\b)WeSEE(?:\b)" 3; - "~*(?:\b)Whack(?:\b)" 3; - "~*(?:\b)Whacker(?:\b)" 3; - "~*(?:\b)Whatweb(?:\b)" 3; - "~*(?:\b)Who.is\ Bot(?:\b)" 3; - "~*(?:\b)Widow(?:\b)" 3; - "~*(?:\b)WinHTTrack(?:\b)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; - "~*(?:\b)WISENutbot(?:\b)" 3; - "~*(?:\b)Wonderbot(?:\b)" 3; - "~*(?:\b)Woobot(?:\b)" 3; - "~*(?:\b)Wotbox(?:\b)" 3; - "~*(?:\b)Wprecon(?:\b)" 3; - "~*(?:\b)WPScan(?:\b)" 3; - "~*(?:\b)WWW-Collector-E(?:\b)" 3; - "~*(?:\b)WWW-Mechanize(?:\b)" 3; - "~*(?:\b)WWW::Mechanize(?:\b)" 3; - "~*(?:\b)WWWOFFLE(?:\b)" 3; - "~*(?:\b)x09Mozilla(?:\b)" 3; - "~*(?:\b)x22Mozilla(?:\b)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; - "~*(?:\b)Xenu(?:\b)" 3; - "~*(?:\b)xpymep1.exe(?:\b)" 3; - "~*(?:\b)YoudaoBot(?:\b)" 3; - "~*(?:\b)Zade(?:\b)" 3; - "~*(?:\b)Zauba(?:\b)" 3; - "~*(?:\b)zauba.io(?:\b)" 3; - "~*(?:\b)Zermelo(?:\b)" 3; - "~*(?:\b)Zeus(?:\b)" 3; - "~*(?:\b)zgrab(?:\b)" 3; - "~*(?:\b)Zitebot(?:\b)" 3; - "~*(?:\b)ZmEu(?:\b)" 3; - "~*(?:\b)ZumBot(?:\b)" 3; - "~*(?:\b)ZyBorg(?:\b)" 3; + "~*(?:\b)360Spider(?:\b|)" 3; + "~*(?:\b)404checker(?:\b|)" 3; + "~*(?:\b)404enemy(?:\b|)" 3; + "~*(?:\b)80legs(?:\b|)" 3; + "~*(?:\b)Abonti(?:\b|)" 3; + "~*(?:\b)Aboundex(?:\b|)" 3; + "~*(?:\b)Aboundexbot(?:\b|)" 3; + "~*(?:\b)Acunetix(?:\b|)" 3; + "~*(?:\b)ADmantX(?:\b|)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; + "~*(?:\b)AhrefsBot(?:\b|)" 3; + "~*(?:\b)AIBOT(?:\b|)" 3; + "~*(?:\b)AiHitBot(?:\b|)" 3; + "~*(?:\b)Aipbot(?:\b|)" 3; + "~*(?:\b)Alexibot(?:\b|)" 3; + "~*(?:\b)Alligator(?:\b|)" 3; + "~*(?:\b)AllSubmitter(?:\b|)" 3; + "~*(?:\b)AlphaBot(?:\b|)" 3; + "~*(?:\b)Anarchie(?:\b|)" 3; + "~*(?:\b)Apexoo(?:\b|)" 3; + "~*(?:\b)archive.org_bot(?:\b|)" 3; + "~*(?:\b)ASPSeek(?:\b|)" 3; + "~*(?:\b)Asterias(?:\b|)" 3; + "~*(?:\b)Attach(?:\b|)" 3; + "~*(?:\b)autoemailspider(?:\b|)" 3; + "~*(?:\b)BackDoorBot(?:\b|)" 3; + "~*(?:\b)Backlink-Ceck(?:\b|)" 3; + "~*(?:\b)backlink-check(?:\b|)" 3; + "~*(?:\b)BacklinkCrawler(?:\b|)" 3; + "~*(?:\b)BackStreet(?:\b|)" 3; + "~*(?:\b)BackWeb(?:\b|)" 3; + "~*(?:\b)Badass(?:\b|)" 3; + "~*(?:\b)Bandit(?:\b|)" 3; + "~*(?:\b)Barkrowler(?:\b|)" 3; + "~*(?:\b)BatchFTP(?:\b|)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; + "~*(?:\b)BBBike(?:\b|)" 3; + "~*(?:\b)BDCbot(?:\b|)" 3; + "~*(?:\b)BDFetch(?:\b|)" 3; + "~*(?:\b)BetaBot(?:\b|)" 3; + "~*(?:\b)Bigfoot(?:\b|)" 3; + "~*(?:\b)Bitacle(?:\b|)" 3; + "~*(?:\b)Blackboard(?:\b|)" 3; + "~*(?:\b)Black\ Hole(?:\b|)" 3; + "~*(?:\b)BlackWidow(?:\b|)" 3; + "~*(?:\b)BLEXBot(?:\b|)" 3; + "~*(?:\b)Blow(?:\b|)" 3; + "~*(?:\b)BlowFish(?:\b|)" 3; + "~*(?:\b)Boardreader(?:\b|)" 3; + "~*(?:\b)Bolt(?:\b|)" 3; + "~*(?:\b)BotALot(?:\b|)" 3; + "~*(?:\b)Brandprotect(?:\b|)" 3; + "~*(?:\b)Brandwatch(?:\b|)" 3; + "~*(?:\b)Buddy(?:\b|)" 3; + "~*(?:\b)BuiltBotTough(?:\b|)" 3; + "~*(?:\b)BuiltWith(?:\b|)" 3; + "~*(?:\b)Bullseye(?:\b|)" 3; + "~*(?:\b)BunnySlippers(?:\b|)" 3; + "~*(?:\b)BuzzSumo(?:\b|)" 3; + "~*(?:\b)Calculon(?:\b|)" 3; + "~*(?:\b)CATExplorador(?:\b|)" 3; + "~*(?:\b)CazoodleBot(?:\b|)" 3; + "~*(?:\b)CCBot(?:\b|)" 3; + "~*(?:\b)Cegbfeieh(?:\b|)" 3; + "~*(?:\b)CheeseBot(?:\b|)" 3; + "~*(?:\b)CherryPicker(?:\b|)" 3; + "~*(?:\b)CheTeam(?:\b|)" 3; + "~*(?:\b)ChinaClaw(?:\b|)" 3; + "~*(?:\b)Chlooe(?:\b|)" 3; + "~*(?:\b)Claritybot(?:\b|)" 3; + "~*(?:\b)Cliqzbot(?:\b|)" 3; + "~*(?:\b)Cloud\ mapping(?:\b|)" 3; + "~*(?:\b)coccocbot-web(?:\b|)" 3; + "~*(?:\b)Cogentbot(?:\b|)" 3; + "~*(?:\b)cognitiveseo(?:\b|)" 3; + "~*(?:\b)Collector(?:\b|)" 3; + "~*(?:\b)com.plumanalytics(?:\b|)" 3; + "~*(?:\b)Copier(?:\b|)" 3; + "~*(?:\b)CopyRightCheck(?:\b|)" 3; + "~*(?:\b)Copyscape(?:\b|)" 3; + "~*(?:\b)Cosmos(?:\b|)" 3; + "~*(?:\b)Craftbot(?:\b|)" 3; + "~*(?:\b)crawler4j(?:\b|)" 3; + "~*(?:\b)crawler.feedback(?:\b|)" 3; + "~*(?:\b)crawl.sogou.com(?:\b|)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; + "~*(?:\b)Crescent(?:\b|)" 3; + "~*(?:\b)CrunchBot(?:\b|)" 3; + "~*(?:\b)CSHttp(?:\b|)" 3; + "~*(?:\b)Curious(?:\b|)" 3; + "~*(?:\b)Custo(?:\b|)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; + "~*(?:\b)DataCha0s(?:\b|)" 3; + "~*(?:\b)DBLBot(?:\b|)" 3; + "~*(?:\b)demandbase-bot(?:\b|)" 3; + "~*(?:\b)Demon(?:\b|)" 3; + "~*(?:\b)Deusu(?:\b|)" 3; + "~*(?:\b)Devil(?:\b|)" 3; + "~*(?:\b)Digincore(?:\b|)" 3; + "~*(?:\b)DigitalPebble(?:\b|)" 3; + "~*(?:\b)DIIbot(?:\b|)" 3; + "~*(?:\b)Dirbuster(?:\b|)" 3; + "~*(?:\b)Disco(?:\b|)" 3; + "~*(?:\b)Discobot(?:\b|)" 3; + "~*(?:\b)Discoverybot(?:\b|)" 3; + "~*(?:\b)Dispatch(?:\b|)" 3; + "~*(?:\b)DittoSpyder(?:\b|)" 3; + "~*(?:\b)DnyzBot(?:\b|)" 3; + "~*(?:\b)DomainAppender(?:\b|)" 3; + "~*(?:\b)DomainCrawler(?:\b|)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; + "~*(?:\b)DomainStatsBot(?:\b|)" 3; + "~*(?:\b)Dotbot(?:\b|)" 3; + "~*(?:\b)Download\ Wonder(?:\b|)" 3; + "~*(?:\b)Dragonfly(?:\b|)" 3; + "~*(?:\b)Drip(?:\b|)" 3; + "~*(?:\b)DSearch(?:\b|)" 3; + "~*(?:\b)DTS\ Agent(?:\b|)" 3; + "~*(?:\b)EasyDL(?:\b|)" 3; + "~*(?:\b)Ebingbong(?:\b|)" 3; + "~*(?:\b)eCatch(?:\b|)" 3; + "~*(?:\b)ECCP/1.0(?:\b|)" 3; + "~*(?:\b)Ecxi(?:\b|)" 3; + "~*(?:\b)EirGrabber(?:\b|)" 3; + "~*(?:\b)EMail\ Siphon(?:\b|)" 3; + "~*(?:\b)EMail\ Wolf(?:\b|)" 3; + "~*(?:\b)EroCrawler(?:\b|)" 3; + "~*(?:\b)evc-batch(?:\b|)" 3; + "~*(?:\b)Evil(?:\b|)" 3; + "~*(?:\b)Exabot(?:\b|)" 3; + "~*(?:\b)Express\ WebPictures(?:\b|)" 3; + "~*(?:\b)ExtLinksBot(?:\b|)" 3; + "~*(?:\b)Extractor(?:\b|)" 3; + "~*(?:\b)ExtractorPro(?:\b|)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; + "~*(?:\b)EyeNetIE(?:\b|)" 3; + "~*(?:\b)Ezooms(?:\b|)" 3; + "~*(?:\b)facebookscraper(?:\b|)" 3; + "~*(?:\b)FDM(?:\b|)" 3; + "~*(?:\b)FemtosearchBot(?:\b|)" 3; + "~*(?:\b)FHscan(?:\b|)" 3; + "~*(?:\b)Fimap(?:\b|)" 3; + "~*(?:\b)Firefox/7.0(?:\b|)" 3; + "~*(?:\b)FlashGet(?:\b|)" 3; + "~*(?:\b)Flunky(?:\b|)" 3; + "~*(?:\b)Foobot(?:\b|)" 3; + "~*(?:\b)Freeuploader(?:\b|)" 3; + "~*(?:\b)FrontPage(?:\b|)" 3; + "~*(?:\b)FyberSpider(?:\b|)" 3; + "~*(?:\b)Fyrebot(?:\b|)" 3; + "~*(?:\b)GalaxyBot(?:\b|)" 3; + "~*(?:\b)Genieo(?:\b|)" 3; + "~*(?:\b)GermCrawler(?:\b|)" 3; + "~*(?:\b)Getintent(?:\b|)" 3; + "~*(?:\b)GetRight(?:\b|)" 3; + "~*(?:\b)GetWeb(?:\b|)" 3; + "~*(?:\b)Gigablast(?:\b|)" 3; + "~*(?:\b)Gigabot(?:\b|)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; + "~*(?:\b)Gotit(?:\b|)" 3; + "~*(?:\b)GoZilla(?:\b|)" 3; + "~*(?:\b)Go!Zilla(?:\b|)" 3; + "~*(?:\b)Grabber(?:\b|)" 3; + "~*(?:\b)GrabNet(?:\b|)" 3; + "~*(?:\b)Grafula(?:\b|)" 3; + "~*(?:\b)GrapeFX(?:\b|)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; + "~*(?:\b)GridBot(?:\b|)" 3; + "~*(?:\b)GT::WWW(?:\b|)" 3; + "~*(?:\b)Haansoft(?:\b|)" 3; + "~*(?:\b)HaosouSpider(?:\b|)" 3; + "~*(?:\b)Harvest(?:\b|)" 3; + "~*(?:\b)Havij(?:\b|)" 3; + "~*(?:\b)HEADMasterSEO(?:\b|)" 3; + "~*(?:\b)Heritrix(?:\b|)" 3; + "~*(?:\b)Hloader(?:\b|)" 3; + "~*(?:\b)HMView(?:\b|)" 3; + "~*(?:\b)HTMLparser(?:\b|)" 3; + "~*(?:\b)HTTP::Lite(?:\b|)" 3; + "~*(?:\b)HTTrack(?:\b|)" 3; + "~*(?:\b)Humanlinks(?:\b|)" 3; + "~*(?:\b)HybridBot(?:\b|)" 3; + "~*(?:\b)Iblog(?:\b|)" 3; + "~*(?:\b)IDBot(?:\b|)" 3; + "~*(?:\b)Id-search(?:\b|)" 3; + "~*(?:\b)IlseBot(?:\b|)" 3; + "~*(?:\b)Image\ Fetch(?:\b|)" 3; + "~*(?:\b)Image\ Sucker(?:\b|)" 3; + "~*(?:\b)IndeedBot(?:\b|)" 3; + "~*(?:\b)Indy\ Library(?:\b|)" 3; + "~*(?:\b)InfoNaviRobot(?:\b|)" 3; + "~*(?:\b)InfoTekies(?:\b|)" 3; + "~*(?:\b)instabid(?:\b|)" 3; + "~*(?:\b)Intelliseek(?:\b|)" 3; + "~*(?:\b)InterGET(?:\b|)" 3; + "~*(?:\b)Internet\ Ninja(?:\b|)" 3; + "~*(?:\b)InternetSeer(?:\b|)" 3; + "~*(?:\b)internetVista\ monitor(?:\b|)" 3; + "~*(?:\b)ips-agent(?:\b|)" 3; + "~*(?:\b)Iria(?:\b|)" 3; + "~*(?:\b)IRLbot(?:\b|)" 3; + "~*(?:\b)Iskanie(?:\b|)" 3; + "~*(?:\b)IstellaBot(?:\b|)" 3; + "~*(?:\b)JamesBOT(?:\b|)" 3; + "~*(?:\b)Jbrofuzz(?:\b|)" 3; + "~*(?:\b)JennyBot(?:\b|)" 3; + "~*(?:\b)JetCar(?:\b|)" 3; + "~*(?:\b)Jetty(?:\b|)" 3; + "~*(?:\b)JikeSpider(?:\b|)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; + "~*(?:\b)Joomla(?:\b|)" 3; + "~*(?:\b)Jorgee(?:\b|)" 3; + "~*(?:\b)JustView(?:\b|)" 3; + "~*(?:\b)Jyxobot(?:\b|)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; + "~*(?:\b)Keyword\ Density(?:\b|)" 3; + "~*(?:\b)Kozmosbot(?:\b|)" 3; + "~*(?:\b)Lanshanbot(?:\b|)" 3; + "~*(?:\b)Larbin(?:\b|)" 3; + "~*(?:\b)LeechFTP(?:\b|)" 3; + "~*(?:\b)LeechGet(?:\b|)" 3; + "~*(?:\b)LexiBot(?:\b|)" 3; + "~*(?:\b)Lftp(?:\b|)" 3; + "~*(?:\b)LibWeb(?:\b|)" 3; + "~*(?:\b)Libwhisker(?:\b|)" 3; + "~*(?:\b)Lightspeedsystems(?:\b|)" 3; + "~*(?:\b)Likse(?:\b|)" 3; + "~*(?:\b)Linkdexbot(?:\b|)" 3; + "~*(?:\b)LinkextractorPro(?:\b|)" 3; + "~*(?:\b)LinkpadBot(?:\b|)" 3; + "~*(?:\b)LinkScan(?:\b|)" 3; + "~*(?:\b)LinksManager(?:\b|)" 3; + "~*(?:\b)LinkWalker(?:\b|)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; + "~*(?:\b)Lipperhey(?:\b|)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; + "~*(?:\b)Litemage_walker(?:\b|)" 3; + "~*(?:\b)Lmspider(?:\b|)" 3; + "~*(?:\b)LNSpiderguy(?:\b|)" 3; + "~*(?:\b)Ltx71(?:\b|)" 3; + "~*(?:\b)lwp-request(?:\b|)" 3; + "~*(?:\b)LWP::Simple(?:\b|)" 3; + "~*(?:\b)lwp-trivial(?:\b|)" 3; + "~*(?:\b)Magnet(?:\b|)" 3; + "~*(?:\b)Mag-Net(?:\b|)" 3; + "~*(?:\b)magpie-crawler(?:\b|)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; + "~*(?:\b)Majestic12(?:\b|)" 3; + "~*(?:\b)Majestic-SEO(?:\b|)" 3; + "~*(?:\b)Majestic\ SEO(?:\b|)" 3; + "~*(?:\b)MarkMonitor(?:\b|)" 3; + "~*(?:\b)MarkWatch(?:\b|)" 3; + "~*(?:\b)Masscan(?:\b|)" 3; + "~*(?:\b)Mass\ Downloader(?:\b|)" 3; + "~*(?:\b)Mata\ Hari(?:\b|)" 3; + "~*(?:\b)MauiBot(?:\b|)" 3; + "~*(?:\b)meanpathbot(?:\b|)" 3; + "~*(?:\b)Meanpathbot(?:\b|)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; + "~*(?:\b)mediawords(?:\b|)" 3; + "~*(?:\b)MegaIndex.ru(?:\b|)" 3; + "~*(?:\b)Metauri(?:\b|)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; + "~*(?:\b)MIDown\ tool(?:\b|)" 3; + "~*(?:\b)MIIxpc(?:\b|)" 3; + "~*(?:\b)Mister\ PiX(?:\b|)" 3; + "~*(?:\b)MJ12bot(?:\b|)" 3; + "~*(?:\b)Mojeek(?:\b|)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; + "~*(?:\b)Mr.4x3(?:\b|)" 3; + "~*(?:\b)MSFrontPage(?:\b|)" 3; + "~*(?:\b)MSIECrawler(?:\b|)" 3; + "~*(?:\b)Msrabot(?:\b|)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; + "~*(?:\b)muhstik-scan(?:\b|)" 3; + "~*(?:\b)Musobot(?:\b|)" 3; + "~*(?:\b)Name\ Intelligence(?:\b|)" 3; + "~*(?:\b)Nameprotect(?:\b|)" 3; + "~*(?:\b)Navroad(?:\b|)" 3; + "~*(?:\b)NearSite(?:\b|)" 3; + "~*(?:\b)Needle(?:\b|)" 3; + "~*(?:\b)Nessus(?:\b|)" 3; + "~*(?:\b)NetAnts(?:\b|)" 3; + "~*(?:\b)Netcraft(?:\b|)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; + "~*(?:\b)NetLyzer(?:\b|)" 3; + "~*(?:\b)NetMechanic(?:\b|)" 3; + "~*(?:\b)NetSpider(?:\b|)" 3; + "~*(?:\b)Nettrack(?:\b|)" 3; + "~*(?:\b)Net\ Vampire(?:\b|)" 3; + "~*(?:\b)Netvibes(?:\b|)" 3; + "~*(?:\b)NetZIP(?:\b|)" 3; + "~*(?:\b)NextGenSearchBot(?:\b|)" 3; + "~*(?:\b)Nibbler(?:\b|)" 3; + "~*(?:\b)NICErsPRO(?:\b|)" 3; + "~*(?:\b)Niki-bot(?:\b|)" 3; + "~*(?:\b)Nikto(?:\b|)" 3; + "~*(?:\b)NimbleCrawler(?:\b|)" 3; + "~*(?:\b)Nimbostratus(?:\b|)" 3; + "~*(?:\b)Ninja(?:\b|)" 3; + "~*(?:\b)Nmap(?:\b|)" 3; + "~*(?:\b)NPbot(?:\b|)" 3; + "~*(?:\b)Nutch(?:\b|)" 3; + "~*(?:\b)oBot(?:\b|)" 3; + "~*(?:\b)Octopus(?:\b|)" 3; + "~*(?:\b)Offline\ Explorer(?:\b|)" 3; + "~*(?:\b)Offline\ Navigator(?:\b|)" 3; + "~*(?:\b)OnCrawl(?:\b|)" 3; + "~*(?:\b)Openfind(?:\b|)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; + "~*(?:\b)Openvas(?:\b|)" 3; + "~*(?:\b)OpenVAS(?:\b|)" 3; + "~*(?:\b)OrangeBot(?:\b|)" 3; + "~*(?:\b)OrangeSpider(?:\b|)" 3; + "~*(?:\b)OutclicksBot(?:\b|)" 3; + "~*(?:\b)OutfoxBot(?:\b|)" 3; + "~*(?:\b)PageAnalyzer(?:\b|)" 3; + "~*(?:\b)Page\ Analyzer(?:\b|)" 3; + "~*(?:\b)PageGrabber(?:\b|)" 3; + "~*(?:\b)page\ scorer(?:\b|)" 3; + "~*(?:\b)PageScorer(?:\b|)" 3; + "~*(?:\b)Pandalytics(?:\b|)" 3; + "~*(?:\b)Panscient(?:\b|)" 3; + "~*(?:\b)Papa\ Foto(?:\b|)" 3; + "~*(?:\b)Pavuk(?:\b|)" 3; + "~*(?:\b)pcBrowser(?:\b|)" 3; + "~*(?:\b)PECL::HTTP(?:\b|)" 3; + "~*(?:\b)PeoplePal(?:\b|)" 3; + "~*(?:\b)PHPCrawl(?:\b|)" 3; + "~*(?:\b)Picscout(?:\b|)" 3; + "~*(?:\b)Picsearch(?:\b|)" 3; + "~*(?:\b)PictureFinder(?:\b|)" 3; + "~*(?:\b)Pimonster(?:\b|)" 3; + "~*(?:\b)Pi-Monster(?:\b|)" 3; + "~*(?:\b)Pixray(?:\b|)" 3; + "~*(?:\b)PleaseCrawl(?:\b|)" 3; + "~*(?:\b)plumanalytics(?:\b|)" 3; + "~*(?:\b)Pockey(?:\b|)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; + "~*(?:\b)Probethenet(?:\b|)" 3; + "~*(?:\b)ProPowerBot(?:\b|)" 3; + "~*(?:\b)ProWebWalker(?:\b|)" 3; + "~*(?:\b)Psbot(?:\b|)" 3; + "~*(?:\b)Pump(?:\b|)" 3; + "~*(?:\b)PxBroker(?:\b|)" 3; + "~*(?:\b)PyCurl(?:\b|)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; + "~*(?:\b)Quick-Crawler(?:\b|)" 3; + "~*(?:\b)RankActive(?:\b|)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; + "~*(?:\b)RankFlex(?:\b|)" 3; + "~*(?:\b)RankingBot(?:\b|)" 3; + "~*(?:\b)RankingBot2(?:\b|)" 3; + "~*(?:\b)Rankivabot(?:\b|)" 3; + "~*(?:\b)RankurBot(?:\b|)" 3; + "~*(?:\b)RealDownload(?:\b|)" 3; + "~*(?:\b)Reaper(?:\b|)" 3; + "~*(?:\b)RebelMouse(?:\b|)" 3; + "~*(?:\b)Recorder(?:\b|)" 3; + "~*(?:\b)RedesScrapy(?:\b|)" 3; + "~*(?:\b)ReGet(?:\b|)" 3; + "~*(?:\b)RepoMonkey(?:\b|)" 3; + "~*(?:\b)Ripper(?:\b|)" 3; + "~*(?:\b)RocketCrawler(?:\b|)" 3; + "~*(?:\b)Rogerbot(?:\b|)" 3; + "~*(?:\b)s1z.ru(?:\b|)" 3; + "~*(?:\b)SalesIntelligent(?:\b|)" 3; + "~*(?:\b)SBIder(?:\b|)" 3; + "~*(?:\b)ScanAlert(?:\b|)" 3; + "~*(?:\b)Scanbot(?:\b|)" 3; + "~*(?:\b)scan.lol(?:\b|)" 3; + "~*(?:\b)ScoutJet(?:\b|)" 3; + "~*(?:\b)Scrapy(?:\b|)" 3; + "~*(?:\b)Screaming(?:\b|)" 3; + "~*(?:\b)ScreenerBot(?:\b|)" 3; + "~*(?:\b)Searchestate(?:\b|)" 3; + "~*(?:\b)SearchmetricsBot(?:\b|)" 3; + "~*(?:\b)Semrush(?:\b|)" 3; + "~*(?:\b)SemrushBot(?:\b|)" 3; + "~*(?:\b)SEOkicks(?:\b|)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; + "~*(?:\b)Seomoz(?:\b|)" 3; + "~*(?:\b)SEOprofiler(?:\b|)" 3; + "~*(?:\b)seoscanners(?:\b|)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; + "~*(?:\b)SEOstats(?:\b|)" 3; + "~*(?:\b)serpstatbot(?:\b|)" 3; + "~*(?:\b)sexsearcher(?:\b|)" 3; + "~*(?:\b)Shodan(?:\b|)" 3; + "~*(?:\b)Siphon(?:\b|)" 3; + "~*(?:\b)SISTRIX(?:\b|)" 3; + "~*(?:\b)Sitebeam(?:\b|)" 3; + "~*(?:\b)SiteExplorer(?:\b|)" 3; + "~*(?:\b)Siteimprove(?:\b|)" 3; + "~*(?:\b)SiteLockSpider(?:\b|)" 3; + "~*(?:\b)SiteSnagger(?:\b|)" 3; + "~*(?:\b)SiteSucker(?:\b|)" 3; + "~*(?:\b)Site\ Sucker(?:\b|)" 3; + "~*(?:\b)Sitevigil(?:\b|)" 3; + "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 3; + "~*(?:\b)SlySearch(?:\b|)" 3; + "~*(?:\b)SmartDownload(?:\b|)" 3; + "~*(?:\b)SMTBot(?:\b|)" 3; + "~*(?:\b)Snake(?:\b|)" 3; + "~*(?:\b)Snapbot(?:\b|)" 3; + "~*(?:\b)Snoopy(?:\b|)" 3; + "~*(?:\b)SocialRankIOBot(?:\b|)" 3; + "~*(?:\b)Sociscraper(?:\b|)" 3; + "~*(?:\b)sogouspider(?:\b|)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; + "~*(?:\b)Sosospider(?:\b|)" 3; + "~*(?:\b)Sottopop(?:\b|)" 3; + "~*(?:\b)SpaceBison(?:\b|)" 3; + "~*(?:\b)Spammen(?:\b|)" 3; + "~*(?:\b)SpankBot(?:\b|)" 3; + "~*(?:\b)Spanner(?:\b|)" 3; + "~*(?:\b)sp_auditbot(?:\b|)" 3; + "~*(?:\b)Spbot(?:\b|)" 3; + "~*(?:\b)Spinn3r(?:\b|)" 3; + "~*(?:\b)SputnikBot(?:\b|)" 3; + "~*(?:\b)spyfu(?:\b|)" 3; + "~*(?:\b)Sqlmap(?:\b|)" 3; + "~*(?:\b)Sqlworm(?:\b|)" 3; + "~*(?:\b)Sqworm(?:\b|)" 3; + "~*(?:\b)Steeler(?:\b|)" 3; + "~*(?:\b)Stripper(?:\b|)" 3; + "~*(?:\b)Sucker(?:\b|)" 3; + "~*(?:\b)Sucuri(?:\b|)" 3; + "~*(?:\b)SuperBot(?:\b|)" 3; + "~*(?:\b)SuperHTTP(?:\b|)" 3; + "~*(?:\b)Surfbot(?:\b|)" 3; + "~*(?:\b)SurveyBot(?:\b|)" 3; + "~*(?:\b)Suzuran(?:\b|)" 3; + "~*(?:\b)Swiftbot(?:\b|)" 3; + "~*(?:\b)sysscan(?:\b|)" 3; + "~*(?:\b)Szukacz(?:\b|)" 3; + "~*(?:\b)T0PHackTeam(?:\b|)" 3; + "~*(?:\b)T8Abot(?:\b|)" 3; + "~*(?:\b)tAkeOut(?:\b|)" 3; + "~*(?:\b)Teleport(?:\b|)" 3; + "~*(?:\b)TeleportPro(?:\b|)" 3; + "~*(?:\b)Telesoft(?:\b|)" 3; + "~*(?:\b)Telesphoreo(?:\b|)" 3; + "~*(?:\b)Telesphorep(?:\b|)" 3; + "~*(?:\b)The\ Intraformant(?:\b|)" 3; + "~*(?:\b)TheNomad(?:\b|)" 3; + "~*(?:\b)TightTwatBot(?:\b|)" 3; + "~*(?:\b)Titan(?:\b|)" 3; + "~*(?:\b)Toata(?:\b|)" 3; + "~*(?:\b)Toweyabot(?:\b|)" 3; + "~*(?:\b)Tracemyfile(?:\b|)" 3; + "~*(?:\b)Trendiction(?:\b|)" 3; + "~*(?:\b)Trendictionbot(?:\b|)" 3; + "~*(?:\b)trendiction.com(?:\b|)" 3; + "~*(?:\b)trendiction.de(?:\b|)" 3; + "~*(?:\b)True_Robot(?:\b|)" 3; + "~*(?:\b)Turingos(?:\b|)" 3; + "~*(?:\b)Turnitin(?:\b|)" 3; + "~*(?:\b)TurnitinBot(?:\b|)" 3; + "~*(?:\b)TwengaBot(?:\b|)" 3; + "~*(?:\b)Twice(?:\b|)" 3; + "~*(?:\b)Typhoeus(?:\b|)" 3; + "~*(?:\b)UnisterBot(?:\b|)" 3; + "~*(?:\b)Upflow(?:\b|)" 3; + "~*(?:\b)URLy.Warning(?:\b|)" 3; + "~*(?:\b)URLy\ Warning(?:\b|)" 3; + "~*(?:\b)Vacuum(?:\b|)" 3; + "~*(?:\b)Vagabondo(?:\b|)" 3; + "~*(?:\b)VB\ Project(?:\b|)" 3; + "~*(?:\b)VCI(?:\b|)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; + "~*(?:\b)VidibleScraper(?:\b|)" 3; + "~*(?:\b)Virusdie(?:\b|)" 3; + "~*(?:\b)VoidEYE(?:\b|)" 3; + "~*(?:\b)Voil(?:\b|)" 3; + "~*(?:\b)Voltron(?:\b|)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; + "~*(?:\b)WallpapersHD(?:\b|)" 3; + "~*(?:\b)WASALive-Bot(?:\b|)" 3; + "~*(?:\b)WBSearchBot(?:\b|)" 3; + "~*(?:\b)Webalta(?:\b|)" 3; + "~*(?:\b)WebAuto(?:\b|)" 3; + "~*(?:\b)Web\ Auto(?:\b|)" 3; + "~*(?:\b)WebBandit(?:\b|)" 3; + "~*(?:\b)WebCollage(?:\b|)" 3; + "~*(?:\b)Web\ Collage(?:\b|)" 3; + "~*(?:\b)WebCopier(?:\b|)" 3; + "~*(?:\b)WEBDAV(?:\b|)" 3; + "~*(?:\b)WebEnhancer(?:\b|)" 3; + "~*(?:\b)Web\ Enhancer(?:\b|)" 3; + "~*(?:\b)WebFetch(?:\b|)" 3; + "~*(?:\b)Web\ Fetch(?:\b|)" 3; + "~*(?:\b)WebFuck(?:\b|)" 3; + "~*(?:\b)Web\ Fuck(?:\b|)" 3; + "~*(?:\b)WebGo\ IS(?:\b|)" 3; + "~*(?:\b)WebImageCollector(?:\b|)" 3; + "~*(?:\b)WebLeacher(?:\b|)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; + "~*(?:\b)webmeup-crawler(?:\b|)" 3; + "~*(?:\b)WebPix(?:\b|)" 3; + "~*(?:\b)Web\ Pix(?:\b|)" 3; + "~*(?:\b)WebReaper(?:\b|)" 3; + "~*(?:\b)WebSauger(?:\b|)" 3; + "~*(?:\b)Web\ Sauger(?:\b|)" 3; + "~*(?:\b)Webshag(?:\b|)" 3; + "~*(?:\b)WebsiteExtractor(?:\b|)" 3; + "~*(?:\b)WebsiteQuester(?:\b|)" 3; + "~*(?:\b)Website\ Quester(?:\b|)" 3; + "~*(?:\b)Webster(?:\b|)" 3; + "~*(?:\b)WebStripper(?:\b|)" 3; + "~*(?:\b)WebSucker(?:\b|)" 3; + "~*(?:\b)Web\ Sucker(?:\b|)" 3; + "~*(?:\b)WebWhacker(?:\b|)" 3; + "~*(?:\b)WebZIP(?:\b|)" 3; + "~*(?:\b)WeSEE(?:\b|)" 3; + "~*(?:\b)Whack(?:\b|)" 3; + "~*(?:\b)Whacker(?:\b|)" 3; + "~*(?:\b)Whatweb(?:\b|)" 3; + "~*(?:\b)Who.is\ Bot(?:\b|)" 3; + "~*(?:\b)Widow(?:\b|)" 3; + "~*(?:\b)WinHTTrack(?:\b|)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; + "~*(?:\b)WISENutbot(?:\b|)" 3; + "~*(?:\b)Wonderbot(?:\b|)" 3; + "~*(?:\b)Woobot(?:\b|)" 3; + "~*(?:\b)Wotbox(?:\b|)" 3; + "~*(?:\b)Wprecon(?:\b|)" 3; + "~*(?:\b)WPScan(?:\b|)" 3; + "~*(?:\b)WWW-Collector-E(?:\b|)" 3; + "~*(?:\b)WWW-Mechanize(?:\b|)" 3; + "~*(?:\b)WWW::Mechanize(?:\b|)" 3; + "~*(?:\b)WWWOFFLE(?:\b|)" 3; + "~*(?:\b)x09Mozilla(?:\b|)" 3; + "~*(?:\b)x22Mozilla(?:\b|)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; + "~*(?:\b)Xenu(?:\b|)" 3; + "~*(?:\b)xpymep1.exe(?:\b|)" 3; + "~*(?:\b)YoudaoBot(?:\b|)" 3; + "~*(?:\b)Zade(?:\b|)" 3; + "~*(?:\b)Zauba(?:\b|)" 3; + "~*(?:\b)zauba.io(?:\b|)" 3; + "~*(?:\b)Zermelo(?:\b|)" 3; + "~*(?:\b)Zeus(?:\b|)" 3; + "~*(?:\b)zgrab(?:\b|)" 3; + "~*(?:\b)Zitebot(?:\b|)" 3; + "~*(?:\b)ZmEu(?:\b|)" 3; + "~*(?:\b)ZumBot(?:\b|)" 3; + "~*(?:\b)ZyBorg(?:\b|)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # *********************************************** diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 60944cc49..d39f16e78 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1540 -### Updated: Sun Jun 23 15:20:43 SAST 2019 +### Version: V3.2019.06.1541 +### Updated: Sun Jun 23 16:11:00 SAST 2019 ### Bad Referrer Count: 6693 ### Bad Bot Count: 556 ################################################### diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 60865f5af..5811e6ecd 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,8 +6,8 @@ ### Version Information # ################################################### -### Version: V3.2019.06.1540 -### Updated: Sun Jun 23 15:20:45 SAST 2019 +### Version: V3.2019.06.1541 +### Updated: Sun Jun 23 16:11:02 SAST 2019 ### Bad Bot Count: 556 ################################################### ### Version Information ##