diff --git a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf index 7ff066d24..60959195e 100644 --- a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf @@ -33,4 +33,4 @@ # 111.111.111.111 0; -104.154.113.151 0; +35.192.85.2 0; diff --git a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf index 59bf98ecb..6a8bec7e6 100644 --- a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1474 -### Updated: Fri Jun 21 11:40:42 SAST 2019 +### Version: V3.2019.06.1475 +### Updated: Fri Jun 21 13:54:40 SAST 2019 ### Bad Referrer Count: 6693 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf index 7ff066d24..60959195e 100644 --- a/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf @@ -33,4 +33,4 @@ # 111.111.111.111 0; -104.154.113.151 0; +35.192.85.2 0; diff --git a/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf index cc51842f2..fb7253649 100644 --- a/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf @@ -10,8 +10,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1474 -### Updated: Fri Jun 21 11:40:43 SAST 2019 +### Version: V3.2019.06.1475 +### Updated: Fri Jun 21 13:54:40 SAST 2019 ### Bad Referrer Count: 6693 ### Bad Bot Count: 557 ################################################### @@ -143,563 +143,563 @@ map $http_user_agent $bad_bot { # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)404checker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)404enemy(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)80legs(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Abonti(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Aboundex(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Aboundexbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Acunetix(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ADmantX(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)AhrefsBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)AIBOT(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)AiHitBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Aipbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Alexibot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Alligator(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)AllSubmitter(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)AlphaBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Anarchie(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Apexoo(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)archive.org_bot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ASPSeek(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Asterias(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Attach(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)autoemailspider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BackDoorBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Backlink-Ceck(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)backlink-check(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BacklinkCrawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BackStreet(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BackWeb(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Badass(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Bandit(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Barkrowler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BatchFTP(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BBBike(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BDCbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BDFetch(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BetaBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Bigfoot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Bitacle(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Blackboard(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Black\ Hole(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BlackWidow(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BLEXBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Blow(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BlowFish(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Boardreader(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Bolt(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BotALot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Brandprotect(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Brandwatch(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Bubing(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Buddy(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BuiltBotTough(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BuiltWith(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Bullseye(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BunnySlippers(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)BuzzSumo(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Calculon(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)CATExplorador(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)CazoodleBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)CCBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Cegbfeieh(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)CheeseBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)CherryPicker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)CheTeam(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ChinaClaw(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Chlooe(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Claritybot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Cliqzbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Cloud\ mapping(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)coccocbot-web(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Cogentbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)cognitiveseo(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Collector(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)com.plumanalytics(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Copier(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)CopyRightCheck(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Copyscape(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Cosmos(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Craftbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)crawler4j(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)crawler.feedback(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)crawl.sogou.com(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)CrazyWebCrawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Crescent(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)CrunchBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)CSHttp(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Curious(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Custo(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)DataCha0s(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)DBLBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)demandbase-bot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Demon(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Deusu(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Devil(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Digincore(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)DigitalPebble(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)DIIbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Dirbuster(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Disco(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Discobot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Discoverybot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Dispatch(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)DittoSpyder(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)DnyzBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)DomainAppender(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)DomainCrawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)DomainSigmaCrawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)DomainStatsBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Dotbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Download\ Wonder(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Dragonfly(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Drip(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)DSearch(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)DTS\ Agent(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)EasyDL(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Ebingbong(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)eCatch(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ECCP/1.0(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Ecxi(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)EirGrabber(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)EMail\ Siphon(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)EMail\ Wolf(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)EroCrawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)evc-batch(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Evil(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Exabot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Express\ WebPictures(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ExtLinksBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Extractor(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ExtractorPro(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)EyeNetIE(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Ezooms(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)facebookscraper(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)FDM(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)FemtosearchBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)FHscan(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Fimap(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Firefox/7.0(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)FlashGet(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Flunky(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Foobot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Freeuploader(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)FrontPage(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)FyberSpider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Fyrebot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)GalaxyBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Genieo(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)GermCrawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Getintent(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)GetRight(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)GetWeb(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Gigablast(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Gigabot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Gotit(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)GoZilla(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Go!Zilla(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Grabber(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)GrabNet(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Grafula(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)GrapeFX(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)GrapeshotCrawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)GridBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)GT::WWW(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Haansoft(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)HaosouSpider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Harvest(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Havij(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)HEADMasterSEO(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Heritrix(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Hloader(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)HMView(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)HTMLparser(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)HTTP::Lite(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)HTTrack(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Humanlinks(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)HybridBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Iblog(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)IDBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Id-search(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)IlseBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Image\ Fetch(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Image\ Sucker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)IndeedBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Indy\ Library(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)InfoNaviRobot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)InfoTekies(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)instabid(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Intelliseek(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)InterGET(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Internet\ Ninja(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)InternetSeer(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)internetVista\ monitor(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ips-agent(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Iria(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)IRLbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Iskanie(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)IstellaBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)JamesBOT(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Jbrofuzz(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)JennyBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)JetCar(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Jetty(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)JikeSpider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Joomla(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Jorgee(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)JustView(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Jyxobot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Kenjin\ Spider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Keyword\ Density(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Kozmosbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Lanshanbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Larbin(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)LeechFTP(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)LeechGet(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)LexiBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Lftp(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)LibWeb(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Libwhisker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Lightspeedsystems(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Likse(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Linkdexbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)LinkextractorPro(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)LinkpadBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)LinkScan(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)LinksManager(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)LinkWalker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)LinqiaRSSBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)LinqiaScrapeBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Lipperhey(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Lipperhey\ Spider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Litemage_walker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Lmspider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)LNSpiderguy(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Ltx71(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)lwp-request(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)LWP::Simple(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)lwp-trivial(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Magnet(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Mag-Net(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)magpie-crawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Mail.RU_Bot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Majestic12(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Majestic-SEO(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Majestic\ SEO(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)MarkMonitor(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)MarkWatch(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Masscan(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Mass\ Downloader(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Mata\ Hari(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)MauiBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)meanpathbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Meanpathbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)MeanPath\ Bot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Mediatoolkitbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)mediawords(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)MegaIndex.ru(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Metauri(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)MFC_Tear_Sample(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)MIDown\ tool(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)MIIxpc(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Mister\ PiX(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)MJ12bot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Mojeek(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Mr.4x3(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)MSFrontPage(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)MSIECrawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Msrabot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)muhstik-scan(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Musobot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Name\ Intelligence(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Nameprotect(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Navroad(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)NearSite(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Needle(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Nessus(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)NetAnts(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Netcraft(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)NetLyzer(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)NetMechanic(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)NetSpider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Nettrack(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Net\ Vampire(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Netvibes(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)NetZIP(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)NextGenSearchBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Nibbler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)NICErsPRO(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Niki-bot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Nikto(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)NimbleCrawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Nimbostratus(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Ninja(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Nmap(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)NPbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Nutch(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)oBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Octopus(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Offline\ Explorer(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Offline\ Navigator(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)OnCrawl(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Openfind(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)OpenLinkProfiler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Openvas(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)OpenVAS(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)OrangeBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)OrangeSpider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)OutclicksBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)OutfoxBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)PageAnalyzer(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Page\ Analyzer(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)PageGrabber(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)page\ scorer(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)PageScorer(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Pandalytics(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Panscient(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Papa\ Foto(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Pavuk(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)pcBrowser(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)PECL::HTTP(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)PeoplePal(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)PHPCrawl(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Picscout(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Picsearch(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)PictureFinder(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Pimonster(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Pi-Monster(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Pixray(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)PleaseCrawl(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)plumanalytics(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Pockey(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Probethenet(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ProPowerBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ProWebWalker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Psbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Pump(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)PxBroker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)PyCurl(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)QueryN\ Metasearch(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Quick-Crawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)RankActive(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)RankActiveLinkBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)RankFlex(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)RankingBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)RankingBot2(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Rankivabot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)RankurBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)RealDownload(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Reaper(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)RebelMouse(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Recorder(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)RedesScrapy(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ReGet(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)RepoMonkey(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Ripper(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)RocketCrawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Rogerbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)s1z.ru(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SalesIntelligent(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SBIder(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ScanAlert(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Scanbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)scan.lol(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ScoutJet(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Scrapy(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Screaming(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ScreenerBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Searchestate(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SearchmetricsBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Semrush(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SemrushBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SEOkicks(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SEOkicks-Robot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SEOlyticsCrawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Seomoz(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SEOprofiler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)seoscanners(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SeoSiteCheckup(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SEOstats(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)serpstatbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)sexsearcher(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Shodan(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Siphon(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SISTRIX(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Sitebeam(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SiteExplorer(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Siteimprove(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SiteLockSpider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SiteSnagger(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SiteSucker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Site\ Sucker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Sitevigil(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Slackbot-LinkExpanding(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SlySearch(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SmartDownload(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SMTBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Snake(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Snapbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Snoopy(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SocialRankIOBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Sociscraper(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)sogouspider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Sogou\ web\ spider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Sosospider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Sottopop(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SpaceBison(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Spammen(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SpankBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Spanner(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)sp_auditbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Spbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Spinn3r(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SputnikBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)spyfu(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Sqlmap(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Sqlworm(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Sqworm(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Steeler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Stripper(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Sucker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Sucuri(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SuperBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SuperHTTP(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Surfbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)SurveyBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Suzuran(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Swiftbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)sysscan(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Szukacz(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)T0PHackTeam(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)T8Abot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)tAkeOut(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Teleport(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)TeleportPro(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Telesoft(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Telesphoreo(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Telesphorep(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)The\ Intraformant(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)TheNomad(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)TightTwatBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Titan(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Toata(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Toweyabot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Tracemyfile(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Trendiction(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Trendictionbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)trendiction.com(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)trendiction.de(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)True_Robot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Turingos(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Turnitin(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)TurnitinBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)TwengaBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Twice(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Typhoeus(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)UnisterBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Upflow(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)URLy.Warning(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)URLy\ Warning(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Vacuum(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Vagabondo(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)VB\ Project(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)VCI(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)VeriCiteCrawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)VidibleScraper(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Virusdie(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)VoidEYE(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Voil(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Voltron(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Wallpapers/3.0(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WallpapersHD(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WASALive-Bot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WBSearchBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Webalta(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebAuto(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Web\ Auto(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebBandit(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebCollage(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Web\ Collage(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebCopier(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WEBDAV(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebEnhancer(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Web\ Enhancer(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebFetch(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Web\ Fetch(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebFuck(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Web\ Fuck(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebGo\ IS(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebImageCollector(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebLeacher(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)webmeup-crawler(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebPix(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Web\ Pix(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebReaper(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebSauger(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Web\ Sauger(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Webshag(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebsiteExtractor(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebsiteQuester(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Website\ Quester(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Webster(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebStripper(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebSucker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Web\ Sucker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebWhacker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WebZIP(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WeSEE(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Whack(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Whacker(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Whatweb(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Who.is\ Bot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Widow(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WinHTTrack(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WiseGuys\ Robot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WISENutbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Wonderbot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Woobot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Wotbox(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Wprecon(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WPScan(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WWW-Collector-E(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WWW-Mechanize(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WWW::Mechanize(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)WWWOFFLE(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)x09Mozilla(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)x22Mozilla(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Xaldon_WebSpider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Xenu(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)xpymep1.exe(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)YoudaoBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Zade(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Zauba(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)zauba.io(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Zermelo(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Zeus(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)zgrab(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)Zitebot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ZmEu(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ZumBot(?:|\s|,|\.|\b|\/)" 3; - "~*(?:\b)ZyBorg(?:|\s|,|\.|\b|\/)" 3; + "~*(?:\b)360Spider(?:\b|)" 3; + "~*(?:\b)404checker(?:\b|)" 3; + "~*(?:\b)404enemy(?:\b|)" 3; + "~*(?:\b)80legs(?:\b|)" 3; + "~*(?:\b)Abonti(?:\b|)" 3; + "~*(?:\b)Aboundex(?:\b|)" 3; + "~*(?:\b)Aboundexbot(?:\b|)" 3; + "~*(?:\b)Acunetix(?:\b|)" 3; + "~*(?:\b)ADmantX(?:\b|)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; + "~*(?:\b)AhrefsBot(?:\b|)" 3; + "~*(?:\b)AIBOT(?:\b|)" 3; + "~*(?:\b)AiHitBot(?:\b|)" 3; + "~*(?:\b)Aipbot(?:\b|)" 3; + "~*(?:\b)Alexibot(?:\b|)" 3; + "~*(?:\b)Alligator(?:\b|)" 3; + "~*(?:\b)AllSubmitter(?:\b|)" 3; + "~*(?:\b)AlphaBot(?:\b|)" 3; + "~*(?:\b)Anarchie(?:\b|)" 3; + "~*(?:\b)Apexoo(?:\b|)" 3; + "~*(?:\b)archive.org_bot(?:\b|)" 3; + "~*(?:\b)ASPSeek(?:\b|)" 3; + "~*(?:\b)Asterias(?:\b|)" 3; + "~*(?:\b)Attach(?:\b|)" 3; + "~*(?:\b)autoemailspider(?:\b|)" 3; + "~*(?:\b)BackDoorBot(?:\b|)" 3; + "~*(?:\b)Backlink-Ceck(?:\b|)" 3; + "~*(?:\b)backlink-check(?:\b|)" 3; + "~*(?:\b)BacklinkCrawler(?:\b|)" 3; + "~*(?:\b)BackStreet(?:\b|)" 3; + "~*(?:\b)BackWeb(?:\b|)" 3; + "~*(?:\b)Badass(?:\b|)" 3; + "~*(?:\b)Bandit(?:\b|)" 3; + "~*(?:\b)Barkrowler(?:\b|)" 3; + "~*(?:\b)BatchFTP(?:\b|)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; + "~*(?:\b)BBBike(?:\b|)" 3; + "~*(?:\b)BDCbot(?:\b|)" 3; + "~*(?:\b)BDFetch(?:\b|)" 3; + "~*(?:\b)BetaBot(?:\b|)" 3; + "~*(?:\b)Bigfoot(?:\b|)" 3; + "~*(?:\b)Bitacle(?:\b|)" 3; + "~*(?:\b)Blackboard(?:\b|)" 3; + "~*(?:\b)Black\ Hole(?:\b|)" 3; + "~*(?:\b)BlackWidow(?:\b|)" 3; + "~*(?:\b)BLEXBot(?:\b|)" 3; + "~*(?:\b)Blow(?:\b|)" 3; + "~*(?:\b)BlowFish(?:\b|)" 3; + "~*(?:\b)Boardreader(?:\b|)" 3; + "~*(?:\b)Bolt(?:\b|)" 3; + "~*(?:\b)BotALot(?:\b|)" 3; + "~*(?:\b)Brandprotect(?:\b|)" 3; + "~*(?:\b)Brandwatch(?:\b|)" 3; + "~*(?:\b)Bubing(?:\b|)" 3; + "~*(?:\b)Buddy(?:\b|)" 3; + "~*(?:\b)BuiltBotTough(?:\b|)" 3; + "~*(?:\b)BuiltWith(?:\b|)" 3; + "~*(?:\b)Bullseye(?:\b|)" 3; + "~*(?:\b)BunnySlippers(?:\b|)" 3; + "~*(?:\b)BuzzSumo(?:\b|)" 3; + "~*(?:\b)Calculon(?:\b|)" 3; + "~*(?:\b)CATExplorador(?:\b|)" 3; + "~*(?:\b)CazoodleBot(?:\b|)" 3; + "~*(?:\b)CCBot(?:\b|)" 3; + "~*(?:\b)Cegbfeieh(?:\b|)" 3; + "~*(?:\b)CheeseBot(?:\b|)" 3; + "~*(?:\b)CherryPicker(?:\b|)" 3; + "~*(?:\b)CheTeam(?:\b|)" 3; + "~*(?:\b)ChinaClaw(?:\b|)" 3; + "~*(?:\b)Chlooe(?:\b|)" 3; + "~*(?:\b)Claritybot(?:\b|)" 3; + "~*(?:\b)Cliqzbot(?:\b|)" 3; + "~*(?:\b)Cloud\ mapping(?:\b|)" 3; + "~*(?:\b)coccocbot-web(?:\b|)" 3; + "~*(?:\b)Cogentbot(?:\b|)" 3; + "~*(?:\b)cognitiveseo(?:\b|)" 3; + "~*(?:\b)Collector(?:\b|)" 3; + "~*(?:\b)com.plumanalytics(?:\b|)" 3; + "~*(?:\b)Copier(?:\b|)" 3; + "~*(?:\b)CopyRightCheck(?:\b|)" 3; + "~*(?:\b)Copyscape(?:\b|)" 3; + "~*(?:\b)Cosmos(?:\b|)" 3; + "~*(?:\b)Craftbot(?:\b|)" 3; + "~*(?:\b)crawler4j(?:\b|)" 3; + "~*(?:\b)crawler.feedback(?:\b|)" 3; + "~*(?:\b)crawl.sogou.com(?:\b|)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; + "~*(?:\b)Crescent(?:\b|)" 3; + "~*(?:\b)CrunchBot(?:\b|)" 3; + "~*(?:\b)CSHttp(?:\b|)" 3; + "~*(?:\b)Curious(?:\b|)" 3; + "~*(?:\b)Custo(?:\b|)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; + "~*(?:\b)DataCha0s(?:\b|)" 3; + "~*(?:\b)DBLBot(?:\b|)" 3; + "~*(?:\b)demandbase-bot(?:\b|)" 3; + "~*(?:\b)Demon(?:\b|)" 3; + "~*(?:\b)Deusu(?:\b|)" 3; + "~*(?:\b)Devil(?:\b|)" 3; + "~*(?:\b)Digincore(?:\b|)" 3; + "~*(?:\b)DigitalPebble(?:\b|)" 3; + "~*(?:\b)DIIbot(?:\b|)" 3; + "~*(?:\b)Dirbuster(?:\b|)" 3; + "~*(?:\b)Disco(?:\b|)" 3; + "~*(?:\b)Discobot(?:\b|)" 3; + "~*(?:\b)Discoverybot(?:\b|)" 3; + "~*(?:\b)Dispatch(?:\b|)" 3; + "~*(?:\b)DittoSpyder(?:\b|)" 3; + "~*(?:\b)DnyzBot(?:\b|)" 3; + "~*(?:\b)DomainAppender(?:\b|)" 3; + "~*(?:\b)DomainCrawler(?:\b|)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; + "~*(?:\b)DomainStatsBot(?:\b|)" 3; + "~*(?:\b)Dotbot(?:\b|)" 3; + "~*(?:\b)Download\ Wonder(?:\b|)" 3; + "~*(?:\b)Dragonfly(?:\b|)" 3; + "~*(?:\b)Drip(?:\b|)" 3; + "~*(?:\b)DSearch(?:\b|)" 3; + "~*(?:\b)DTS\ Agent(?:\b|)" 3; + "~*(?:\b)EasyDL(?:\b|)" 3; + "~*(?:\b)Ebingbong(?:\b|)" 3; + "~*(?:\b)eCatch(?:\b|)" 3; + "~*(?:\b)ECCP/1.0(?:\b|)" 3; + "~*(?:\b)Ecxi(?:\b|)" 3; + "~*(?:\b)EirGrabber(?:\b|)" 3; + "~*(?:\b)EMail\ Siphon(?:\b|)" 3; + "~*(?:\b)EMail\ Wolf(?:\b|)" 3; + "~*(?:\b)EroCrawler(?:\b|)" 3; + "~*(?:\b)evc-batch(?:\b|)" 3; + "~*(?:\b)Evil(?:\b|)" 3; + "~*(?:\b)Exabot(?:\b|)" 3; + "~*(?:\b)Express\ WebPictures(?:\b|)" 3; + "~*(?:\b)ExtLinksBot(?:\b|)" 3; + "~*(?:\b)Extractor(?:\b|)" 3; + "~*(?:\b)ExtractorPro(?:\b|)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; + "~*(?:\b)EyeNetIE(?:\b|)" 3; + "~*(?:\b)Ezooms(?:\b|)" 3; + "~*(?:\b)facebookscraper(?:\b|)" 3; + "~*(?:\b)FDM(?:\b|)" 3; + "~*(?:\b)FemtosearchBot(?:\b|)" 3; + "~*(?:\b)FHscan(?:\b|)" 3; + "~*(?:\b)Fimap(?:\b|)" 3; + "~*(?:\b)Firefox/7.0(?:\b|)" 3; + "~*(?:\b)FlashGet(?:\b|)" 3; + "~*(?:\b)Flunky(?:\b|)" 3; + "~*(?:\b)Foobot(?:\b|)" 3; + "~*(?:\b)Freeuploader(?:\b|)" 3; + "~*(?:\b)FrontPage(?:\b|)" 3; + "~*(?:\b)FyberSpider(?:\b|)" 3; + "~*(?:\b)Fyrebot(?:\b|)" 3; + "~*(?:\b)GalaxyBot(?:\b|)" 3; + "~*(?:\b)Genieo(?:\b|)" 3; + "~*(?:\b)GermCrawler(?:\b|)" 3; + "~*(?:\b)Getintent(?:\b|)" 3; + "~*(?:\b)GetRight(?:\b|)" 3; + "~*(?:\b)GetWeb(?:\b|)" 3; + "~*(?:\b)Gigablast(?:\b|)" 3; + "~*(?:\b)Gigabot(?:\b|)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; + "~*(?:\b)Gotit(?:\b|)" 3; + "~*(?:\b)GoZilla(?:\b|)" 3; + "~*(?:\b)Go!Zilla(?:\b|)" 3; + "~*(?:\b)Grabber(?:\b|)" 3; + "~*(?:\b)GrabNet(?:\b|)" 3; + "~*(?:\b)Grafula(?:\b|)" 3; + "~*(?:\b)GrapeFX(?:\b|)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; + "~*(?:\b)GridBot(?:\b|)" 3; + "~*(?:\b)GT::WWW(?:\b|)" 3; + "~*(?:\b)Haansoft(?:\b|)" 3; + "~*(?:\b)HaosouSpider(?:\b|)" 3; + "~*(?:\b)Harvest(?:\b|)" 3; + "~*(?:\b)Havij(?:\b|)" 3; + "~*(?:\b)HEADMasterSEO(?:\b|)" 3; + "~*(?:\b)Heritrix(?:\b|)" 3; + "~*(?:\b)Hloader(?:\b|)" 3; + "~*(?:\b)HMView(?:\b|)" 3; + "~*(?:\b)HTMLparser(?:\b|)" 3; + "~*(?:\b)HTTP::Lite(?:\b|)" 3; + "~*(?:\b)HTTrack(?:\b|)" 3; + "~*(?:\b)Humanlinks(?:\b|)" 3; + "~*(?:\b)HybridBot(?:\b|)" 3; + "~*(?:\b)Iblog(?:\b|)" 3; + "~*(?:\b)IDBot(?:\b|)" 3; + "~*(?:\b)Id-search(?:\b|)" 3; + "~*(?:\b)IlseBot(?:\b|)" 3; + "~*(?:\b)Image\ Fetch(?:\b|)" 3; + "~*(?:\b)Image\ Sucker(?:\b|)" 3; + "~*(?:\b)IndeedBot(?:\b|)" 3; + "~*(?:\b)Indy\ Library(?:\b|)" 3; + "~*(?:\b)InfoNaviRobot(?:\b|)" 3; + "~*(?:\b)InfoTekies(?:\b|)" 3; + "~*(?:\b)instabid(?:\b|)" 3; + "~*(?:\b)Intelliseek(?:\b|)" 3; + "~*(?:\b)InterGET(?:\b|)" 3; + "~*(?:\b)Internet\ Ninja(?:\b|)" 3; + "~*(?:\b)InternetSeer(?:\b|)" 3; + "~*(?:\b)internetVista\ monitor(?:\b|)" 3; + "~*(?:\b)ips-agent(?:\b|)" 3; + "~*(?:\b)Iria(?:\b|)" 3; + "~*(?:\b)IRLbot(?:\b|)" 3; + "~*(?:\b)Iskanie(?:\b|)" 3; + "~*(?:\b)IstellaBot(?:\b|)" 3; + "~*(?:\b)JamesBOT(?:\b|)" 3; + "~*(?:\b)Jbrofuzz(?:\b|)" 3; + "~*(?:\b)JennyBot(?:\b|)" 3; + "~*(?:\b)JetCar(?:\b|)" 3; + "~*(?:\b)Jetty(?:\b|)" 3; + "~*(?:\b)JikeSpider(?:\b|)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; + "~*(?:\b)Joomla(?:\b|)" 3; + "~*(?:\b)Jorgee(?:\b|)" 3; + "~*(?:\b)JustView(?:\b|)" 3; + "~*(?:\b)Jyxobot(?:\b|)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; + "~*(?:\b)Keyword\ Density(?:\b|)" 3; + "~*(?:\b)Kozmosbot(?:\b|)" 3; + "~*(?:\b)Lanshanbot(?:\b|)" 3; + "~*(?:\b)Larbin(?:\b|)" 3; + "~*(?:\b)LeechFTP(?:\b|)" 3; + "~*(?:\b)LeechGet(?:\b|)" 3; + "~*(?:\b)LexiBot(?:\b|)" 3; + "~*(?:\b)Lftp(?:\b|)" 3; + "~*(?:\b)LibWeb(?:\b|)" 3; + "~*(?:\b)Libwhisker(?:\b|)" 3; + "~*(?:\b)Lightspeedsystems(?:\b|)" 3; + "~*(?:\b)Likse(?:\b|)" 3; + "~*(?:\b)Linkdexbot(?:\b|)" 3; + "~*(?:\b)LinkextractorPro(?:\b|)" 3; + "~*(?:\b)LinkpadBot(?:\b|)" 3; + "~*(?:\b)LinkScan(?:\b|)" 3; + "~*(?:\b)LinksManager(?:\b|)" 3; + "~*(?:\b)LinkWalker(?:\b|)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; + "~*(?:\b)Lipperhey(?:\b|)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; + "~*(?:\b)Litemage_walker(?:\b|)" 3; + "~*(?:\b)Lmspider(?:\b|)" 3; + "~*(?:\b)LNSpiderguy(?:\b|)" 3; + "~*(?:\b)Ltx71(?:\b|)" 3; + "~*(?:\b)lwp-request(?:\b|)" 3; + "~*(?:\b)LWP::Simple(?:\b|)" 3; + "~*(?:\b)lwp-trivial(?:\b|)" 3; + "~*(?:\b)Magnet(?:\b|)" 3; + "~*(?:\b)Mag-Net(?:\b|)" 3; + "~*(?:\b)magpie-crawler(?:\b|)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; + "~*(?:\b)Majestic12(?:\b|)" 3; + "~*(?:\b)Majestic-SEO(?:\b|)" 3; + "~*(?:\b)Majestic\ SEO(?:\b|)" 3; + "~*(?:\b)MarkMonitor(?:\b|)" 3; + "~*(?:\b)MarkWatch(?:\b|)" 3; + "~*(?:\b)Masscan(?:\b|)" 3; + "~*(?:\b)Mass\ Downloader(?:\b|)" 3; + "~*(?:\b)Mata\ Hari(?:\b|)" 3; + "~*(?:\b)MauiBot(?:\b|)" 3; + "~*(?:\b)meanpathbot(?:\b|)" 3; + "~*(?:\b)Meanpathbot(?:\b|)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; + "~*(?:\b)mediawords(?:\b|)" 3; + "~*(?:\b)MegaIndex.ru(?:\b|)" 3; + "~*(?:\b)Metauri(?:\b|)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; + "~*(?:\b)MIDown\ tool(?:\b|)" 3; + "~*(?:\b)MIIxpc(?:\b|)" 3; + "~*(?:\b)Mister\ PiX(?:\b|)" 3; + "~*(?:\b)MJ12bot(?:\b|)" 3; + "~*(?:\b)Mojeek(?:\b|)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; + "~*(?:\b)Mr.4x3(?:\b|)" 3; + "~*(?:\b)MSFrontPage(?:\b|)" 3; + "~*(?:\b)MSIECrawler(?:\b|)" 3; + "~*(?:\b)Msrabot(?:\b|)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; + "~*(?:\b)muhstik-scan(?:\b|)" 3; + "~*(?:\b)Musobot(?:\b|)" 3; + "~*(?:\b)Name\ Intelligence(?:\b|)" 3; + "~*(?:\b)Nameprotect(?:\b|)" 3; + "~*(?:\b)Navroad(?:\b|)" 3; + "~*(?:\b)NearSite(?:\b|)" 3; + "~*(?:\b)Needle(?:\b|)" 3; + "~*(?:\b)Nessus(?:\b|)" 3; + "~*(?:\b)NetAnts(?:\b|)" 3; + "~*(?:\b)Netcraft(?:\b|)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; + "~*(?:\b)NetLyzer(?:\b|)" 3; + "~*(?:\b)NetMechanic(?:\b|)" 3; + "~*(?:\b)NetSpider(?:\b|)" 3; + "~*(?:\b)Nettrack(?:\b|)" 3; + "~*(?:\b)Net\ Vampire(?:\b|)" 3; + "~*(?:\b)Netvibes(?:\b|)" 3; + "~*(?:\b)NetZIP(?:\b|)" 3; + "~*(?:\b)NextGenSearchBot(?:\b|)" 3; + "~*(?:\b)Nibbler(?:\b|)" 3; + "~*(?:\b)NICErsPRO(?:\b|)" 3; + "~*(?:\b)Niki-bot(?:\b|)" 3; + "~*(?:\b)Nikto(?:\b|)" 3; + "~*(?:\b)NimbleCrawler(?:\b|)" 3; + "~*(?:\b)Nimbostratus(?:\b|)" 3; + "~*(?:\b)Ninja(?:\b|)" 3; + "~*(?:\b)Nmap(?:\b|)" 3; + "~*(?:\b)NPbot(?:\b|)" 3; + "~*(?:\b)Nutch(?:\b|)" 3; + "~*(?:\b)oBot(?:\b|)" 3; + "~*(?:\b)Octopus(?:\b|)" 3; + "~*(?:\b)Offline\ Explorer(?:\b|)" 3; + "~*(?:\b)Offline\ Navigator(?:\b|)" 3; + "~*(?:\b)OnCrawl(?:\b|)" 3; + "~*(?:\b)Openfind(?:\b|)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; + "~*(?:\b)Openvas(?:\b|)" 3; + "~*(?:\b)OpenVAS(?:\b|)" 3; + "~*(?:\b)OrangeBot(?:\b|)" 3; + "~*(?:\b)OrangeSpider(?:\b|)" 3; + "~*(?:\b)OutclicksBot(?:\b|)" 3; + "~*(?:\b)OutfoxBot(?:\b|)" 3; + "~*(?:\b)PageAnalyzer(?:\b|)" 3; + "~*(?:\b)Page\ Analyzer(?:\b|)" 3; + "~*(?:\b)PageGrabber(?:\b|)" 3; + "~*(?:\b)page\ scorer(?:\b|)" 3; + "~*(?:\b)PageScorer(?:\b|)" 3; + "~*(?:\b)Pandalytics(?:\b|)" 3; + "~*(?:\b)Panscient(?:\b|)" 3; + "~*(?:\b)Papa\ Foto(?:\b|)" 3; + "~*(?:\b)Pavuk(?:\b|)" 3; + "~*(?:\b)pcBrowser(?:\b|)" 3; + "~*(?:\b)PECL::HTTP(?:\b|)" 3; + "~*(?:\b)PeoplePal(?:\b|)" 3; + "~*(?:\b)PHPCrawl(?:\b|)" 3; + "~*(?:\b)Picscout(?:\b|)" 3; + "~*(?:\b)Picsearch(?:\b|)" 3; + "~*(?:\b)PictureFinder(?:\b|)" 3; + "~*(?:\b)Pimonster(?:\b|)" 3; + "~*(?:\b)Pi-Monster(?:\b|)" 3; + "~*(?:\b)Pixray(?:\b|)" 3; + "~*(?:\b)PleaseCrawl(?:\b|)" 3; + "~*(?:\b)plumanalytics(?:\b|)" 3; + "~*(?:\b)Pockey(?:\b|)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; + "~*(?:\b)Probethenet(?:\b|)" 3; + "~*(?:\b)ProPowerBot(?:\b|)" 3; + "~*(?:\b)ProWebWalker(?:\b|)" 3; + "~*(?:\b)Psbot(?:\b|)" 3; + "~*(?:\b)Pump(?:\b|)" 3; + "~*(?:\b)PxBroker(?:\b|)" 3; + "~*(?:\b)PyCurl(?:\b|)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; + "~*(?:\b)Quick-Crawler(?:\b|)" 3; + "~*(?:\b)RankActive(?:\b|)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; + "~*(?:\b)RankFlex(?:\b|)" 3; + "~*(?:\b)RankingBot(?:\b|)" 3; + "~*(?:\b)RankingBot2(?:\b|)" 3; + "~*(?:\b)Rankivabot(?:\b|)" 3; + "~*(?:\b)RankurBot(?:\b|)" 3; + "~*(?:\b)RealDownload(?:\b|)" 3; + "~*(?:\b)Reaper(?:\b|)" 3; + "~*(?:\b)RebelMouse(?:\b|)" 3; + "~*(?:\b)Recorder(?:\b|)" 3; + "~*(?:\b)RedesScrapy(?:\b|)" 3; + "~*(?:\b)ReGet(?:\b|)" 3; + "~*(?:\b)RepoMonkey(?:\b|)" 3; + "~*(?:\b)Ripper(?:\b|)" 3; + "~*(?:\b)RocketCrawler(?:\b|)" 3; + "~*(?:\b)Rogerbot(?:\b|)" 3; + "~*(?:\b)s1z.ru(?:\b|)" 3; + "~*(?:\b)SalesIntelligent(?:\b|)" 3; + "~*(?:\b)SBIder(?:\b|)" 3; + "~*(?:\b)ScanAlert(?:\b|)" 3; + "~*(?:\b)Scanbot(?:\b|)" 3; + "~*(?:\b)scan.lol(?:\b|)" 3; + "~*(?:\b)ScoutJet(?:\b|)" 3; + "~*(?:\b)Scrapy(?:\b|)" 3; + "~*(?:\b)Screaming(?:\b|)" 3; + "~*(?:\b)ScreenerBot(?:\b|)" 3; + "~*(?:\b)Searchestate(?:\b|)" 3; + "~*(?:\b)SearchmetricsBot(?:\b|)" 3; + "~*(?:\b)Semrush(?:\b|)" 3; + "~*(?:\b)SemrushBot(?:\b|)" 3; + "~*(?:\b)SEOkicks(?:\b|)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; + "~*(?:\b)Seomoz(?:\b|)" 3; + "~*(?:\b)SEOprofiler(?:\b|)" 3; + "~*(?:\b)seoscanners(?:\b|)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; + "~*(?:\b)SEOstats(?:\b|)" 3; + "~*(?:\b)serpstatbot(?:\b|)" 3; + "~*(?:\b)sexsearcher(?:\b|)" 3; + "~*(?:\b)Shodan(?:\b|)" 3; + "~*(?:\b)Siphon(?:\b|)" 3; + "~*(?:\b)SISTRIX(?:\b|)" 3; + "~*(?:\b)Sitebeam(?:\b|)" 3; + "~*(?:\b)SiteExplorer(?:\b|)" 3; + "~*(?:\b)Siteimprove(?:\b|)" 3; + "~*(?:\b)SiteLockSpider(?:\b|)" 3; + "~*(?:\b)SiteSnagger(?:\b|)" 3; + "~*(?:\b)SiteSucker(?:\b|)" 3; + "~*(?:\b)Site\ Sucker(?:\b|)" 3; + "~*(?:\b)Sitevigil(?:\b|)" 3; + "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 3; + "~*(?:\b)SlySearch(?:\b|)" 3; + "~*(?:\b)SmartDownload(?:\b|)" 3; + "~*(?:\b)SMTBot(?:\b|)" 3; + "~*(?:\b)Snake(?:\b|)" 3; + "~*(?:\b)Snapbot(?:\b|)" 3; + "~*(?:\b)Snoopy(?:\b|)" 3; + "~*(?:\b)SocialRankIOBot(?:\b|)" 3; + "~*(?:\b)Sociscraper(?:\b|)" 3; + "~*(?:\b)sogouspider(?:\b|)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; + "~*(?:\b)Sosospider(?:\b|)" 3; + "~*(?:\b)Sottopop(?:\b|)" 3; + "~*(?:\b)SpaceBison(?:\b|)" 3; + "~*(?:\b)Spammen(?:\b|)" 3; + "~*(?:\b)SpankBot(?:\b|)" 3; + "~*(?:\b)Spanner(?:\b|)" 3; + "~*(?:\b)sp_auditbot(?:\b|)" 3; + "~*(?:\b)Spbot(?:\b|)" 3; + "~*(?:\b)Spinn3r(?:\b|)" 3; + "~*(?:\b)SputnikBot(?:\b|)" 3; + "~*(?:\b)spyfu(?:\b|)" 3; + "~*(?:\b)Sqlmap(?:\b|)" 3; + "~*(?:\b)Sqlworm(?:\b|)" 3; + "~*(?:\b)Sqworm(?:\b|)" 3; + "~*(?:\b)Steeler(?:\b|)" 3; + "~*(?:\b)Stripper(?:\b|)" 3; + "~*(?:\b)Sucker(?:\b|)" 3; + "~*(?:\b)Sucuri(?:\b|)" 3; + "~*(?:\b)SuperBot(?:\b|)" 3; + "~*(?:\b)SuperHTTP(?:\b|)" 3; + "~*(?:\b)Surfbot(?:\b|)" 3; + "~*(?:\b)SurveyBot(?:\b|)" 3; + "~*(?:\b)Suzuran(?:\b|)" 3; + "~*(?:\b)Swiftbot(?:\b|)" 3; + "~*(?:\b)sysscan(?:\b|)" 3; + "~*(?:\b)Szukacz(?:\b|)" 3; + "~*(?:\b)T0PHackTeam(?:\b|)" 3; + "~*(?:\b)T8Abot(?:\b|)" 3; + "~*(?:\b)tAkeOut(?:\b|)" 3; + "~*(?:\b)Teleport(?:\b|)" 3; + "~*(?:\b)TeleportPro(?:\b|)" 3; + "~*(?:\b)Telesoft(?:\b|)" 3; + "~*(?:\b)Telesphoreo(?:\b|)" 3; + "~*(?:\b)Telesphorep(?:\b|)" 3; + "~*(?:\b)The\ Intraformant(?:\b|)" 3; + "~*(?:\b)TheNomad(?:\b|)" 3; + "~*(?:\b)TightTwatBot(?:\b|)" 3; + "~*(?:\b)Titan(?:\b|)" 3; + "~*(?:\b)Toata(?:\b|)" 3; + "~*(?:\b)Toweyabot(?:\b|)" 3; + "~*(?:\b)Tracemyfile(?:\b|)" 3; + "~*(?:\b)Trendiction(?:\b|)" 3; + "~*(?:\b)Trendictionbot(?:\b|)" 3; + "~*(?:\b)trendiction.com(?:\b|)" 3; + "~*(?:\b)trendiction.de(?:\b|)" 3; + "~*(?:\b)True_Robot(?:\b|)" 3; + "~*(?:\b)Turingos(?:\b|)" 3; + "~*(?:\b)Turnitin(?:\b|)" 3; + "~*(?:\b)TurnitinBot(?:\b|)" 3; + "~*(?:\b)TwengaBot(?:\b|)" 3; + "~*(?:\b)Twice(?:\b|)" 3; + "~*(?:\b)Typhoeus(?:\b|)" 3; + "~*(?:\b)UnisterBot(?:\b|)" 3; + "~*(?:\b)Upflow(?:\b|)" 3; + "~*(?:\b)URLy.Warning(?:\b|)" 3; + "~*(?:\b)URLy\ Warning(?:\b|)" 3; + "~*(?:\b)Vacuum(?:\b|)" 3; + "~*(?:\b)Vagabondo(?:\b|)" 3; + "~*(?:\b)VB\ Project(?:\b|)" 3; + "~*(?:\b)VCI(?:\b|)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; + "~*(?:\b)VidibleScraper(?:\b|)" 3; + "~*(?:\b)Virusdie(?:\b|)" 3; + "~*(?:\b)VoidEYE(?:\b|)" 3; + "~*(?:\b)Voil(?:\b|)" 3; + "~*(?:\b)Voltron(?:\b|)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; + "~*(?:\b)WallpapersHD(?:\b|)" 3; + "~*(?:\b)WASALive-Bot(?:\b|)" 3; + "~*(?:\b)WBSearchBot(?:\b|)" 3; + "~*(?:\b)Webalta(?:\b|)" 3; + "~*(?:\b)WebAuto(?:\b|)" 3; + "~*(?:\b)Web\ Auto(?:\b|)" 3; + "~*(?:\b)WebBandit(?:\b|)" 3; + "~*(?:\b)WebCollage(?:\b|)" 3; + "~*(?:\b)Web\ Collage(?:\b|)" 3; + "~*(?:\b)WebCopier(?:\b|)" 3; + "~*(?:\b)WEBDAV(?:\b|)" 3; + "~*(?:\b)WebEnhancer(?:\b|)" 3; + "~*(?:\b)Web\ Enhancer(?:\b|)" 3; + "~*(?:\b)WebFetch(?:\b|)" 3; + "~*(?:\b)Web\ Fetch(?:\b|)" 3; + "~*(?:\b)WebFuck(?:\b|)" 3; + "~*(?:\b)Web\ Fuck(?:\b|)" 3; + "~*(?:\b)WebGo\ IS(?:\b|)" 3; + "~*(?:\b)WebImageCollector(?:\b|)" 3; + "~*(?:\b)WebLeacher(?:\b|)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; + "~*(?:\b)webmeup-crawler(?:\b|)" 3; + "~*(?:\b)WebPix(?:\b|)" 3; + "~*(?:\b)Web\ Pix(?:\b|)" 3; + "~*(?:\b)WebReaper(?:\b|)" 3; + "~*(?:\b)WebSauger(?:\b|)" 3; + "~*(?:\b)Web\ Sauger(?:\b|)" 3; + "~*(?:\b)Webshag(?:\b|)" 3; + "~*(?:\b)WebsiteExtractor(?:\b|)" 3; + "~*(?:\b)WebsiteQuester(?:\b|)" 3; + "~*(?:\b)Website\ Quester(?:\b|)" 3; + "~*(?:\b)Webster(?:\b|)" 3; + "~*(?:\b)WebStripper(?:\b|)" 3; + "~*(?:\b)WebSucker(?:\b|)" 3; + "~*(?:\b)Web\ Sucker(?:\b|)" 3; + "~*(?:\b)WebWhacker(?:\b|)" 3; + "~*(?:\b)WebZIP(?:\b|)" 3; + "~*(?:\b)WeSEE(?:\b|)" 3; + "~*(?:\b)Whack(?:\b|)" 3; + "~*(?:\b)Whacker(?:\b|)" 3; + "~*(?:\b)Whatweb(?:\b|)" 3; + "~*(?:\b)Who.is\ Bot(?:\b|)" 3; + "~*(?:\b)Widow(?:\b|)" 3; + "~*(?:\b)WinHTTrack(?:\b|)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; + "~*(?:\b)WISENutbot(?:\b|)" 3; + "~*(?:\b)Wonderbot(?:\b|)" 3; + "~*(?:\b)Woobot(?:\b|)" 3; + "~*(?:\b)Wotbox(?:\b|)" 3; + "~*(?:\b)Wprecon(?:\b|)" 3; + "~*(?:\b)WPScan(?:\b|)" 3; + "~*(?:\b)WWW-Collector-E(?:\b|)" 3; + "~*(?:\b)WWW-Mechanize(?:\b|)" 3; + "~*(?:\b)WWW::Mechanize(?:\b|)" 3; + "~*(?:\b)WWWOFFLE(?:\b|)" 3; + "~*(?:\b)x09Mozilla(?:\b|)" 3; + "~*(?:\b)x22Mozilla(?:\b|)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; + "~*(?:\b)Xenu(?:\b|)" 3; + "~*(?:\b)xpymep1.exe(?:\b|)" 3; + "~*(?:\b)YoudaoBot(?:\b|)" 3; + "~*(?:\b)Zade(?:\b|)" 3; + "~*(?:\b)Zauba(?:\b|)" 3; + "~*(?:\b)zauba.io(?:\b|)" 3; + "~*(?:\b)Zermelo(?:\b|)" 3; + "~*(?:\b)Zeus(?:\b|)" 3; + "~*(?:\b)zgrab(?:\b|)" 3; + "~*(?:\b)Zitebot(?:\b|)" 3; + "~*(?:\b)ZmEu(?:\b|)" 3; + "~*(?:\b)ZumBot(?:\b|)" 3; + "~*(?:\b)ZyBorg(?:\b|)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # *********************************************** diff --git a/.dev-tools/_curl_tests/curltest1.txt b/.dev-tools/_curl_tests/curltest1.txt index 1ee7fa89b..31349d9f5 100755 --- a/.dev-tools/_curl_tests/curltest1.txt +++ b/.dev-tools/_curl_tests/curltest1.txt @@ -1,4 +1,4 @@ -Last Tested: Fri Jun 21 11:40:57 SAST 2019 +Last Tested: Fri Jun 21 13:54:53 SAST 2019 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest2.txt b/.dev-tools/_curl_tests/curltest2.txt index 1ee7fa89b..31349d9f5 100755 --- a/.dev-tools/_curl_tests/curltest2.txt +++ b/.dev-tools/_curl_tests/curltest2.txt @@ -1,4 +1,4 @@ -Last Tested: Fri Jun 21 11:40:57 SAST 2019 +Last Tested: Fri Jun 21 13:54:53 SAST 2019 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest3.txt b/.dev-tools/_curl_tests/curltest3.txt index 1ee7fa89b..31349d9f5 100755 --- a/.dev-tools/_curl_tests/curltest3.txt +++ b/.dev-tools/_curl_tests/curltest3.txt @@ -1,4 +1,4 @@ -Last Tested: Fri Jun 21 11:40:57 SAST 2019 +Last Tested: Fri Jun 21 13:54:53 SAST 2019 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest4.txt b/.dev-tools/_curl_tests/curltest4.txt index 1ee7fa89b..31349d9f5 100755 --- a/.dev-tools/_curl_tests/curltest4.txt +++ b/.dev-tools/_curl_tests/curltest4.txt @@ -1,4 +1,4 @@ -Last Tested: Fri Jun 21 11:40:57 SAST 2019 +Last Tested: Fri Jun 21 13:54:53 SAST 2019 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest5.txt b/.dev-tools/_curl_tests/curltest5.txt index 330633a6e..64a027c95 100755 --- a/.dev-tools/_curl_tests/curltest5.txt +++ b/.dev-tools/_curl_tests/curltest5.txt @@ -1,4 +1,4 @@ -Last Tested: Fri Jun 21 11:40:57 SAST 2019 +Last Tested: Fri Jun 21 13:54:53 SAST 2019