diff --git a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf index b6d7e67e9..96f57ae80 100644 --- a/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1709 -### Updated: Mon Jul 1 10:37:05 SAST 2019 +### Version: V3.2019.07.1712 +### Updated: Mon Jul 1 11:24:53 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 6893ce0fc..b9d24324a 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 1; +34.68.144.114 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 879967dc9..ea117da9f 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 0; +34.68.144.114 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index b6d7e67e9..96f57ae80 100644 --- a/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1709 -### Updated: Mon Jul 1 10:37:05 SAST 2019 +### Version: V3.2019.07.1712 +### Updated: Mon Jul 1 11:24:53 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 6893ce0fc..b9d24324a 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 1; +34.68.144.114 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 879967dc9..ea117da9f 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 0; +34.68.144.114 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index b6d7e67e9..96f57ae80 100644 --- a/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1709 -### Updated: Mon Jul 1 10:37:05 SAST 2019 +### Version: V3.2019.07.1712 +### Updated: Mon Jul 1 11:24:53 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 6893ce0fc..b9d24324a 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 1; +34.68.144.114 1; 127.0.0.1 1; diff --git a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 879967dc9..ea117da9f 100644 --- a/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 0; +34.68.144.114 0; 127.0.0.1 0; diff --git a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index b6d7e67e9..96f57ae80 100644 --- a/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1709 -### Updated: Mon Jul 1 10:37:05 SAST 2019 +### Version: V3.2019.07.1712 +### Updated: Mon Jul 1 11:24:53 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist-testing.template b/.dev-tools/globalblacklist-testing.template index b6d7e67e9..96f57ae80 100644 --- a/.dev-tools/globalblacklist-testing.template +++ b/.dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.07.1709 -### Updated: Mon Jul 1 10:37:05 SAST 2019 +### Version: V3.2019.07.1712 +### Updated: Mon Jul 1 11:24:53 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### diff --git a/.dev-tools/globalblacklist.template b/.dev-tools/globalblacklist.template index f26e52add..a1999739a 100755 --- a/.dev-tools/globalblacklist.template +++ b/.dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1709 -### Updated: Mon Jul 1 10:37:04 SAST 2019 +### Version: V4.2019.07.1712 +### Updated: Mon Jul 1 11:24:52 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### @@ -147,562 +147,562 @@ map $http_user_agent $bad_bot { # -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b|)" 3; - "~*(?:\b)404checker(?:\b|)" 3; - "~*(?:\b)404enemy(?:\b|)" 3; - "~*(?:\b)80legs(?:\b|)" 3; - "~*(?:\b)Abonti(?:\b|)" 3; - "~*(?:\b)Aboundex(?:\b|)" 3; - "~*(?:\b)Aboundexbot(?:\b|)" 3; - "~*(?:\b)Acunetix(?:\b|)" 3; - "~*(?:\b)ADmantX(?:\b|)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; - "~*(?:\b)AhrefsBot(?:\b|)" 3; - "~*(?:\b)AIBOT(?:\b|)" 3; - "~*(?:\b)AiHitBot(?:\b|)" 3; - "~*(?:\b)Aipbot(?:\b|)" 3; - "~*(?:\b)Alexibot(?:\b|)" 3; - "~*(?:\b)Alligator(?:\b|)" 3; - "~*(?:\b)AllSubmitter(?:\b|)" 3; - "~*(?:\b)AlphaBot(?:\b|)" 3; - "~*(?:\b)Anarchie(?:\b|)" 3; - "~*(?:\b)Apexoo(?:\b|)" 3; - "~*(?:\b)archive.org_bot(?:\b|)" 3; - "~*(?:\b)ASPSeek(?:\b|)" 3; - "~*(?:\b)Asterias(?:\b|)" 3; - "~*(?:\b)Attach(?:\b|)" 3; - "~*(?:\b)autoemailspider(?:\b|)" 3; - "~*(?:\b)BackDoorBot(?:\b|)" 3; - "~*(?:\b)Backlink-Ceck(?:\b|)" 3; - "~*(?:\b)backlink-check(?:\b|)" 3; - "~*(?:\b)BacklinkCrawler(?:\b|)" 3; - "~*(?:\b)BackStreet(?:\b|)" 3; - "~*(?:\b)BackWeb(?:\b|)" 3; - "~*(?:\b)Badass(?:\b|)" 3; - "~*(?:\b)Bandit(?:\b|)" 3; - "~*(?:\b)Barkrowler(?:\b|)" 3; - "~*(?:\b)BatchFTP(?:\b|)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; - "~*(?:\b)BBBike(?:\b|)" 3; - "~*(?:\b)BDCbot(?:\b|)" 3; - "~*(?:\b)BDFetch(?:\b|)" 3; - "~*(?:\b)BetaBot(?:\b|)" 3; - "~*(?:\b)Bigfoot(?:\b|)" 3; - "~*(?:\b)Bitacle(?:\b|)" 3; - "~*(?:\b)Blackboard(?:\b|)" 3; - "~*(?:\b)Black\ Hole(?:\b|)" 3; - "~*(?:\b)BlackWidow(?:\b|)" 3; - "~*(?:\b)BLEXBot(?:\b|)" 3; - "~*(?:\b)Blow(?:\b|)" 3; - "~*(?:\b)BlowFish(?:\b|)" 3; - "~*(?:\b)Boardreader(?:\b|)" 3; - "~*(?:\b)Bolt(?:\b|)" 3; - "~*(?:\b)BotALot(?:\b|)" 3; - "~*(?:\b)Brandprotect(?:\b|)" 3; - "~*(?:\b)Brandwatch(?:\b|)" 3; - "~*(?:\b)Buddy(?:\b|)" 3; - "~*(?:\b)BuiltBotTough(?:\b|)" 3; - "~*(?:\b)BuiltWith(?:\b|)" 3; - "~*(?:\b)Bullseye(?:\b|)" 3; - "~*(?:\b)BunnySlippers(?:\b|)" 3; - "~*(?:\b)BuzzSumo(?:\b|)" 3; - "~*(?:\b)Calculon(?:\b|)" 3; - "~*(?:\b)CATExplorador(?:\b|)" 3; - "~*(?:\b)CazoodleBot(?:\b|)" 3; - "~*(?:\b)CCBot(?:\b|)" 3; - "~*(?:\b)Cegbfeieh(?:\b|)" 3; - "~*(?:\b)CheeseBot(?:\b|)" 3; - "~*(?:\b)CherryPicker(?:\b|)" 3; - "~*(?:\b)CheTeam(?:\b|)" 3; - "~*(?:\b)ChinaClaw(?:\b|)" 3; - "~*(?:\b)Chlooe(?:\b|)" 3; - "~*(?:\b)Claritybot(?:\b|)" 3; - "~*(?:\b)Cliqzbot(?:\b|)" 3; - "~*(?:\b)Cloud\ mapping(?:\b|)" 3; - "~*(?:\b)coccocbot-web(?:\b|)" 3; - "~*(?:\b)Cogentbot(?:\b|)" 3; - "~*(?:\b)cognitiveseo(?:\b|)" 3; - "~*(?:\b)Collector(?:\b|)" 3; - "~*(?:\b)com.plumanalytics(?:\b|)" 3; - "~*(?:\b)Copier(?:\b|)" 3; - "~*(?:\b)CopyRightCheck(?:\b|)" 3; - "~*(?:\b)Copyscape(?:\b|)" 3; - "~*(?:\b)Cosmos(?:\b|)" 3; - "~*(?:\b)Craftbot(?:\b|)" 3; - "~*(?:\b)crawler4j(?:\b|)" 3; - "~*(?:\b)crawler.feedback(?:\b|)" 3; - "~*(?:\b)crawl.sogou.com(?:\b|)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; - "~*(?:\b)Crescent(?:\b|)" 3; - "~*(?:\b)CrunchBot(?:\b|)" 3; - "~*(?:\b)CSHttp(?:\b|)" 3; - "~*(?:\b)Curious(?:\b|)" 3; - "~*(?:\b)Custo(?:\b|)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; - "~*(?:\b)DataCha0s(?:\b|)" 3; - "~*(?:\b)DBLBot(?:\b|)" 3; - "~*(?:\b)demandbase-bot(?:\b|)" 3; - "~*(?:\b)Demon(?:\b|)" 3; - "~*(?:\b)Deusu(?:\b|)" 3; - "~*(?:\b)Devil(?:\b|)" 3; - "~*(?:\b)Digincore(?:\b|)" 3; - "~*(?:\b)DigitalPebble(?:\b|)" 3; - "~*(?:\b)DIIbot(?:\b|)" 3; - "~*(?:\b)Dirbuster(?:\b|)" 3; - "~*(?:\b)Disco(?:\b|)" 3; - "~*(?:\b)Discobot(?:\b|)" 3; - "~*(?:\b)Discoverybot(?:\b|)" 3; - "~*(?:\b)Dispatch(?:\b|)" 3; - "~*(?:\b)DittoSpyder(?:\b|)" 3; - "~*(?:\b)DnyzBot(?:\b|)" 3; - "~*(?:\b)DomainAppender(?:\b|)" 3; - "~*(?:\b)DomainCrawler(?:\b|)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; - "~*(?:\b)DomainStatsBot(?:\b|)" 3; - "~*(?:\b)Dotbot(?:\b|)" 3; - "~*(?:\b)Download\ Wonder(?:\b|)" 3; - "~*(?:\b)Dragonfly(?:\b|)" 3; - "~*(?:\b)Drip(?:\b|)" 3; - "~*(?:\b)DSearch(?:\b|)" 3; - "~*(?:\b)DTS\ Agent(?:\b|)" 3; - "~*(?:\b)EasyDL(?:\b|)" 3; - "~*(?:\b)Ebingbong(?:\b|)" 3; - "~*(?:\b)eCatch(?:\b|)" 3; - "~*(?:\b)ECCP/1.0(?:\b|)" 3; - "~*(?:\b)Ecxi(?:\b|)" 3; - "~*(?:\b)EirGrabber(?:\b|)" 3; - "~*(?:\b)EMail\ Siphon(?:\b|)" 3; - "~*(?:\b)EMail\ Wolf(?:\b|)" 3; - "~*(?:\b)EroCrawler(?:\b|)" 3; - "~*(?:\b)evc-batch(?:\b|)" 3; - "~*(?:\b)Evil(?:\b|)" 3; - "~*(?:\b)Exabot(?:\b|)" 3; - "~*(?:\b)Express\ WebPictures(?:\b|)" 3; - "~*(?:\b)ExtLinksBot(?:\b|)" 3; - "~*(?:\b)Extractor(?:\b|)" 3; - "~*(?:\b)ExtractorPro(?:\b|)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; - "~*(?:\b)EyeNetIE(?:\b|)" 3; - "~*(?:\b)Ezooms(?:\b|)" 3; - "~*(?:\b)facebookscraper(?:\b|)" 3; - "~*(?:\b)FDM(?:\b|)" 3; - "~*(?:\b)FemtosearchBot(?:\b|)" 3; - "~*(?:\b)FHscan(?:\b|)" 3; - "~*(?:\b)Fimap(?:\b|)" 3; - "~*(?:\b)Firefox/7.0(?:\b|)" 3; - "~*(?:\b)FlashGet(?:\b|)" 3; - "~*(?:\b)Flunky(?:\b|)" 3; - "~*(?:\b)Foobot(?:\b|)" 3; - "~*(?:\b)Freeuploader(?:\b|)" 3; - "~*(?:\b)FrontPage(?:\b|)" 3; - "~*(?:\b)FyberSpider(?:\b|)" 3; - "~*(?:\b)Fyrebot(?:\b|)" 3; - "~*(?:\b)GalaxyBot(?:\b|)" 3; - "~*(?:\b)Genieo(?:\b|)" 3; - "~*(?:\b)GermCrawler(?:\b|)" 3; - "~*(?:\b)Getintent(?:\b|)" 3; - "~*(?:\b)GetRight(?:\b|)" 3; - "~*(?:\b)GetWeb(?:\b|)" 3; - "~*(?:\b)Gigablast(?:\b|)" 3; - "~*(?:\b)Gigabot(?:\b|)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; - "~*(?:\b)Gotit(?:\b|)" 3; - "~*(?:\b)GoZilla(?:\b|)" 3; - "~*(?:\b)Go!Zilla(?:\b|)" 3; - "~*(?:\b)Grabber(?:\b|)" 3; - "~*(?:\b)GrabNet(?:\b|)" 3; - "~*(?:\b)Grafula(?:\b|)" 3; - "~*(?:\b)GrapeFX(?:\b|)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; - "~*(?:\b)GridBot(?:\b|)" 3; - "~*(?:\b)GT::WWW(?:\b|)" 3; - "~*(?:\b)Haansoft(?:\b|)" 3; - "~*(?:\b)HaosouSpider(?:\b|)" 3; - "~*(?:\b)Harvest(?:\b|)" 3; - "~*(?:\b)Havij(?:\b|)" 3; - "~*(?:\b)HEADMasterSEO(?:\b|)" 3; - "~*(?:\b)Heritrix(?:\b|)" 3; - "~*(?:\b)Hloader(?:\b|)" 3; - "~*(?:\b)HMView(?:\b|)" 3; - "~*(?:\b)HTMLparser(?:\b|)" 3; - "~*(?:\b)HTTP::Lite(?:\b|)" 3; - "~*(?:\b)HTTrack(?:\b|)" 3; - "~*(?:\b)Humanlinks(?:\b|)" 3; - "~*(?:\b)HybridBot(?:\b|)" 3; - "~*(?:\b)Iblog(?:\b|)" 3; - "~*(?:\b)IDBot(?:\b|)" 3; - "~*(?:\b)Id-search(?:\b|)" 3; - "~*(?:\b)IlseBot(?:\b|)" 3; - "~*(?:\b)Image\ Fetch(?:\b|)" 3; - "~*(?:\b)Image\ Sucker(?:\b|)" 3; - "~*(?:\b)IndeedBot(?:\b|)" 3; - "~*(?:\b)Indy\ Library(?:\b|)" 3; - "~*(?:\b)InfoNaviRobot(?:\b|)" 3; - "~*(?:\b)InfoTekies(?:\b|)" 3; - "~*(?:\b)instabid(?:\b|)" 3; - "~*(?:\b)Intelliseek(?:\b|)" 3; - "~*(?:\b)InterGET(?:\b|)" 3; - "~*(?:\b)Internet\ Ninja(?:\b|)" 3; - "~*(?:\b)InternetSeer(?:\b|)" 3; - "~*(?:\b)internetVista\ monitor(?:\b|)" 3; - "~*(?:\b)ips-agent(?:\b|)" 3; - "~*(?:\b)Iria(?:\b|)" 3; - "~*(?:\b)IRLbot(?:\b|)" 3; - "~*(?:\b)Iskanie(?:\b|)" 3; - "~*(?:\b)IstellaBot(?:\b|)" 3; - "~*(?:\b)JamesBOT(?:\b|)" 3; - "~*(?:\b)Jbrofuzz(?:\b|)" 3; - "~*(?:\b)JennyBot(?:\b|)" 3; - "~*(?:\b)JetCar(?:\b|)" 3; - "~*(?:\b)Jetty(?:\b|)" 3; - "~*(?:\b)JikeSpider(?:\b|)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; - "~*(?:\b)Joomla(?:\b|)" 3; - "~*(?:\b)Jorgee(?:\b|)" 3; - "~*(?:\b)JustView(?:\b|)" 3; - "~*(?:\b)Jyxobot(?:\b|)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; - "~*(?:\b)Keyword\ Density(?:\b|)" 3; - "~*(?:\b)Kozmosbot(?:\b|)" 3; - "~*(?:\b)Lanshanbot(?:\b|)" 3; - "~*(?:\b)Larbin(?:\b|)" 3; - "~*(?:\b)LeechFTP(?:\b|)" 3; - "~*(?:\b)LeechGet(?:\b|)" 3; - "~*(?:\b)LexiBot(?:\b|)" 3; - "~*(?:\b)Lftp(?:\b|)" 3; - "~*(?:\b)LibWeb(?:\b|)" 3; - "~*(?:\b)Libwhisker(?:\b|)" 3; - "~*(?:\b)Lightspeedsystems(?:\b|)" 3; - "~*(?:\b)Likse(?:\b|)" 3; - "~*(?:\b)Linkdexbot(?:\b|)" 3; - "~*(?:\b)LinkextractorPro(?:\b|)" 3; - "~*(?:\b)LinkpadBot(?:\b|)" 3; - "~*(?:\b)LinkScan(?:\b|)" 3; - "~*(?:\b)LinksManager(?:\b|)" 3; - "~*(?:\b)LinkWalker(?:\b|)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; - "~*(?:\b)Lipperhey(?:\b|)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; - "~*(?:\b)Litemage_walker(?:\b|)" 3; - "~*(?:\b)Lmspider(?:\b|)" 3; - "~*(?:\b)LNSpiderguy(?:\b|)" 3; - "~*(?:\b)Ltx71(?:\b|)" 3; - "~*(?:\b)lwp-request(?:\b|)" 3; - "~*(?:\b)LWP::Simple(?:\b|)" 3; - "~*(?:\b)lwp-trivial(?:\b|)" 3; - "~*(?:\b)Magnet(?:\b|)" 3; - "~*(?:\b)Mag-Net(?:\b|)" 3; - "~*(?:\b)magpie-crawler(?:\b|)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; - "~*(?:\b)Majestic12(?:\b|)" 3; - "~*(?:\b)Majestic-SEO(?:\b|)" 3; - "~*(?:\b)Majestic\ SEO(?:\b|)" 3; - "~*(?:\b)MarkMonitor(?:\b|)" 3; - "~*(?:\b)MarkWatch(?:\b|)" 3; - "~*(?:\b)Masscan(?:\b|)" 3; - "~*(?:\b)Mass\ Downloader(?:\b|)" 3; - "~*(?:\b)Mata\ Hari(?:\b|)" 3; - "~*(?:\b)MauiBot(?:\b|)" 3; - "~*(?:\b)meanpathbot(?:\b|)" 3; - "~*(?:\b)Meanpathbot(?:\b|)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; - "~*(?:\b)mediawords(?:\b|)" 3; - "~*(?:\b)MegaIndex.ru(?:\b|)" 3; - "~*(?:\b)Metauri(?:\b|)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; - "~*(?:\b)MIDown\ tool(?:\b|)" 3; - "~*(?:\b)MIIxpc(?:\b|)" 3; - "~*(?:\b)Mister\ PiX(?:\b|)" 3; - "~*(?:\b)MJ12bot(?:\b|)" 3; - "~*(?:\b)Mojeek(?:\b|)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; - "~*(?:\b)Mr.4x3(?:\b|)" 3; - "~*(?:\b)MSFrontPage(?:\b|)" 3; - "~*(?:\b)MSIECrawler(?:\b|)" 3; - "~*(?:\b)Msrabot(?:\b|)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; - "~*(?:\b)muhstik-scan(?:\b|)" 3; - "~*(?:\b)Musobot(?:\b|)" 3; - "~*(?:\b)Name\ Intelligence(?:\b|)" 3; - "~*(?:\b)Nameprotect(?:\b|)" 3; - "~*(?:\b)Navroad(?:\b|)" 3; - "~*(?:\b)NearSite(?:\b|)" 3; - "~*(?:\b)Needle(?:\b|)" 3; - "~*(?:\b)Nessus(?:\b|)" 3; - "~*(?:\b)NetAnts(?:\b|)" 3; - "~*(?:\b)Netcraft(?:\b|)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; - "~*(?:\b)NetLyzer(?:\b|)" 3; - "~*(?:\b)NetMechanic(?:\b|)" 3; - "~*(?:\b)NetSpider(?:\b|)" 3; - "~*(?:\b)Nettrack(?:\b|)" 3; - "~*(?:\b)Net\ Vampire(?:\b|)" 3; - "~*(?:\b)Netvibes(?:\b|)" 3; - "~*(?:\b)NetZIP(?:\b|)" 3; - "~*(?:\b)NextGenSearchBot(?:\b|)" 3; - "~*(?:\b)Nibbler(?:\b|)" 3; - "~*(?:\b)NICErsPRO(?:\b|)" 3; - "~*(?:\b)Niki-bot(?:\b|)" 3; - "~*(?:\b)Nikto(?:\b|)" 3; - "~*(?:\b)NimbleCrawler(?:\b|)" 3; - "~*(?:\b)Nimbostratus(?:\b|)" 3; - "~*(?:\b)Ninja(?:\b|)" 3; - "~*(?:\b)Nmap(?:\b|)" 3; - "~*(?:\b)NPbot(?:\b|)" 3; - "~*(?:\b)Nutch(?:\b|)" 3; - "~*(?:\b)oBot(?:\b|)" 3; - "~*(?:\b)Octopus(?:\b|)" 3; - "~*(?:\b)Offline\ Explorer(?:\b|)" 3; - "~*(?:\b)Offline\ Navigator(?:\b|)" 3; - "~*(?:\b)OnCrawl(?:\b|)" 3; - "~*(?:\b)Openfind(?:\b|)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; - "~*(?:\b)Openvas(?:\b|)" 3; - "~*(?:\b)OpenVAS(?:\b|)" 3; - "~*(?:\b)OrangeBot(?:\b|)" 3; - "~*(?:\b)OrangeSpider(?:\b|)" 3; - "~*(?:\b)OutclicksBot(?:\b|)" 3; - "~*(?:\b)OutfoxBot(?:\b|)" 3; - "~*(?:\b)PageAnalyzer(?:\b|)" 3; - "~*(?:\b)Page\ Analyzer(?:\b|)" 3; - "~*(?:\b)PageGrabber(?:\b|)" 3; - "~*(?:\b)page\ scorer(?:\b|)" 3; - "~*(?:\b)PageScorer(?:\b|)" 3; - "~*(?:\b)Pandalytics(?:\b|)" 3; - "~*(?:\b)Panscient(?:\b|)" 3; - "~*(?:\b)Papa\ Foto(?:\b|)" 3; - "~*(?:\b)Pavuk(?:\b|)" 3; - "~*(?:\b)pcBrowser(?:\b|)" 3; - "~*(?:\b)PECL::HTTP(?:\b|)" 3; - "~*(?:\b)PeoplePal(?:\b|)" 3; - "~*(?:\b)PHPCrawl(?:\b|)" 3; - "~*(?:\b)Picscout(?:\b|)" 3; - "~*(?:\b)Picsearch(?:\b|)" 3; - "~*(?:\b)PictureFinder(?:\b|)" 3; - "~*(?:\b)Pimonster(?:\b|)" 3; - "~*(?:\b)Pi-Monster(?:\b|)" 3; - "~*(?:\b)Pixray(?:\b|)" 3; - "~*(?:\b)PleaseCrawl(?:\b|)" 3; - "~*(?:\b)plumanalytics(?:\b|)" 3; - "~*(?:\b)Pockey(?:\b|)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; - "~*(?:\b)Probethenet(?:\b|)" 3; - "~*(?:\b)ProPowerBot(?:\b|)" 3; - "~*(?:\b)ProWebWalker(?:\b|)" 3; - "~*(?:\b)Psbot(?:\b|)" 3; - "~*(?:\b)Pump(?:\b|)" 3; - "~*(?:\b)PxBroker(?:\b|)" 3; - "~*(?:\b)PyCurl(?:\b|)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; - "~*(?:\b)Quick-Crawler(?:\b|)" 3; - "~*(?:\b)RankActive(?:\b|)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; - "~*(?:\b)RankFlex(?:\b|)" 3; - "~*(?:\b)RankingBot(?:\b|)" 3; - "~*(?:\b)RankingBot2(?:\b|)" 3; - "~*(?:\b)Rankivabot(?:\b|)" 3; - "~*(?:\b)RankurBot(?:\b|)" 3; - "~*(?:\b)RealDownload(?:\b|)" 3; - "~*(?:\b)Reaper(?:\b|)" 3; - "~*(?:\b)RebelMouse(?:\b|)" 3; - "~*(?:\b)Recorder(?:\b|)" 3; - "~*(?:\b)RedesScrapy(?:\b|)" 3; - "~*(?:\b)ReGet(?:\b|)" 3; - "~*(?:\b)RepoMonkey(?:\b|)" 3; - "~*(?:\b)Ripper(?:\b|)" 3; - "~*(?:\b)RocketCrawler(?:\b|)" 3; - "~*(?:\b)Rogerbot(?:\b|)" 3; - "~*(?:\b)s1z.ru(?:\b|)" 3; - "~*(?:\b)SalesIntelligent(?:\b|)" 3; - "~*(?:\b)SBIder(?:\b|)" 3; - "~*(?:\b)ScanAlert(?:\b|)" 3; - "~*(?:\b)Scanbot(?:\b|)" 3; - "~*(?:\b)scan.lol(?:\b|)" 3; - "~*(?:\b)ScoutJet(?:\b|)" 3; - "~*(?:\b)Scrapy(?:\b|)" 3; - "~*(?:\b)Screaming(?:\b|)" 3; - "~*(?:\b)ScreenerBot(?:\b|)" 3; - "~*(?:\b)Searchestate(?:\b|)" 3; - "~*(?:\b)SearchmetricsBot(?:\b|)" 3; - "~*(?:\b)Semrush(?:\b|)" 3; - "~*(?:\b)SemrushBot(?:\b|)" 3; - "~*(?:\b)SEOkicks(?:\b|)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; - "~*(?:\b)Seomoz(?:\b|)" 3; - "~*(?:\b)SEOprofiler(?:\b|)" 3; - "~*(?:\b)seoscanners(?:\b|)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; - "~*(?:\b)SEOstats(?:\b|)" 3; - "~*(?:\b)serpstatbot(?:\b|)" 3; - "~*(?:\b)sexsearcher(?:\b|)" 3; - "~*(?:\b)Shodan(?:\b|)" 3; - "~*(?:\b)Siphon(?:\b|)" 3; - "~*(?:\b)SISTRIX(?:\b|)" 3; - "~*(?:\b)Sitebeam(?:\b|)" 3; - "~*(?:\b)SiteExplorer(?:\b|)" 3; - "~*(?:\b)Siteimprove(?:\b|)" 3; - "~*(?:\b)SiteLockSpider(?:\b|)" 3; - "~*(?:\b)SiteSnagger(?:\b|)" 3; - "~*(?:\b)SiteSucker(?:\b|)" 3; - "~*(?:\b)Site\ Sucker(?:\b|)" 3; - "~*(?:\b)Sitevigil(?:\b|)" 3; - "~*(?:\b)SlySearch(?:\b|)" 3; - "~*(?:\b)SmartDownload(?:\b|)" 3; - "~*(?:\b)SMTBot(?:\b|)" 3; - "~*(?:\b)Snake(?:\b|)" 3; - "~*(?:\b)Snapbot(?:\b|)" 3; - "~*(?:\b)Snoopy(?:\b|)" 3; - "~*(?:\b)SocialRankIOBot(?:\b|)" 3; - "~*(?:\b)Sociscraper(?:\b|)" 3; - "~*(?:\b)sogouspider(?:\b|)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; - "~*(?:\b)Sosospider(?:\b|)" 3; - "~*(?:\b)Sottopop(?:\b|)" 3; - "~*(?:\b)SpaceBison(?:\b|)" 3; - "~*(?:\b)Spammen(?:\b|)" 3; - "~*(?:\b)SpankBot(?:\b|)" 3; - "~*(?:\b)Spanner(?:\b|)" 3; - "~*(?:\b)sp_auditbot(?:\b|)" 3; - "~*(?:\b)Spbot(?:\b|)" 3; - "~*(?:\b)Spinn3r(?:\b|)" 3; - "~*(?:\b)SputnikBot(?:\b|)" 3; - "~*(?:\b)spyfu(?:\b|)" 3; - "~*(?:\b)Sqlmap(?:\b|)" 3; - "~*(?:\b)Sqlworm(?:\b|)" 3; - "~*(?:\b)Sqworm(?:\b|)" 3; - "~*(?:\b)Steeler(?:\b|)" 3; - "~*(?:\b)Stripper(?:\b|)" 3; - "~*(?:\b)Sucker(?:\b|)" 3; - "~*(?:\b)Sucuri(?:\b|)" 3; - "~*(?:\b)SuperBot(?:\b|)" 3; - "~*(?:\b)SuperHTTP(?:\b|)" 3; - "~*(?:\b)Surfbot(?:\b|)" 3; - "~*(?:\b)SurveyBot(?:\b|)" 3; - "~*(?:\b)Suzuran(?:\b|)" 3; - "~*(?:\b)Swiftbot(?:\b|)" 3; - "~*(?:\b)sysscan(?:\b|)" 3; - "~*(?:\b)Szukacz(?:\b|)" 3; - "~*(?:\b)T0PHackTeam(?:\b|)" 3; - "~*(?:\b)T8Abot(?:\b|)" 3; - "~*(?:\b)tAkeOut(?:\b|)" 3; - "~*(?:\b)Teleport(?:\b|)" 3; - "~*(?:\b)TeleportPro(?:\b|)" 3; - "~*(?:\b)Telesoft(?:\b|)" 3; - "~*(?:\b)Telesphoreo(?:\b|)" 3; - "~*(?:\b)Telesphorep(?:\b|)" 3; - "~*(?:\b)The\ Intraformant(?:\b|)" 3; - "~*(?:\b)TheNomad(?:\b|)" 3; - "~*(?:\b)Thumbor(?:\b|)" 3; - "~*(?:\b)TightTwatBot(?:\b|)" 3; - "~*(?:\b)Titan(?:\b|)" 3; - "~*(?:\b)Toata(?:\b|)" 3; - "~*(?:\b)Toweyabot(?:\b|)" 3; - "~*(?:\b)Tracemyfile(?:\b|)" 3; - "~*(?:\b)Trendiction(?:\b|)" 3; - "~*(?:\b)Trendictionbot(?:\b|)" 3; - "~*(?:\b)trendiction.com(?:\b|)" 3; - "~*(?:\b)trendiction.de(?:\b|)" 3; - "~*(?:\b)True_Robot(?:\b|)" 3; - "~*(?:\b)Turingos(?:\b|)" 3; - "~*(?:\b)Turnitin(?:\b|)" 3; - "~*(?:\b)TurnitinBot(?:\b|)" 3; - "~*(?:\b)TwengaBot(?:\b|)" 3; - "~*(?:\b)Twice(?:\b|)" 3; - "~*(?:\b)Typhoeus(?:\b|)" 3; - "~*(?:\b)UnisterBot(?:\b|)" 3; - "~*(?:\b)Upflow(?:\b|)" 3; - "~*(?:\b)URLy.Warning(?:\b|)" 3; - "~*(?:\b)URLy\ Warning(?:\b|)" 3; - "~*(?:\b)Vacuum(?:\b|)" 3; - "~*(?:\b)Vagabondo(?:\b|)" 3; - "~*(?:\b)VB\ Project(?:\b|)" 3; - "~*(?:\b)VCI(?:\b|)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; - "~*(?:\b)VidibleScraper(?:\b|)" 3; - "~*(?:\b)Virusdie(?:\b|)" 3; - "~*(?:\b)VoidEYE(?:\b|)" 3; - "~*(?:\b)Voil(?:\b|)" 3; - "~*(?:\b)Voltron(?:\b|)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; - "~*(?:\b)WallpapersHD(?:\b|)" 3; - "~*(?:\b)WASALive-Bot(?:\b|)" 3; - "~*(?:\b)WBSearchBot(?:\b|)" 3; - "~*(?:\b)Webalta(?:\b|)" 3; - "~*(?:\b)WebAuto(?:\b|)" 3; - "~*(?:\b)Web\ Auto(?:\b|)" 3; - "~*(?:\b)WebBandit(?:\b|)" 3; - "~*(?:\b)WebCollage(?:\b|)" 3; - "~*(?:\b)Web\ Collage(?:\b|)" 3; - "~*(?:\b)WebCopier(?:\b|)" 3; - "~*(?:\b)WEBDAV(?:\b|)" 3; - "~*(?:\b)WebEnhancer(?:\b|)" 3; - "~*(?:\b)Web\ Enhancer(?:\b|)" 3; - "~*(?:\b)WebFetch(?:\b|)" 3; - "~*(?:\b)Web\ Fetch(?:\b|)" 3; - "~*(?:\b)WebFuck(?:\b|)" 3; - "~*(?:\b)Web\ Fuck(?:\b|)" 3; - "~*(?:\b)WebGo\ IS(?:\b|)" 3; - "~*(?:\b)WebImageCollector(?:\b|)" 3; - "~*(?:\b)WebLeacher(?:\b|)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; - "~*(?:\b)webmeup-crawler(?:\b|)" 3; - "~*(?:\b)WebPix(?:\b|)" 3; - "~*(?:\b)Web\ Pix(?:\b|)" 3; - "~*(?:\b)WebReaper(?:\b|)" 3; - "~*(?:\b)WebSauger(?:\b|)" 3; - "~*(?:\b)Web\ Sauger(?:\b|)" 3; - "~*(?:\b)Webshag(?:\b|)" 3; - "~*(?:\b)WebsiteExtractor(?:\b|)" 3; - "~*(?:\b)WebsiteQuester(?:\b|)" 3; - "~*(?:\b)Website\ Quester(?:\b|)" 3; - "~*(?:\b)Webster(?:\b|)" 3; - "~*(?:\b)WebStripper(?:\b|)" 3; - "~*(?:\b)WebSucker(?:\b|)" 3; - "~*(?:\b)Web\ Sucker(?:\b|)" 3; - "~*(?:\b)WebWhacker(?:\b|)" 3; - "~*(?:\b)WebZIP(?:\b|)" 3; - "~*(?:\b)WeSEE(?:\b|)" 3; - "~*(?:\b)Whack(?:\b|)" 3; - "~*(?:\b)Whacker(?:\b|)" 3; - "~*(?:\b)Whatweb(?:\b|)" 3; - "~*(?:\b)Who.is\ Bot(?:\b|)" 3; - "~*(?:\b)Widow(?:\b|)" 3; - "~*(?:\b)WinHTTrack(?:\b|)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; - "~*(?:\b)WISENutbot(?:\b|)" 3; - "~*(?:\b)Wonderbot(?:\b|)" 3; - "~*(?:\b)Woobot(?:\b|)" 3; - "~*(?:\b)Wotbox(?:\b|)" 3; - "~*(?:\b)Wprecon(?:\b|)" 3; - "~*(?:\b)WPScan(?:\b|)" 3; - "~*(?:\b)WWW-Collector-E(?:\b|)" 3; - "~*(?:\b)WWW-Mechanize(?:\b|)" 3; - "~*(?:\b)WWW::Mechanize(?:\b|)" 3; - "~*(?:\b)WWWOFFLE(?:\b|)" 3; - "~*(?:\b)x09Mozilla(?:\b|)" 3; - "~*(?:\b)x22Mozilla(?:\b|)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; - "~*(?:\b)Xenu(?:\b|)" 3; - "~*(?:\b)xpymep1.exe(?:\b|)" 3; - "~*(?:\b)YoudaoBot(?:\b|)" 3; - "~*(?:\b)Zade(?:\b|)" 3; - "~*(?:\b)Zauba(?:\b|)" 3; - "~*(?:\b)zauba.io(?:\b|)" 3; - "~*(?:\b)Zermelo(?:\b|)" 3; - "~*(?:\b)Zeus(?:\b|)" 3; - "~*(?:\b)zgrab(?:\b|)" 3; - "~*(?:\b)Zitebot(?:\b|)" 3; - "~*(?:\b)ZmEu(?:\b|)" 3; - "~*(?:\b)ZumBot(?:\b|)" 3; - "~*(?:\b)ZyBorg(?:\b|)" 3; + "~*(?:\b)360Spider(?:\b)" 3; + "~*(?:\b)404checker(?:\b)" 3; + "~*(?:\b)404enemy(?:\b)" 3; + "~*(?:\b)80legs(?:\b)" 3; + "~*(?:\b)Abonti(?:\b)" 3; + "~*(?:\b)Aboundex(?:\b)" 3; + "~*(?:\b)Aboundexbot(?:\b)" 3; + "~*(?:\b)Acunetix(?:\b)" 3; + "~*(?:\b)ADmantX(?:\b)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; + "~*(?:\b)AhrefsBot(?:\b)" 3; + "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)AiHitBot(?:\b)" 3; + "~*(?:\b)Aipbot(?:\b)" 3; + "~*(?:\b)Alexibot(?:\b)" 3; + "~*(?:\b)Alligator(?:\b)" 3; + "~*(?:\b)AllSubmitter(?:\b)" 3; + "~*(?:\b)AlphaBot(?:\b)" 3; + "~*(?:\b)Anarchie(?:\b)" 3; + "~*(?:\b)Apexoo(?:\b)" 3; + "~*(?:\b)archive.org_bot(?:\b)" 3; + "~*(?:\b)ASPSeek(?:\b)" 3; + "~*(?:\b)Asterias(?:\b)" 3; + "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)BackDoorBot(?:\b)" 3; + "~*(?:\b)Backlink-Ceck(?:\b)" 3; + "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)BacklinkCrawler(?:\b)" 3; + "~*(?:\b)BackStreet(?:\b)" 3; + "~*(?:\b)BackWeb(?:\b)" 3; + "~*(?:\b)Badass(?:\b)" 3; + "~*(?:\b)Bandit(?:\b)" 3; + "~*(?:\b)Barkrowler(?:\b)" 3; + "~*(?:\b)BatchFTP(?:\b)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; + "~*(?:\b)BBBike(?:\b)" 3; + "~*(?:\b)BDCbot(?:\b)" 3; + "~*(?:\b)BDFetch(?:\b)" 3; + "~*(?:\b)BetaBot(?:\b)" 3; + "~*(?:\b)Bigfoot(?:\b)" 3; + "~*(?:\b)Bitacle(?:\b)" 3; + "~*(?:\b)Blackboard(?:\b)" 3; + "~*(?:\b)Black\ Hole(?:\b)" 3; + "~*(?:\b)BlackWidow(?:\b)" 3; + "~*(?:\b)BLEXBot(?:\b)" 3; + "~*(?:\b)Blow(?:\b)" 3; + "~*(?:\b)BlowFish(?:\b)" 3; + "~*(?:\b)Boardreader(?:\b)" 3; + "~*(?:\b)Bolt(?:\b)" 3; + "~*(?:\b)BotALot(?:\b)" 3; + "~*(?:\b)Brandprotect(?:\b)" 3; + "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buddy(?:\b)" 3; + "~*(?:\b)BuiltBotTough(?:\b)" 3; + "~*(?:\b)BuiltWith(?:\b)" 3; + "~*(?:\b)Bullseye(?:\b)" 3; + "~*(?:\b)BunnySlippers(?:\b)" 3; + "~*(?:\b)BuzzSumo(?:\b)" 3; + "~*(?:\b)Calculon(?:\b)" 3; + "~*(?:\b)CATExplorador(?:\b)" 3; + "~*(?:\b)CazoodleBot(?:\b)" 3; + "~*(?:\b)CCBot(?:\b)" 3; + "~*(?:\b)Cegbfeieh(?:\b)" 3; + "~*(?:\b)CheeseBot(?:\b)" 3; + "~*(?:\b)CherryPicker(?:\b)" 3; + "~*(?:\b)CheTeam(?:\b)" 3; + "~*(?:\b)ChinaClaw(?:\b)" 3; + "~*(?:\b)Chlooe(?:\b)" 3; + "~*(?:\b)Claritybot(?:\b)" 3; + "~*(?:\b)Cliqzbot(?:\b)" 3; + "~*(?:\b)Cloud\ mapping(?:\b)" 3; + "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cogentbot(?:\b)" 3; + "~*(?:\b)cognitiveseo(?:\b)" 3; + "~*(?:\b)Collector(?:\b)" 3; + "~*(?:\b)com.plumanalytics(?:\b)" 3; + "~*(?:\b)Copier(?:\b)" 3; + "~*(?:\b)CopyRightCheck(?:\b)" 3; + "~*(?:\b)Copyscape(?:\b)" 3; + "~*(?:\b)Cosmos(?:\b)" 3; + "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)crawler4j(?:\b)" 3; + "~*(?:\b)crawler.feedback(?:\b)" 3; + "~*(?:\b)crawl.sogou.com(?:\b)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b)" 3; + "~*(?:\b)Crescent(?:\b)" 3; + "~*(?:\b)CrunchBot(?:\b)" 3; + "~*(?:\b)CSHttp(?:\b)" 3; + "~*(?:\b)Curious(?:\b)" 3; + "~*(?:\b)Custo(?:\b)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; + "~*(?:\b)DataCha0s(?:\b)" 3; + "~*(?:\b)DBLBot(?:\b)" 3; + "~*(?:\b)demandbase-bot(?:\b)" 3; + "~*(?:\b)Demon(?:\b)" 3; + "~*(?:\b)Deusu(?:\b)" 3; + "~*(?:\b)Devil(?:\b)" 3; + "~*(?:\b)Digincore(?:\b)" 3; + "~*(?:\b)DigitalPebble(?:\b)" 3; + "~*(?:\b)DIIbot(?:\b)" 3; + "~*(?:\b)Dirbuster(?:\b)" 3; + "~*(?:\b)Disco(?:\b)" 3; + "~*(?:\b)Discobot(?:\b)" 3; + "~*(?:\b)Discoverybot(?:\b)" 3; + "~*(?:\b)Dispatch(?:\b)" 3; + "~*(?:\b)DittoSpyder(?:\b)" 3; + "~*(?:\b)DnyzBot(?:\b)" 3; + "~*(?:\b)DomainAppender(?:\b)" 3; + "~*(?:\b)DomainCrawler(?:\b)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)DomainStatsBot(?:\b)" 3; + "~*(?:\b)Dotbot(?:\b)" 3; + "~*(?:\b)Download\ Wonder(?:\b)" 3; + "~*(?:\b)Dragonfly(?:\b)" 3; + "~*(?:\b)Drip(?:\b)" 3; + "~*(?:\b)DSearch(?:\b)" 3; + "~*(?:\b)DTS\ Agent(?:\b)" 3; + "~*(?:\b)EasyDL(?:\b)" 3; + "~*(?:\b)Ebingbong(?:\b)" 3; + "~*(?:\b)eCatch(?:\b)" 3; + "~*(?:\b)ECCP/1.0(?:\b)" 3; + "~*(?:\b)Ecxi(?:\b)" 3; + "~*(?:\b)EirGrabber(?:\b)" 3; + "~*(?:\b)EMail\ Siphon(?:\b)" 3; + "~*(?:\b)EMail\ Wolf(?:\b)" 3; + "~*(?:\b)EroCrawler(?:\b)" 3; + "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)Evil(?:\b)" 3; + "~*(?:\b)Exabot(?:\b)" 3; + "~*(?:\b)Express\ WebPictures(?:\b)" 3; + "~*(?:\b)ExtLinksBot(?:\b)" 3; + "~*(?:\b)Extractor(?:\b)" 3; + "~*(?:\b)ExtractorPro(?:\b)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; + "~*(?:\b)EyeNetIE(?:\b)" 3; + "~*(?:\b)Ezooms(?:\b)" 3; + "~*(?:\b)facebookscraper(?:\b)" 3; + "~*(?:\b)FDM(?:\b)" 3; + "~*(?:\b)FemtosearchBot(?:\b)" 3; + "~*(?:\b)FHscan(?:\b)" 3; + "~*(?:\b)Fimap(?:\b)" 3; + "~*(?:\b)Firefox/7.0(?:\b)" 3; + "~*(?:\b)FlashGet(?:\b)" 3; + "~*(?:\b)Flunky(?:\b)" 3; + "~*(?:\b)Foobot(?:\b)" 3; + "~*(?:\b)Freeuploader(?:\b)" 3; + "~*(?:\b)FrontPage(?:\b)" 3; + "~*(?:\b)FyberSpider(?:\b)" 3; + "~*(?:\b)Fyrebot(?:\b)" 3; + "~*(?:\b)GalaxyBot(?:\b)" 3; + "~*(?:\b)Genieo(?:\b)" 3; + "~*(?:\b)GermCrawler(?:\b)" 3; + "~*(?:\b)Getintent(?:\b)" 3; + "~*(?:\b)GetRight(?:\b)" 3; + "~*(?:\b)GetWeb(?:\b)" 3; + "~*(?:\b)Gigablast(?:\b)" 3; + "~*(?:\b)Gigabot(?:\b)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; + "~*(?:\b)Gotit(?:\b)" 3; + "~*(?:\b)GoZilla(?:\b)" 3; + "~*(?:\b)Go!Zilla(?:\b)" 3; + "~*(?:\b)Grabber(?:\b)" 3; + "~*(?:\b)GrabNet(?:\b)" 3; + "~*(?:\b)Grafula(?:\b)" 3; + "~*(?:\b)GrapeFX(?:\b)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b)" 3; + "~*(?:\b)GridBot(?:\b)" 3; + "~*(?:\b)GT::WWW(?:\b)" 3; + "~*(?:\b)Haansoft(?:\b)" 3; + "~*(?:\b)HaosouSpider(?:\b)" 3; + "~*(?:\b)Harvest(?:\b)" 3; + "~*(?:\b)Havij(?:\b)" 3; + "~*(?:\b)HEADMasterSEO(?:\b)" 3; + "~*(?:\b)Heritrix(?:\b)" 3; + "~*(?:\b)Hloader(?:\b)" 3; + "~*(?:\b)HMView(?:\b)" 3; + "~*(?:\b)HTMLparser(?:\b)" 3; + "~*(?:\b)HTTP::Lite(?:\b)" 3; + "~*(?:\b)HTTrack(?:\b)" 3; + "~*(?:\b)Humanlinks(?:\b)" 3; + "~*(?:\b)HybridBot(?:\b)" 3; + "~*(?:\b)Iblog(?:\b)" 3; + "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)Id-search(?:\b)" 3; + "~*(?:\b)IlseBot(?:\b)" 3; + "~*(?:\b)Image\ Fetch(?:\b)" 3; + "~*(?:\b)Image\ Sucker(?:\b)" 3; + "~*(?:\b)IndeedBot(?:\b)" 3; + "~*(?:\b)Indy\ Library(?:\b)" 3; + "~*(?:\b)InfoNaviRobot(?:\b)" 3; + "~*(?:\b)InfoTekies(?:\b)" 3; + "~*(?:\b)instabid(?:\b)" 3; + "~*(?:\b)Intelliseek(?:\b)" 3; + "~*(?:\b)InterGET(?:\b)" 3; + "~*(?:\b)Internet\ Ninja(?:\b)" 3; + "~*(?:\b)InternetSeer(?:\b)" 3; + "~*(?:\b)internetVista\ monitor(?:\b)" 3; + "~*(?:\b)ips-agent(?:\b)" 3; + "~*(?:\b)Iria(?:\b)" 3; + "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)Iskanie(?:\b)" 3; + "~*(?:\b)IstellaBot(?:\b)" 3; + "~*(?:\b)JamesBOT(?:\b)" 3; + "~*(?:\b)Jbrofuzz(?:\b)" 3; + "~*(?:\b)JennyBot(?:\b)" 3; + "~*(?:\b)JetCar(?:\b)" 3; + "~*(?:\b)Jetty(?:\b)" 3; + "~*(?:\b)JikeSpider(?:\b)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; + "~*(?:\b)Joomla(?:\b)" 3; + "~*(?:\b)Jorgee(?:\b)" 3; + "~*(?:\b)JustView(?:\b)" 3; + "~*(?:\b)Jyxobot(?:\b)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b)" 3; + "~*(?:\b)Keyword\ Density(?:\b)" 3; + "~*(?:\b)Kozmosbot(?:\b)" 3; + "~*(?:\b)Lanshanbot(?:\b)" 3; + "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)LeechFTP(?:\b)" 3; + "~*(?:\b)LeechGet(?:\b)" 3; + "~*(?:\b)LexiBot(?:\b)" 3; + "~*(?:\b)Lftp(?:\b)" 3; + "~*(?:\b)LibWeb(?:\b)" 3; + "~*(?:\b)Libwhisker(?:\b)" 3; + "~*(?:\b)Lightspeedsystems(?:\b)" 3; + "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkdexbot(?:\b)" 3; + "~*(?:\b)LinkextractorPro(?:\b)" 3; + "~*(?:\b)LinkpadBot(?:\b)" 3; + "~*(?:\b)LinkScan(?:\b)" 3; + "~*(?:\b)LinksManager(?:\b)" 3; + "~*(?:\b)LinkWalker(?:\b)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; + "~*(?:\b)Lipperhey(?:\b)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; + "~*(?:\b)Litemage_walker(?:\b)" 3; + "~*(?:\b)Lmspider(?:\b)" 3; + "~*(?:\b)LNSpiderguy(?:\b)" 3; + "~*(?:\b)Ltx71(?:\b)" 3; + "~*(?:\b)lwp-request(?:\b)" 3; + "~*(?:\b)LWP::Simple(?:\b)" 3; + "~*(?:\b)lwp-trivial(?:\b)" 3; + "~*(?:\b)Magnet(?:\b)" 3; + "~*(?:\b)Mag-Net(?:\b)" 3; + "~*(?:\b)magpie-crawler(?:\b)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b)" 3; + "~*(?:\b)Majestic12(?:\b)" 3; + "~*(?:\b)Majestic-SEO(?:\b)" 3; + "~*(?:\b)Majestic\ SEO(?:\b)" 3; + "~*(?:\b)MarkMonitor(?:\b)" 3; + "~*(?:\b)MarkWatch(?:\b)" 3; + "~*(?:\b)Masscan(?:\b)" 3; + "~*(?:\b)Mass\ Downloader(?:\b)" 3; + "~*(?:\b)Mata\ Hari(?:\b)" 3; + "~*(?:\b)MauiBot(?:\b)" 3; + "~*(?:\b)meanpathbot(?:\b)" 3; + "~*(?:\b)Meanpathbot(?:\b)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b)" 3; + "~*(?:\b)mediawords(?:\b)" 3; + "~*(?:\b)MegaIndex.ru(?:\b)" 3; + "~*(?:\b)Metauri(?:\b)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; + "~*(?:\b)MIDown\ tool(?:\b)" 3; + "~*(?:\b)MIIxpc(?:\b)" 3; + "~*(?:\b)Mister\ PiX(?:\b)" 3; + "~*(?:\b)MJ12bot(?:\b)" 3; + "~*(?:\b)Mojeek(?:\b)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; + "~*(?:\b)Mr.4x3(?:\b)" 3; + "~*(?:\b)MSFrontPage(?:\b)" 3; + "~*(?:\b)MSIECrawler(?:\b)" 3; + "~*(?:\b)Msrabot(?:\b)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; + "~*(?:\b)muhstik-scan(?:\b)" 3; + "~*(?:\b)Musobot(?:\b)" 3; + "~*(?:\b)Name\ Intelligence(?:\b)" 3; + "~*(?:\b)Nameprotect(?:\b)" 3; + "~*(?:\b)Navroad(?:\b)" 3; + "~*(?:\b)NearSite(?:\b)" 3; + "~*(?:\b)Needle(?:\b)" 3; + "~*(?:\b)Nessus(?:\b)" 3; + "~*(?:\b)NetAnts(?:\b)" 3; + "~*(?:\b)Netcraft(?:\b)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; + "~*(?:\b)NetLyzer(?:\b)" 3; + "~*(?:\b)NetMechanic(?:\b)" 3; + "~*(?:\b)NetSpider(?:\b)" 3; + "~*(?:\b)Nettrack(?:\b)" 3; + "~*(?:\b)Net\ Vampire(?:\b)" 3; + "~*(?:\b)Netvibes(?:\b)" 3; + "~*(?:\b)NetZIP(?:\b)" 3; + "~*(?:\b)NextGenSearchBot(?:\b)" 3; + "~*(?:\b)Nibbler(?:\b)" 3; + "~*(?:\b)NICErsPRO(?:\b)" 3; + "~*(?:\b)Niki-bot(?:\b)" 3; + "~*(?:\b)Nikto(?:\b)" 3; + "~*(?:\b)NimbleCrawler(?:\b)" 3; + "~*(?:\b)Nimbostratus(?:\b)" 3; + "~*(?:\b)Ninja(?:\b)" 3; + "~*(?:\b)Nmap(?:\b)" 3; + "~*(?:\b)NPbot(?:\b)" 3; + "~*(?:\b)Nutch(?:\b)" 3; + "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)Octopus(?:\b)" 3; + "~*(?:\b)Offline\ Explorer(?:\b)" 3; + "~*(?:\b)Offline\ Navigator(?:\b)" 3; + "~*(?:\b)OnCrawl(?:\b)" 3; + "~*(?:\b)Openfind(?:\b)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b)" 3; + "~*(?:\b)Openvas(?:\b)" 3; + "~*(?:\b)OpenVAS(?:\b)" 3; + "~*(?:\b)OrangeBot(?:\b)" 3; + "~*(?:\b)OrangeSpider(?:\b)" 3; + "~*(?:\b)OutclicksBot(?:\b)" 3; + "~*(?:\b)OutfoxBot(?:\b)" 3; + "~*(?:\b)PageAnalyzer(?:\b)" 3; + "~*(?:\b)Page\ Analyzer(?:\b)" 3; + "~*(?:\b)PageGrabber(?:\b)" 3; + "~*(?:\b)page\ scorer(?:\b)" 3; + "~*(?:\b)PageScorer(?:\b)" 3; + "~*(?:\b)Pandalytics(?:\b)" 3; + "~*(?:\b)Panscient(?:\b)" 3; + "~*(?:\b)Papa\ Foto(?:\b)" 3; + "~*(?:\b)Pavuk(?:\b)" 3; + "~*(?:\b)pcBrowser(?:\b)" 3; + "~*(?:\b)PECL::HTTP(?:\b)" 3; + "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)PHPCrawl(?:\b)" 3; + "~*(?:\b)Picscout(?:\b)" 3; + "~*(?:\b)Picsearch(?:\b)" 3; + "~*(?:\b)PictureFinder(?:\b)" 3; + "~*(?:\b)Pimonster(?:\b)" 3; + "~*(?:\b)Pi-Monster(?:\b)" 3; + "~*(?:\b)Pixray(?:\b)" 3; + "~*(?:\b)PleaseCrawl(?:\b)" 3; + "~*(?:\b)plumanalytics(?:\b)" 3; + "~*(?:\b)Pockey(?:\b)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; + "~*(?:\b)Probethenet(?:\b)" 3; + "~*(?:\b)ProPowerBot(?:\b)" 3; + "~*(?:\b)ProWebWalker(?:\b)" 3; + "~*(?:\b)Psbot(?:\b)" 3; + "~*(?:\b)Pump(?:\b)" 3; + "~*(?:\b)PxBroker(?:\b)" 3; + "~*(?:\b)PyCurl(?:\b)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; + "~*(?:\b)Quick-Crawler(?:\b)" 3; + "~*(?:\b)RankActive(?:\b)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b)" 3; + "~*(?:\b)RankFlex(?:\b)" 3; + "~*(?:\b)RankingBot(?:\b)" 3; + "~*(?:\b)RankingBot2(?:\b)" 3; + "~*(?:\b)Rankivabot(?:\b)" 3; + "~*(?:\b)RankurBot(?:\b)" 3; + "~*(?:\b)RealDownload(?:\b)" 3; + "~*(?:\b)Reaper(?:\b)" 3; + "~*(?:\b)RebelMouse(?:\b)" 3; + "~*(?:\b)Recorder(?:\b)" 3; + "~*(?:\b)RedesScrapy(?:\b)" 3; + "~*(?:\b)ReGet(?:\b)" 3; + "~*(?:\b)RepoMonkey(?:\b)" 3; + "~*(?:\b)Ripper(?:\b)" 3; + "~*(?:\b)RocketCrawler(?:\b)" 3; + "~*(?:\b)Rogerbot(?:\b)" 3; + "~*(?:\b)s1z.ru(?:\b)" 3; + "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)SBIder(?:\b)" 3; + "~*(?:\b)ScanAlert(?:\b)" 3; + "~*(?:\b)Scanbot(?:\b)" 3; + "~*(?:\b)scan.lol(?:\b)" 3; + "~*(?:\b)ScoutJet(?:\b)" 3; + "~*(?:\b)Scrapy(?:\b)" 3; + "~*(?:\b)Screaming(?:\b)" 3; + "~*(?:\b)ScreenerBot(?:\b)" 3; + "~*(?:\b)Searchestate(?:\b)" 3; + "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Semrush(?:\b)" 3; + "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SEOkicks(?:\b)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; + "~*(?:\b)Seomoz(?:\b)" 3; + "~*(?:\b)SEOprofiler(?:\b)" 3; + "~*(?:\b)seoscanners(?:\b)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b)" 3; + "~*(?:\b)SEOstats(?:\b)" 3; + "~*(?:\b)serpstatbot(?:\b)" 3; + "~*(?:\b)sexsearcher(?:\b)" 3; + "~*(?:\b)Shodan(?:\b)" 3; + "~*(?:\b)Siphon(?:\b)" 3; + "~*(?:\b)SISTRIX(?:\b)" 3; + "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteExplorer(?:\b)" 3; + "~*(?:\b)Siteimprove(?:\b)" 3; + "~*(?:\b)SiteLockSpider(?:\b)" 3; + "~*(?:\b)SiteSnagger(?:\b)" 3; + "~*(?:\b)SiteSucker(?:\b)" 3; + "~*(?:\b)Site\ Sucker(?:\b)" 3; + "~*(?:\b)Sitevigil(?:\b)" 3; + "~*(?:\b)SlySearch(?:\b)" 3; + "~*(?:\b)SmartDownload(?:\b)" 3; + "~*(?:\b)SMTBot(?:\b)" 3; + "~*(?:\b)Snake(?:\b)" 3; + "~*(?:\b)Snapbot(?:\b)" 3; + "~*(?:\b)Snoopy(?:\b)" 3; + "~*(?:\b)SocialRankIOBot(?:\b)" 3; + "~*(?:\b)Sociscraper(?:\b)" 3; + "~*(?:\b)sogouspider(?:\b)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; + "~*(?:\b)Sosospider(?:\b)" 3; + "~*(?:\b)Sottopop(?:\b)" 3; + "~*(?:\b)SpaceBison(?:\b)" 3; + "~*(?:\b)Spammen(?:\b)" 3; + "~*(?:\b)SpankBot(?:\b)" 3; + "~*(?:\b)Spanner(?:\b)" 3; + "~*(?:\b)sp_auditbot(?:\b)" 3; + "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spinn3r(?:\b)" 3; + "~*(?:\b)SputnikBot(?:\b)" 3; + "~*(?:\b)spyfu(?:\b)" 3; + "~*(?:\b)Sqlmap(?:\b)" 3; + "~*(?:\b)Sqlworm(?:\b)" 3; + "~*(?:\b)Sqworm(?:\b)" 3; + "~*(?:\b)Steeler(?:\b)" 3; + "~*(?:\b)Stripper(?:\b)" 3; + "~*(?:\b)Sucker(?:\b)" 3; + "~*(?:\b)Sucuri(?:\b)" 3; + "~*(?:\b)SuperBot(?:\b)" 3; + "~*(?:\b)SuperHTTP(?:\b)" 3; + "~*(?:\b)Surfbot(?:\b)" 3; + "~*(?:\b)SurveyBot(?:\b)" 3; + "~*(?:\b)Suzuran(?:\b)" 3; + "~*(?:\b)Swiftbot(?:\b)" 3; + "~*(?:\b)sysscan(?:\b)" 3; + "~*(?:\b)Szukacz(?:\b)" 3; + "~*(?:\b)T0PHackTeam(?:\b)" 3; + "~*(?:\b)T8Abot(?:\b)" 3; + "~*(?:\b)tAkeOut(?:\b)" 3; + "~*(?:\b)Teleport(?:\b)" 3; + "~*(?:\b)TeleportPro(?:\b)" 3; + "~*(?:\b)Telesoft(?:\b)" 3; + "~*(?:\b)Telesphoreo(?:\b)" 3; + "~*(?:\b)Telesphorep(?:\b)" 3; + "~*(?:\b)The\ Intraformant(?:\b)" 3; + "~*(?:\b)TheNomad(?:\b)" 3; + "~*(?:\b)Thumbor(?:\b)" 3; + "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)Titan(?:\b)" 3; + "~*(?:\b)Toata(?:\b)" 3; + "~*(?:\b)Toweyabot(?:\b)" 3; + "~*(?:\b)Tracemyfile(?:\b)" 3; + "~*(?:\b)Trendiction(?:\b)" 3; + "~*(?:\b)Trendictionbot(?:\b)" 3; + "~*(?:\b)trendiction.com(?:\b)" 3; + "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)True_Robot(?:\b)" 3; + "~*(?:\b)Turingos(?:\b)" 3; + "~*(?:\b)Turnitin(?:\b)" 3; + "~*(?:\b)TurnitinBot(?:\b)" 3; + "~*(?:\b)TwengaBot(?:\b)" 3; + "~*(?:\b)Twice(?:\b)" 3; + "~*(?:\b)Typhoeus(?:\b)" 3; + "~*(?:\b)UnisterBot(?:\b)" 3; + "~*(?:\b)Upflow(?:\b)" 3; + "~*(?:\b)URLy.Warning(?:\b)" 3; + "~*(?:\b)URLy\ Warning(?:\b)" 3; + "~*(?:\b)Vacuum(?:\b)" 3; + "~*(?:\b)Vagabondo(?:\b)" 3; + "~*(?:\b)VB\ Project(?:\b)" 3; + "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b)" 3; + "~*(?:\b)VidibleScraper(?:\b)" 3; + "~*(?:\b)Virusdie(?:\b)" 3; + "~*(?:\b)VoidEYE(?:\b)" 3; + "~*(?:\b)Voil(?:\b)" 3; + "~*(?:\b)Voltron(?:\b)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b)" 3; + "~*(?:\b)WallpapersHD(?:\b)" 3; + "~*(?:\b)WASALive-Bot(?:\b)" 3; + "~*(?:\b)WBSearchBot(?:\b)" 3; + "~*(?:\b)Webalta(?:\b)" 3; + "~*(?:\b)WebAuto(?:\b)" 3; + "~*(?:\b)Web\ Auto(?:\b)" 3; + "~*(?:\b)WebBandit(?:\b)" 3; + "~*(?:\b)WebCollage(?:\b)" 3; + "~*(?:\b)Web\ Collage(?:\b)" 3; + "~*(?:\b)WebCopier(?:\b)" 3; + "~*(?:\b)WEBDAV(?:\b)" 3; + "~*(?:\b)WebEnhancer(?:\b)" 3; + "~*(?:\b)Web\ Enhancer(?:\b)" 3; + "~*(?:\b)WebFetch(?:\b)" 3; + "~*(?:\b)Web\ Fetch(?:\b)" 3; + "~*(?:\b)WebFuck(?:\b)" 3; + "~*(?:\b)Web\ Fuck(?:\b)" 3; + "~*(?:\b)WebGo\ IS(?:\b)" 3; + "~*(?:\b)WebImageCollector(?:\b)" 3; + "~*(?:\b)WebLeacher(?:\b)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; + "~*(?:\b)webmeup-crawler(?:\b)" 3; + "~*(?:\b)WebPix(?:\b)" 3; + "~*(?:\b)Web\ Pix(?:\b)" 3; + "~*(?:\b)WebReaper(?:\b)" 3; + "~*(?:\b)WebSauger(?:\b)" 3; + "~*(?:\b)Web\ Sauger(?:\b)" 3; + "~*(?:\b)Webshag(?:\b)" 3; + "~*(?:\b)WebsiteExtractor(?:\b)" 3; + "~*(?:\b)WebsiteQuester(?:\b)" 3; + "~*(?:\b)Website\ Quester(?:\b)" 3; + "~*(?:\b)Webster(?:\b)" 3; + "~*(?:\b)WebStripper(?:\b)" 3; + "~*(?:\b)WebSucker(?:\b)" 3; + "~*(?:\b)Web\ Sucker(?:\b)" 3; + "~*(?:\b)WebWhacker(?:\b)" 3; + "~*(?:\b)WebZIP(?:\b)" 3; + "~*(?:\b)WeSEE(?:\b)" 3; + "~*(?:\b)Whack(?:\b)" 3; + "~*(?:\b)Whacker(?:\b)" 3; + "~*(?:\b)Whatweb(?:\b)" 3; + "~*(?:\b)Who.is\ Bot(?:\b)" 3; + "~*(?:\b)Widow(?:\b)" 3; + "~*(?:\b)WinHTTrack(?:\b)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; + "~*(?:\b)WISENutbot(?:\b)" 3; + "~*(?:\b)Wonderbot(?:\b)" 3; + "~*(?:\b)Woobot(?:\b)" 3; + "~*(?:\b)Wotbox(?:\b)" 3; + "~*(?:\b)Wprecon(?:\b)" 3; + "~*(?:\b)WPScan(?:\b)" 3; + "~*(?:\b)WWW-Collector-E(?:\b)" 3; + "~*(?:\b)WWW-Mechanize(?:\b)" 3; + "~*(?:\b)WWW::Mechanize(?:\b)" 3; + "~*(?:\b)WWWOFFLE(?:\b)" 3; + "~*(?:\b)x09Mozilla(?:\b)" 3; + "~*(?:\b)x22Mozilla(?:\b)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; + "~*(?:\b)Xenu(?:\b)" 3; + "~*(?:\b)xpymep1.exe(?:\b)" 3; + "~*(?:\b)YoudaoBot(?:\b)" 3; + "~*(?:\b)Zade(?:\b)" 3; + "~*(?:\b)Zauba(?:\b)" 3; + "~*(?:\b)zauba.io(?:\b)" 3; + "~*(?:\b)Zermelo(?:\b)" 3; + "~*(?:\b)Zeus(?:\b)" 3; + "~*(?:\b)zgrab(?:\b)" 3; + "~*(?:\b)Zitebot(?:\b)" 3; + "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZumBot(?:\b)" 3; + "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------- @@ -717,38 +717,38 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)adidxbot(?:\b|)" 0; - "~*(?:\b)AdsBot-Google(?:\b|)" 0; - "~*(?:\b)aolbuild(?:\b|)" 0; - "~*(?:\b)bingbot(?:\b|)" 0; - "~*(?:\b)bingpreview(?:\b|)" 0; - "~*(?:\b)DoCoMo(?:\b|)" 0; - "~*(?:\b)duckduckgo(?:\b|)" 0; - "~*(?:\b)facebookexternalhit(?:\b|)" 0; - "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; - "~*(?:\b)Googlebot(?:\b|)" 0; - "~*(?:\b)Googlebot-Image(?:\b|)" 0; - "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; - "~*(?:\b)Googlebot-News(?:\b|)" 0; - "~*(?:\b)Googlebot/Test(?:\b|)" 0; - "~*(?:\b)Googlebot-Video(?:\b|)" 0; - "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; - "~*(?:\b)Gravityscan(?:\b|)" 0; - "~*(?:\b)gsa-crawler(?:\b|)" 0; - "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; - "~*(?:\b)Kraken/0.1(?:\b|)" 0; - "~*(?:\b)LinkedInBot(?:\b|)" 0; - "~*(?:\b)Mediapartners-Google(?:\b|)" 0; - "~*(?:\b)msnbot(?:\b|)" 0; - "~*(?:\b)msnbot-media(?:\b|)" 0; - "~*(?:\b)SAMSUNG(?:\b|)" 0; - "~*(?:\b)Slackbot(?:\b|)" 0; - "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; - "~*(?:\b)slurp(?:\b|)" 0; - "~*(?:\b)teoma(?:\b|)" 0; - "~*(?:\b)TwitterBot(?:\b|)" 0; - "~*(?:\b)Wordpress(?:\b|)" 0; - "~*(?:\b)yahoo(?:\b|)" 0; + "~*(?:\b)adidxbot(?:\b)" 0; + "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bingbot(?:\b)" 0; + "~*(?:\b)bingpreview(?:\b)" 0; + "~*(?:\b)DoCoMo(?:\b)" 0; + "~*(?:\b)duckduckgo(?:\b)" 0; + "~*(?:\b)facebookexternalhit(?:\b)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b)" 0; + "~*(?:\b)Googlebot(?:\b)" 0; + "~*(?:\b)Googlebot-Image(?:\b)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b)" 0; + "~*(?:\b)Googlebot-News(?:\b)" 0; + "~*(?:\b)Googlebot/Test(?:\b)" 0; + "~*(?:\b)Googlebot-Video(?:\b)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; + "~*(?:\b)Gravityscan(?:\b)" 0; + "~*(?:\b)gsa-crawler(?:\b)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b)" 0; + "~*(?:\b)Kraken/0.1(?:\b)" 0; + "~*(?:\b)LinkedInBot(?:\b)" 0; + "~*(?:\b)Mediapartners-Google(?:\b)" 0; + "~*(?:\b)msnbot(?:\b)" 0; + "~*(?:\b)msnbot-media(?:\b)" 0; + "~*(?:\b)SAMSUNG(?:\b)" 0; + "~*(?:\b)Slackbot(?:\b)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 0; + "~*(?:\b)slurp(?:\b)" 0; + "~*(?:\b)teoma(?:\b)" 0; + "~*(?:\b)TwitterBot(?:\b)" 0; + "~*(?:\b)Wordpress(?:\b)" 0; + "~*(?:\b)yahoo(?:\b)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------------------- @@ -760,12 +760,12 @@ map $http_user_agent $bad_bot { #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)jetmon(?:\b|)" 1; - "~*(?:\b)libwww-perl(?:\b|)" 1; - "~*(?:\b)Lynx(?:\b|)" 1; - "~*(?:\b)munin(?:\b|)" 1; - "~*(?:\b)Presto(?:\b|)" 1; - "~*(?:\b)Wget/1.15(?:\b|)" 1; + "~*(?:\b)jetmon(?:\b)" 1; + "~*(?:\b)libwww-perl(?:\b)" 1; + "~*(?:\b)Lynx(?:\b)" 1; + "~*(?:\b)munin(?:\b)" 1; + "~*(?:\b)Presto(?:\b)" 1; + "~*(?:\b)Wget/1.15(?:\b)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ------------------------------------------------------- @@ -778,23 +778,23 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)Alexa(?:\b|)" 2; - "~*(?:\b)archive.org(?:\b|)" 2; - "~*(?:\b)Baidu(?:\b|)" 2; - "~*(?:\b)BUbiNG(?:\b|)" 2; - "~*(?:\b)FlipboardProxy(?:\b|)" 2; - "~*(?:\b)ia_archiver(?:\b|)" 2; - "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; - "~*(?:\b)Proximic(?:\b|)" 2; - "~*(?:\b)R6_CommentReader(?:\b|)" 2; - "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; - "~*(?:\b)RED/1(?:\b|)" 2; - "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; - "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; - "~*(?:\b)Spaidu(?:\b|)" 2; - "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; - "~*(?:\b)YandexBot(?:\b|)" 2; - "~*(?:\b)YandexImages(?:\b|)" 2; + "~*(?:\b)Alexa(?:\b)" 2; + "~*(?:\b)archive.org(?:\b)" 2; + "~*(?:\b)Baidu(?:\b)" 2; + "~*(?:\b)BUbiNG(?:\b)" 2; + "~*(?:\b)FlipboardProxy(?:\b)" 2; + "~*(?:\b)ia_archiver(?:\b)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b)" 2; + "~*(?:\b)Proximic(?:\b)" 2; + "~*(?:\b)R6_CommentReader(?:\b)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b)" 2; + "~*(?:\b)RED/1(?:\b)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b)" 2; + "~*(?:\b)Spaidu(?:\b)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b)" 2; + "~*(?:\b)YandexBot(?:\b)" 2; + "~*(?:\b)YandexImages(?:\b)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/test-blocker-false-positives.sh b/.dev-tools/test-blocker-false-positives.sh old mode 100644 new mode 100755 diff --git a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf index f26e52add..a1999739a 100644 --- a/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1709 -### Updated: Mon Jul 1 10:37:04 SAST 2019 +### Version: V4.2019.07.1712 +### Updated: Mon Jul 1 11:24:52 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### @@ -147,562 +147,562 @@ map $http_user_agent $bad_bot { # -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b|)" 3; - "~*(?:\b)404checker(?:\b|)" 3; - "~*(?:\b)404enemy(?:\b|)" 3; - "~*(?:\b)80legs(?:\b|)" 3; - "~*(?:\b)Abonti(?:\b|)" 3; - "~*(?:\b)Aboundex(?:\b|)" 3; - "~*(?:\b)Aboundexbot(?:\b|)" 3; - "~*(?:\b)Acunetix(?:\b|)" 3; - "~*(?:\b)ADmantX(?:\b|)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; - "~*(?:\b)AhrefsBot(?:\b|)" 3; - "~*(?:\b)AIBOT(?:\b|)" 3; - "~*(?:\b)AiHitBot(?:\b|)" 3; - "~*(?:\b)Aipbot(?:\b|)" 3; - "~*(?:\b)Alexibot(?:\b|)" 3; - "~*(?:\b)Alligator(?:\b|)" 3; - "~*(?:\b)AllSubmitter(?:\b|)" 3; - "~*(?:\b)AlphaBot(?:\b|)" 3; - "~*(?:\b)Anarchie(?:\b|)" 3; - "~*(?:\b)Apexoo(?:\b|)" 3; - "~*(?:\b)archive.org_bot(?:\b|)" 3; - "~*(?:\b)ASPSeek(?:\b|)" 3; - "~*(?:\b)Asterias(?:\b|)" 3; - "~*(?:\b)Attach(?:\b|)" 3; - "~*(?:\b)autoemailspider(?:\b|)" 3; - "~*(?:\b)BackDoorBot(?:\b|)" 3; - "~*(?:\b)Backlink-Ceck(?:\b|)" 3; - "~*(?:\b)backlink-check(?:\b|)" 3; - "~*(?:\b)BacklinkCrawler(?:\b|)" 3; - "~*(?:\b)BackStreet(?:\b|)" 3; - "~*(?:\b)BackWeb(?:\b|)" 3; - "~*(?:\b)Badass(?:\b|)" 3; - "~*(?:\b)Bandit(?:\b|)" 3; - "~*(?:\b)Barkrowler(?:\b|)" 3; - "~*(?:\b)BatchFTP(?:\b|)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; - "~*(?:\b)BBBike(?:\b|)" 3; - "~*(?:\b)BDCbot(?:\b|)" 3; - "~*(?:\b)BDFetch(?:\b|)" 3; - "~*(?:\b)BetaBot(?:\b|)" 3; - "~*(?:\b)Bigfoot(?:\b|)" 3; - "~*(?:\b)Bitacle(?:\b|)" 3; - "~*(?:\b)Blackboard(?:\b|)" 3; - "~*(?:\b)Black\ Hole(?:\b|)" 3; - "~*(?:\b)BlackWidow(?:\b|)" 3; - "~*(?:\b)BLEXBot(?:\b|)" 3; - "~*(?:\b)Blow(?:\b|)" 3; - "~*(?:\b)BlowFish(?:\b|)" 3; - "~*(?:\b)Boardreader(?:\b|)" 3; - "~*(?:\b)Bolt(?:\b|)" 3; - "~*(?:\b)BotALot(?:\b|)" 3; - "~*(?:\b)Brandprotect(?:\b|)" 3; - "~*(?:\b)Brandwatch(?:\b|)" 3; - "~*(?:\b)Buddy(?:\b|)" 3; - "~*(?:\b)BuiltBotTough(?:\b|)" 3; - "~*(?:\b)BuiltWith(?:\b|)" 3; - "~*(?:\b)Bullseye(?:\b|)" 3; - "~*(?:\b)BunnySlippers(?:\b|)" 3; - "~*(?:\b)BuzzSumo(?:\b|)" 3; - "~*(?:\b)Calculon(?:\b|)" 3; - "~*(?:\b)CATExplorador(?:\b|)" 3; - "~*(?:\b)CazoodleBot(?:\b|)" 3; - "~*(?:\b)CCBot(?:\b|)" 3; - "~*(?:\b)Cegbfeieh(?:\b|)" 3; - "~*(?:\b)CheeseBot(?:\b|)" 3; - "~*(?:\b)CherryPicker(?:\b|)" 3; - "~*(?:\b)CheTeam(?:\b|)" 3; - "~*(?:\b)ChinaClaw(?:\b|)" 3; - "~*(?:\b)Chlooe(?:\b|)" 3; - "~*(?:\b)Claritybot(?:\b|)" 3; - "~*(?:\b)Cliqzbot(?:\b|)" 3; - "~*(?:\b)Cloud\ mapping(?:\b|)" 3; - "~*(?:\b)coccocbot-web(?:\b|)" 3; - "~*(?:\b)Cogentbot(?:\b|)" 3; - "~*(?:\b)cognitiveseo(?:\b|)" 3; - "~*(?:\b)Collector(?:\b|)" 3; - "~*(?:\b)com.plumanalytics(?:\b|)" 3; - "~*(?:\b)Copier(?:\b|)" 3; - "~*(?:\b)CopyRightCheck(?:\b|)" 3; - "~*(?:\b)Copyscape(?:\b|)" 3; - "~*(?:\b)Cosmos(?:\b|)" 3; - "~*(?:\b)Craftbot(?:\b|)" 3; - "~*(?:\b)crawler4j(?:\b|)" 3; - "~*(?:\b)crawler.feedback(?:\b|)" 3; - "~*(?:\b)crawl.sogou.com(?:\b|)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; - "~*(?:\b)Crescent(?:\b|)" 3; - "~*(?:\b)CrunchBot(?:\b|)" 3; - "~*(?:\b)CSHttp(?:\b|)" 3; - "~*(?:\b)Curious(?:\b|)" 3; - "~*(?:\b)Custo(?:\b|)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; - "~*(?:\b)DataCha0s(?:\b|)" 3; - "~*(?:\b)DBLBot(?:\b|)" 3; - "~*(?:\b)demandbase-bot(?:\b|)" 3; - "~*(?:\b)Demon(?:\b|)" 3; - "~*(?:\b)Deusu(?:\b|)" 3; - "~*(?:\b)Devil(?:\b|)" 3; - "~*(?:\b)Digincore(?:\b|)" 3; - "~*(?:\b)DigitalPebble(?:\b|)" 3; - "~*(?:\b)DIIbot(?:\b|)" 3; - "~*(?:\b)Dirbuster(?:\b|)" 3; - "~*(?:\b)Disco(?:\b|)" 3; - "~*(?:\b)Discobot(?:\b|)" 3; - "~*(?:\b)Discoverybot(?:\b|)" 3; - "~*(?:\b)Dispatch(?:\b|)" 3; - "~*(?:\b)DittoSpyder(?:\b|)" 3; - "~*(?:\b)DnyzBot(?:\b|)" 3; - "~*(?:\b)DomainAppender(?:\b|)" 3; - "~*(?:\b)DomainCrawler(?:\b|)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; - "~*(?:\b)DomainStatsBot(?:\b|)" 3; - "~*(?:\b)Dotbot(?:\b|)" 3; - "~*(?:\b)Download\ Wonder(?:\b|)" 3; - "~*(?:\b)Dragonfly(?:\b|)" 3; - "~*(?:\b)Drip(?:\b|)" 3; - "~*(?:\b)DSearch(?:\b|)" 3; - "~*(?:\b)DTS\ Agent(?:\b|)" 3; - "~*(?:\b)EasyDL(?:\b|)" 3; - "~*(?:\b)Ebingbong(?:\b|)" 3; - "~*(?:\b)eCatch(?:\b|)" 3; - "~*(?:\b)ECCP/1.0(?:\b|)" 3; - "~*(?:\b)Ecxi(?:\b|)" 3; - "~*(?:\b)EirGrabber(?:\b|)" 3; - "~*(?:\b)EMail\ Siphon(?:\b|)" 3; - "~*(?:\b)EMail\ Wolf(?:\b|)" 3; - "~*(?:\b)EroCrawler(?:\b|)" 3; - "~*(?:\b)evc-batch(?:\b|)" 3; - "~*(?:\b)Evil(?:\b|)" 3; - "~*(?:\b)Exabot(?:\b|)" 3; - "~*(?:\b)Express\ WebPictures(?:\b|)" 3; - "~*(?:\b)ExtLinksBot(?:\b|)" 3; - "~*(?:\b)Extractor(?:\b|)" 3; - "~*(?:\b)ExtractorPro(?:\b|)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; - "~*(?:\b)EyeNetIE(?:\b|)" 3; - "~*(?:\b)Ezooms(?:\b|)" 3; - "~*(?:\b)facebookscraper(?:\b|)" 3; - "~*(?:\b)FDM(?:\b|)" 3; - "~*(?:\b)FemtosearchBot(?:\b|)" 3; - "~*(?:\b)FHscan(?:\b|)" 3; - "~*(?:\b)Fimap(?:\b|)" 3; - "~*(?:\b)Firefox/7.0(?:\b|)" 3; - "~*(?:\b)FlashGet(?:\b|)" 3; - "~*(?:\b)Flunky(?:\b|)" 3; - "~*(?:\b)Foobot(?:\b|)" 3; - "~*(?:\b)Freeuploader(?:\b|)" 3; - "~*(?:\b)FrontPage(?:\b|)" 3; - "~*(?:\b)FyberSpider(?:\b|)" 3; - "~*(?:\b)Fyrebot(?:\b|)" 3; - "~*(?:\b)GalaxyBot(?:\b|)" 3; - "~*(?:\b)Genieo(?:\b|)" 3; - "~*(?:\b)GermCrawler(?:\b|)" 3; - "~*(?:\b)Getintent(?:\b|)" 3; - "~*(?:\b)GetRight(?:\b|)" 3; - "~*(?:\b)GetWeb(?:\b|)" 3; - "~*(?:\b)Gigablast(?:\b|)" 3; - "~*(?:\b)Gigabot(?:\b|)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; - "~*(?:\b)Gotit(?:\b|)" 3; - "~*(?:\b)GoZilla(?:\b|)" 3; - "~*(?:\b)Go!Zilla(?:\b|)" 3; - "~*(?:\b)Grabber(?:\b|)" 3; - "~*(?:\b)GrabNet(?:\b|)" 3; - "~*(?:\b)Grafula(?:\b|)" 3; - "~*(?:\b)GrapeFX(?:\b|)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; - "~*(?:\b)GridBot(?:\b|)" 3; - "~*(?:\b)GT::WWW(?:\b|)" 3; - "~*(?:\b)Haansoft(?:\b|)" 3; - "~*(?:\b)HaosouSpider(?:\b|)" 3; - "~*(?:\b)Harvest(?:\b|)" 3; - "~*(?:\b)Havij(?:\b|)" 3; - "~*(?:\b)HEADMasterSEO(?:\b|)" 3; - "~*(?:\b)Heritrix(?:\b|)" 3; - "~*(?:\b)Hloader(?:\b|)" 3; - "~*(?:\b)HMView(?:\b|)" 3; - "~*(?:\b)HTMLparser(?:\b|)" 3; - "~*(?:\b)HTTP::Lite(?:\b|)" 3; - "~*(?:\b)HTTrack(?:\b|)" 3; - "~*(?:\b)Humanlinks(?:\b|)" 3; - "~*(?:\b)HybridBot(?:\b|)" 3; - "~*(?:\b)Iblog(?:\b|)" 3; - "~*(?:\b)IDBot(?:\b|)" 3; - "~*(?:\b)Id-search(?:\b|)" 3; - "~*(?:\b)IlseBot(?:\b|)" 3; - "~*(?:\b)Image\ Fetch(?:\b|)" 3; - "~*(?:\b)Image\ Sucker(?:\b|)" 3; - "~*(?:\b)IndeedBot(?:\b|)" 3; - "~*(?:\b)Indy\ Library(?:\b|)" 3; - "~*(?:\b)InfoNaviRobot(?:\b|)" 3; - "~*(?:\b)InfoTekies(?:\b|)" 3; - "~*(?:\b)instabid(?:\b|)" 3; - "~*(?:\b)Intelliseek(?:\b|)" 3; - "~*(?:\b)InterGET(?:\b|)" 3; - "~*(?:\b)Internet\ Ninja(?:\b|)" 3; - "~*(?:\b)InternetSeer(?:\b|)" 3; - "~*(?:\b)internetVista\ monitor(?:\b|)" 3; - "~*(?:\b)ips-agent(?:\b|)" 3; - "~*(?:\b)Iria(?:\b|)" 3; - "~*(?:\b)IRLbot(?:\b|)" 3; - "~*(?:\b)Iskanie(?:\b|)" 3; - "~*(?:\b)IstellaBot(?:\b|)" 3; - "~*(?:\b)JamesBOT(?:\b|)" 3; - "~*(?:\b)Jbrofuzz(?:\b|)" 3; - "~*(?:\b)JennyBot(?:\b|)" 3; - "~*(?:\b)JetCar(?:\b|)" 3; - "~*(?:\b)Jetty(?:\b|)" 3; - "~*(?:\b)JikeSpider(?:\b|)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; - "~*(?:\b)Joomla(?:\b|)" 3; - "~*(?:\b)Jorgee(?:\b|)" 3; - "~*(?:\b)JustView(?:\b|)" 3; - "~*(?:\b)Jyxobot(?:\b|)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; - "~*(?:\b)Keyword\ Density(?:\b|)" 3; - "~*(?:\b)Kozmosbot(?:\b|)" 3; - "~*(?:\b)Lanshanbot(?:\b|)" 3; - "~*(?:\b)Larbin(?:\b|)" 3; - "~*(?:\b)LeechFTP(?:\b|)" 3; - "~*(?:\b)LeechGet(?:\b|)" 3; - "~*(?:\b)LexiBot(?:\b|)" 3; - "~*(?:\b)Lftp(?:\b|)" 3; - "~*(?:\b)LibWeb(?:\b|)" 3; - "~*(?:\b)Libwhisker(?:\b|)" 3; - "~*(?:\b)Lightspeedsystems(?:\b|)" 3; - "~*(?:\b)Likse(?:\b|)" 3; - "~*(?:\b)Linkdexbot(?:\b|)" 3; - "~*(?:\b)LinkextractorPro(?:\b|)" 3; - "~*(?:\b)LinkpadBot(?:\b|)" 3; - "~*(?:\b)LinkScan(?:\b|)" 3; - "~*(?:\b)LinksManager(?:\b|)" 3; - "~*(?:\b)LinkWalker(?:\b|)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; - "~*(?:\b)Lipperhey(?:\b|)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; - "~*(?:\b)Litemage_walker(?:\b|)" 3; - "~*(?:\b)Lmspider(?:\b|)" 3; - "~*(?:\b)LNSpiderguy(?:\b|)" 3; - "~*(?:\b)Ltx71(?:\b|)" 3; - "~*(?:\b)lwp-request(?:\b|)" 3; - "~*(?:\b)LWP::Simple(?:\b|)" 3; - "~*(?:\b)lwp-trivial(?:\b|)" 3; - "~*(?:\b)Magnet(?:\b|)" 3; - "~*(?:\b)Mag-Net(?:\b|)" 3; - "~*(?:\b)magpie-crawler(?:\b|)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; - "~*(?:\b)Majestic12(?:\b|)" 3; - "~*(?:\b)Majestic-SEO(?:\b|)" 3; - "~*(?:\b)Majestic\ SEO(?:\b|)" 3; - "~*(?:\b)MarkMonitor(?:\b|)" 3; - "~*(?:\b)MarkWatch(?:\b|)" 3; - "~*(?:\b)Masscan(?:\b|)" 3; - "~*(?:\b)Mass\ Downloader(?:\b|)" 3; - "~*(?:\b)Mata\ Hari(?:\b|)" 3; - "~*(?:\b)MauiBot(?:\b|)" 3; - "~*(?:\b)meanpathbot(?:\b|)" 3; - "~*(?:\b)Meanpathbot(?:\b|)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; - "~*(?:\b)mediawords(?:\b|)" 3; - "~*(?:\b)MegaIndex.ru(?:\b|)" 3; - "~*(?:\b)Metauri(?:\b|)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; - "~*(?:\b)MIDown\ tool(?:\b|)" 3; - "~*(?:\b)MIIxpc(?:\b|)" 3; - "~*(?:\b)Mister\ PiX(?:\b|)" 3; - "~*(?:\b)MJ12bot(?:\b|)" 3; - "~*(?:\b)Mojeek(?:\b|)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; - "~*(?:\b)Mr.4x3(?:\b|)" 3; - "~*(?:\b)MSFrontPage(?:\b|)" 3; - "~*(?:\b)MSIECrawler(?:\b|)" 3; - "~*(?:\b)Msrabot(?:\b|)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; - "~*(?:\b)muhstik-scan(?:\b|)" 3; - "~*(?:\b)Musobot(?:\b|)" 3; - "~*(?:\b)Name\ Intelligence(?:\b|)" 3; - "~*(?:\b)Nameprotect(?:\b|)" 3; - "~*(?:\b)Navroad(?:\b|)" 3; - "~*(?:\b)NearSite(?:\b|)" 3; - "~*(?:\b)Needle(?:\b|)" 3; - "~*(?:\b)Nessus(?:\b|)" 3; - "~*(?:\b)NetAnts(?:\b|)" 3; - "~*(?:\b)Netcraft(?:\b|)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; - "~*(?:\b)NetLyzer(?:\b|)" 3; - "~*(?:\b)NetMechanic(?:\b|)" 3; - "~*(?:\b)NetSpider(?:\b|)" 3; - "~*(?:\b)Nettrack(?:\b|)" 3; - "~*(?:\b)Net\ Vampire(?:\b|)" 3; - "~*(?:\b)Netvibes(?:\b|)" 3; - "~*(?:\b)NetZIP(?:\b|)" 3; - "~*(?:\b)NextGenSearchBot(?:\b|)" 3; - "~*(?:\b)Nibbler(?:\b|)" 3; - "~*(?:\b)NICErsPRO(?:\b|)" 3; - "~*(?:\b)Niki-bot(?:\b|)" 3; - "~*(?:\b)Nikto(?:\b|)" 3; - "~*(?:\b)NimbleCrawler(?:\b|)" 3; - "~*(?:\b)Nimbostratus(?:\b|)" 3; - "~*(?:\b)Ninja(?:\b|)" 3; - "~*(?:\b)Nmap(?:\b|)" 3; - "~*(?:\b)NPbot(?:\b|)" 3; - "~*(?:\b)Nutch(?:\b|)" 3; - "~*(?:\b)oBot(?:\b|)" 3; - "~*(?:\b)Octopus(?:\b|)" 3; - "~*(?:\b)Offline\ Explorer(?:\b|)" 3; - "~*(?:\b)Offline\ Navigator(?:\b|)" 3; - "~*(?:\b)OnCrawl(?:\b|)" 3; - "~*(?:\b)Openfind(?:\b|)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; - "~*(?:\b)Openvas(?:\b|)" 3; - "~*(?:\b)OpenVAS(?:\b|)" 3; - "~*(?:\b)OrangeBot(?:\b|)" 3; - "~*(?:\b)OrangeSpider(?:\b|)" 3; - "~*(?:\b)OutclicksBot(?:\b|)" 3; - "~*(?:\b)OutfoxBot(?:\b|)" 3; - "~*(?:\b)PageAnalyzer(?:\b|)" 3; - "~*(?:\b)Page\ Analyzer(?:\b|)" 3; - "~*(?:\b)PageGrabber(?:\b|)" 3; - "~*(?:\b)page\ scorer(?:\b|)" 3; - "~*(?:\b)PageScorer(?:\b|)" 3; - "~*(?:\b)Pandalytics(?:\b|)" 3; - "~*(?:\b)Panscient(?:\b|)" 3; - "~*(?:\b)Papa\ Foto(?:\b|)" 3; - "~*(?:\b)Pavuk(?:\b|)" 3; - "~*(?:\b)pcBrowser(?:\b|)" 3; - "~*(?:\b)PECL::HTTP(?:\b|)" 3; - "~*(?:\b)PeoplePal(?:\b|)" 3; - "~*(?:\b)PHPCrawl(?:\b|)" 3; - "~*(?:\b)Picscout(?:\b|)" 3; - "~*(?:\b)Picsearch(?:\b|)" 3; - "~*(?:\b)PictureFinder(?:\b|)" 3; - "~*(?:\b)Pimonster(?:\b|)" 3; - "~*(?:\b)Pi-Monster(?:\b|)" 3; - "~*(?:\b)Pixray(?:\b|)" 3; - "~*(?:\b)PleaseCrawl(?:\b|)" 3; - "~*(?:\b)plumanalytics(?:\b|)" 3; - "~*(?:\b)Pockey(?:\b|)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; - "~*(?:\b)Probethenet(?:\b|)" 3; - "~*(?:\b)ProPowerBot(?:\b|)" 3; - "~*(?:\b)ProWebWalker(?:\b|)" 3; - "~*(?:\b)Psbot(?:\b|)" 3; - "~*(?:\b)Pump(?:\b|)" 3; - "~*(?:\b)PxBroker(?:\b|)" 3; - "~*(?:\b)PyCurl(?:\b|)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; - "~*(?:\b)Quick-Crawler(?:\b|)" 3; - "~*(?:\b)RankActive(?:\b|)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; - "~*(?:\b)RankFlex(?:\b|)" 3; - "~*(?:\b)RankingBot(?:\b|)" 3; - "~*(?:\b)RankingBot2(?:\b|)" 3; - "~*(?:\b)Rankivabot(?:\b|)" 3; - "~*(?:\b)RankurBot(?:\b|)" 3; - "~*(?:\b)RealDownload(?:\b|)" 3; - "~*(?:\b)Reaper(?:\b|)" 3; - "~*(?:\b)RebelMouse(?:\b|)" 3; - "~*(?:\b)Recorder(?:\b|)" 3; - "~*(?:\b)RedesScrapy(?:\b|)" 3; - "~*(?:\b)ReGet(?:\b|)" 3; - "~*(?:\b)RepoMonkey(?:\b|)" 3; - "~*(?:\b)Ripper(?:\b|)" 3; - "~*(?:\b)RocketCrawler(?:\b|)" 3; - "~*(?:\b)Rogerbot(?:\b|)" 3; - "~*(?:\b)s1z.ru(?:\b|)" 3; - "~*(?:\b)SalesIntelligent(?:\b|)" 3; - "~*(?:\b)SBIder(?:\b|)" 3; - "~*(?:\b)ScanAlert(?:\b|)" 3; - "~*(?:\b)Scanbot(?:\b|)" 3; - "~*(?:\b)scan.lol(?:\b|)" 3; - "~*(?:\b)ScoutJet(?:\b|)" 3; - "~*(?:\b)Scrapy(?:\b|)" 3; - "~*(?:\b)Screaming(?:\b|)" 3; - "~*(?:\b)ScreenerBot(?:\b|)" 3; - "~*(?:\b)Searchestate(?:\b|)" 3; - "~*(?:\b)SearchmetricsBot(?:\b|)" 3; - "~*(?:\b)Semrush(?:\b|)" 3; - "~*(?:\b)SemrushBot(?:\b|)" 3; - "~*(?:\b)SEOkicks(?:\b|)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; - "~*(?:\b)Seomoz(?:\b|)" 3; - "~*(?:\b)SEOprofiler(?:\b|)" 3; - "~*(?:\b)seoscanners(?:\b|)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; - "~*(?:\b)SEOstats(?:\b|)" 3; - "~*(?:\b)serpstatbot(?:\b|)" 3; - "~*(?:\b)sexsearcher(?:\b|)" 3; - "~*(?:\b)Shodan(?:\b|)" 3; - "~*(?:\b)Siphon(?:\b|)" 3; - "~*(?:\b)SISTRIX(?:\b|)" 3; - "~*(?:\b)Sitebeam(?:\b|)" 3; - "~*(?:\b)SiteExplorer(?:\b|)" 3; - "~*(?:\b)Siteimprove(?:\b|)" 3; - "~*(?:\b)SiteLockSpider(?:\b|)" 3; - "~*(?:\b)SiteSnagger(?:\b|)" 3; - "~*(?:\b)SiteSucker(?:\b|)" 3; - "~*(?:\b)Site\ Sucker(?:\b|)" 3; - "~*(?:\b)Sitevigil(?:\b|)" 3; - "~*(?:\b)SlySearch(?:\b|)" 3; - "~*(?:\b)SmartDownload(?:\b|)" 3; - "~*(?:\b)SMTBot(?:\b|)" 3; - "~*(?:\b)Snake(?:\b|)" 3; - "~*(?:\b)Snapbot(?:\b|)" 3; - "~*(?:\b)Snoopy(?:\b|)" 3; - "~*(?:\b)SocialRankIOBot(?:\b|)" 3; - "~*(?:\b)Sociscraper(?:\b|)" 3; - "~*(?:\b)sogouspider(?:\b|)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; - "~*(?:\b)Sosospider(?:\b|)" 3; - "~*(?:\b)Sottopop(?:\b|)" 3; - "~*(?:\b)SpaceBison(?:\b|)" 3; - "~*(?:\b)Spammen(?:\b|)" 3; - "~*(?:\b)SpankBot(?:\b|)" 3; - "~*(?:\b)Spanner(?:\b|)" 3; - "~*(?:\b)sp_auditbot(?:\b|)" 3; - "~*(?:\b)Spbot(?:\b|)" 3; - "~*(?:\b)Spinn3r(?:\b|)" 3; - "~*(?:\b)SputnikBot(?:\b|)" 3; - "~*(?:\b)spyfu(?:\b|)" 3; - "~*(?:\b)Sqlmap(?:\b|)" 3; - "~*(?:\b)Sqlworm(?:\b|)" 3; - "~*(?:\b)Sqworm(?:\b|)" 3; - "~*(?:\b)Steeler(?:\b|)" 3; - "~*(?:\b)Stripper(?:\b|)" 3; - "~*(?:\b)Sucker(?:\b|)" 3; - "~*(?:\b)Sucuri(?:\b|)" 3; - "~*(?:\b)SuperBot(?:\b|)" 3; - "~*(?:\b)SuperHTTP(?:\b|)" 3; - "~*(?:\b)Surfbot(?:\b|)" 3; - "~*(?:\b)SurveyBot(?:\b|)" 3; - "~*(?:\b)Suzuran(?:\b|)" 3; - "~*(?:\b)Swiftbot(?:\b|)" 3; - "~*(?:\b)sysscan(?:\b|)" 3; - "~*(?:\b)Szukacz(?:\b|)" 3; - "~*(?:\b)T0PHackTeam(?:\b|)" 3; - "~*(?:\b)T8Abot(?:\b|)" 3; - "~*(?:\b)tAkeOut(?:\b|)" 3; - "~*(?:\b)Teleport(?:\b|)" 3; - "~*(?:\b)TeleportPro(?:\b|)" 3; - "~*(?:\b)Telesoft(?:\b|)" 3; - "~*(?:\b)Telesphoreo(?:\b|)" 3; - "~*(?:\b)Telesphorep(?:\b|)" 3; - "~*(?:\b)The\ Intraformant(?:\b|)" 3; - "~*(?:\b)TheNomad(?:\b|)" 3; - "~*(?:\b)Thumbor(?:\b|)" 3; - "~*(?:\b)TightTwatBot(?:\b|)" 3; - "~*(?:\b)Titan(?:\b|)" 3; - "~*(?:\b)Toata(?:\b|)" 3; - "~*(?:\b)Toweyabot(?:\b|)" 3; - "~*(?:\b)Tracemyfile(?:\b|)" 3; - "~*(?:\b)Trendiction(?:\b|)" 3; - "~*(?:\b)Trendictionbot(?:\b|)" 3; - "~*(?:\b)trendiction.com(?:\b|)" 3; - "~*(?:\b)trendiction.de(?:\b|)" 3; - "~*(?:\b)True_Robot(?:\b|)" 3; - "~*(?:\b)Turingos(?:\b|)" 3; - "~*(?:\b)Turnitin(?:\b|)" 3; - "~*(?:\b)TurnitinBot(?:\b|)" 3; - "~*(?:\b)TwengaBot(?:\b|)" 3; - "~*(?:\b)Twice(?:\b|)" 3; - "~*(?:\b)Typhoeus(?:\b|)" 3; - "~*(?:\b)UnisterBot(?:\b|)" 3; - "~*(?:\b)Upflow(?:\b|)" 3; - "~*(?:\b)URLy.Warning(?:\b|)" 3; - "~*(?:\b)URLy\ Warning(?:\b|)" 3; - "~*(?:\b)Vacuum(?:\b|)" 3; - "~*(?:\b)Vagabondo(?:\b|)" 3; - "~*(?:\b)VB\ Project(?:\b|)" 3; - "~*(?:\b)VCI(?:\b|)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; - "~*(?:\b)VidibleScraper(?:\b|)" 3; - "~*(?:\b)Virusdie(?:\b|)" 3; - "~*(?:\b)VoidEYE(?:\b|)" 3; - "~*(?:\b)Voil(?:\b|)" 3; - "~*(?:\b)Voltron(?:\b|)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; - "~*(?:\b)WallpapersHD(?:\b|)" 3; - "~*(?:\b)WASALive-Bot(?:\b|)" 3; - "~*(?:\b)WBSearchBot(?:\b|)" 3; - "~*(?:\b)Webalta(?:\b|)" 3; - "~*(?:\b)WebAuto(?:\b|)" 3; - "~*(?:\b)Web\ Auto(?:\b|)" 3; - "~*(?:\b)WebBandit(?:\b|)" 3; - "~*(?:\b)WebCollage(?:\b|)" 3; - "~*(?:\b)Web\ Collage(?:\b|)" 3; - "~*(?:\b)WebCopier(?:\b|)" 3; - "~*(?:\b)WEBDAV(?:\b|)" 3; - "~*(?:\b)WebEnhancer(?:\b|)" 3; - "~*(?:\b)Web\ Enhancer(?:\b|)" 3; - "~*(?:\b)WebFetch(?:\b|)" 3; - "~*(?:\b)Web\ Fetch(?:\b|)" 3; - "~*(?:\b)WebFuck(?:\b|)" 3; - "~*(?:\b)Web\ Fuck(?:\b|)" 3; - "~*(?:\b)WebGo\ IS(?:\b|)" 3; - "~*(?:\b)WebImageCollector(?:\b|)" 3; - "~*(?:\b)WebLeacher(?:\b|)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; - "~*(?:\b)webmeup-crawler(?:\b|)" 3; - "~*(?:\b)WebPix(?:\b|)" 3; - "~*(?:\b)Web\ Pix(?:\b|)" 3; - "~*(?:\b)WebReaper(?:\b|)" 3; - "~*(?:\b)WebSauger(?:\b|)" 3; - "~*(?:\b)Web\ Sauger(?:\b|)" 3; - "~*(?:\b)Webshag(?:\b|)" 3; - "~*(?:\b)WebsiteExtractor(?:\b|)" 3; - "~*(?:\b)WebsiteQuester(?:\b|)" 3; - "~*(?:\b)Website\ Quester(?:\b|)" 3; - "~*(?:\b)Webster(?:\b|)" 3; - "~*(?:\b)WebStripper(?:\b|)" 3; - "~*(?:\b)WebSucker(?:\b|)" 3; - "~*(?:\b)Web\ Sucker(?:\b|)" 3; - "~*(?:\b)WebWhacker(?:\b|)" 3; - "~*(?:\b)WebZIP(?:\b|)" 3; - "~*(?:\b)WeSEE(?:\b|)" 3; - "~*(?:\b)Whack(?:\b|)" 3; - "~*(?:\b)Whacker(?:\b|)" 3; - "~*(?:\b)Whatweb(?:\b|)" 3; - "~*(?:\b)Who.is\ Bot(?:\b|)" 3; - "~*(?:\b)Widow(?:\b|)" 3; - "~*(?:\b)WinHTTrack(?:\b|)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; - "~*(?:\b)WISENutbot(?:\b|)" 3; - "~*(?:\b)Wonderbot(?:\b|)" 3; - "~*(?:\b)Woobot(?:\b|)" 3; - "~*(?:\b)Wotbox(?:\b|)" 3; - "~*(?:\b)Wprecon(?:\b|)" 3; - "~*(?:\b)WPScan(?:\b|)" 3; - "~*(?:\b)WWW-Collector-E(?:\b|)" 3; - "~*(?:\b)WWW-Mechanize(?:\b|)" 3; - "~*(?:\b)WWW::Mechanize(?:\b|)" 3; - "~*(?:\b)WWWOFFLE(?:\b|)" 3; - "~*(?:\b)x09Mozilla(?:\b|)" 3; - "~*(?:\b)x22Mozilla(?:\b|)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; - "~*(?:\b)Xenu(?:\b|)" 3; - "~*(?:\b)xpymep1.exe(?:\b|)" 3; - "~*(?:\b)YoudaoBot(?:\b|)" 3; - "~*(?:\b)Zade(?:\b|)" 3; - "~*(?:\b)Zauba(?:\b|)" 3; - "~*(?:\b)zauba.io(?:\b|)" 3; - "~*(?:\b)Zermelo(?:\b|)" 3; - "~*(?:\b)Zeus(?:\b|)" 3; - "~*(?:\b)zgrab(?:\b|)" 3; - "~*(?:\b)Zitebot(?:\b|)" 3; - "~*(?:\b)ZmEu(?:\b|)" 3; - "~*(?:\b)ZumBot(?:\b|)" 3; - "~*(?:\b)ZyBorg(?:\b|)" 3; + "~*(?:\b)360Spider(?:\b)" 3; + "~*(?:\b)404checker(?:\b)" 3; + "~*(?:\b)404enemy(?:\b)" 3; + "~*(?:\b)80legs(?:\b)" 3; + "~*(?:\b)Abonti(?:\b)" 3; + "~*(?:\b)Aboundex(?:\b)" 3; + "~*(?:\b)Aboundexbot(?:\b)" 3; + "~*(?:\b)Acunetix(?:\b)" 3; + "~*(?:\b)ADmantX(?:\b)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; + "~*(?:\b)AhrefsBot(?:\b)" 3; + "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)AiHitBot(?:\b)" 3; + "~*(?:\b)Aipbot(?:\b)" 3; + "~*(?:\b)Alexibot(?:\b)" 3; + "~*(?:\b)Alligator(?:\b)" 3; + "~*(?:\b)AllSubmitter(?:\b)" 3; + "~*(?:\b)AlphaBot(?:\b)" 3; + "~*(?:\b)Anarchie(?:\b)" 3; + "~*(?:\b)Apexoo(?:\b)" 3; + "~*(?:\b)archive.org_bot(?:\b)" 3; + "~*(?:\b)ASPSeek(?:\b)" 3; + "~*(?:\b)Asterias(?:\b)" 3; + "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)BackDoorBot(?:\b)" 3; + "~*(?:\b)Backlink-Ceck(?:\b)" 3; + "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)BacklinkCrawler(?:\b)" 3; + "~*(?:\b)BackStreet(?:\b)" 3; + "~*(?:\b)BackWeb(?:\b)" 3; + "~*(?:\b)Badass(?:\b)" 3; + "~*(?:\b)Bandit(?:\b)" 3; + "~*(?:\b)Barkrowler(?:\b)" 3; + "~*(?:\b)BatchFTP(?:\b)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; + "~*(?:\b)BBBike(?:\b)" 3; + "~*(?:\b)BDCbot(?:\b)" 3; + "~*(?:\b)BDFetch(?:\b)" 3; + "~*(?:\b)BetaBot(?:\b)" 3; + "~*(?:\b)Bigfoot(?:\b)" 3; + "~*(?:\b)Bitacle(?:\b)" 3; + "~*(?:\b)Blackboard(?:\b)" 3; + "~*(?:\b)Black\ Hole(?:\b)" 3; + "~*(?:\b)BlackWidow(?:\b)" 3; + "~*(?:\b)BLEXBot(?:\b)" 3; + "~*(?:\b)Blow(?:\b)" 3; + "~*(?:\b)BlowFish(?:\b)" 3; + "~*(?:\b)Boardreader(?:\b)" 3; + "~*(?:\b)Bolt(?:\b)" 3; + "~*(?:\b)BotALot(?:\b)" 3; + "~*(?:\b)Brandprotect(?:\b)" 3; + "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buddy(?:\b)" 3; + "~*(?:\b)BuiltBotTough(?:\b)" 3; + "~*(?:\b)BuiltWith(?:\b)" 3; + "~*(?:\b)Bullseye(?:\b)" 3; + "~*(?:\b)BunnySlippers(?:\b)" 3; + "~*(?:\b)BuzzSumo(?:\b)" 3; + "~*(?:\b)Calculon(?:\b)" 3; + "~*(?:\b)CATExplorador(?:\b)" 3; + "~*(?:\b)CazoodleBot(?:\b)" 3; + "~*(?:\b)CCBot(?:\b)" 3; + "~*(?:\b)Cegbfeieh(?:\b)" 3; + "~*(?:\b)CheeseBot(?:\b)" 3; + "~*(?:\b)CherryPicker(?:\b)" 3; + "~*(?:\b)CheTeam(?:\b)" 3; + "~*(?:\b)ChinaClaw(?:\b)" 3; + "~*(?:\b)Chlooe(?:\b)" 3; + "~*(?:\b)Claritybot(?:\b)" 3; + "~*(?:\b)Cliqzbot(?:\b)" 3; + "~*(?:\b)Cloud\ mapping(?:\b)" 3; + "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cogentbot(?:\b)" 3; + "~*(?:\b)cognitiveseo(?:\b)" 3; + "~*(?:\b)Collector(?:\b)" 3; + "~*(?:\b)com.plumanalytics(?:\b)" 3; + "~*(?:\b)Copier(?:\b)" 3; + "~*(?:\b)CopyRightCheck(?:\b)" 3; + "~*(?:\b)Copyscape(?:\b)" 3; + "~*(?:\b)Cosmos(?:\b)" 3; + "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)crawler4j(?:\b)" 3; + "~*(?:\b)crawler.feedback(?:\b)" 3; + "~*(?:\b)crawl.sogou.com(?:\b)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b)" 3; + "~*(?:\b)Crescent(?:\b)" 3; + "~*(?:\b)CrunchBot(?:\b)" 3; + "~*(?:\b)CSHttp(?:\b)" 3; + "~*(?:\b)Curious(?:\b)" 3; + "~*(?:\b)Custo(?:\b)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; + "~*(?:\b)DataCha0s(?:\b)" 3; + "~*(?:\b)DBLBot(?:\b)" 3; + "~*(?:\b)demandbase-bot(?:\b)" 3; + "~*(?:\b)Demon(?:\b)" 3; + "~*(?:\b)Deusu(?:\b)" 3; + "~*(?:\b)Devil(?:\b)" 3; + "~*(?:\b)Digincore(?:\b)" 3; + "~*(?:\b)DigitalPebble(?:\b)" 3; + "~*(?:\b)DIIbot(?:\b)" 3; + "~*(?:\b)Dirbuster(?:\b)" 3; + "~*(?:\b)Disco(?:\b)" 3; + "~*(?:\b)Discobot(?:\b)" 3; + "~*(?:\b)Discoverybot(?:\b)" 3; + "~*(?:\b)Dispatch(?:\b)" 3; + "~*(?:\b)DittoSpyder(?:\b)" 3; + "~*(?:\b)DnyzBot(?:\b)" 3; + "~*(?:\b)DomainAppender(?:\b)" 3; + "~*(?:\b)DomainCrawler(?:\b)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)DomainStatsBot(?:\b)" 3; + "~*(?:\b)Dotbot(?:\b)" 3; + "~*(?:\b)Download\ Wonder(?:\b)" 3; + "~*(?:\b)Dragonfly(?:\b)" 3; + "~*(?:\b)Drip(?:\b)" 3; + "~*(?:\b)DSearch(?:\b)" 3; + "~*(?:\b)DTS\ Agent(?:\b)" 3; + "~*(?:\b)EasyDL(?:\b)" 3; + "~*(?:\b)Ebingbong(?:\b)" 3; + "~*(?:\b)eCatch(?:\b)" 3; + "~*(?:\b)ECCP/1.0(?:\b)" 3; + "~*(?:\b)Ecxi(?:\b)" 3; + "~*(?:\b)EirGrabber(?:\b)" 3; + "~*(?:\b)EMail\ Siphon(?:\b)" 3; + "~*(?:\b)EMail\ Wolf(?:\b)" 3; + "~*(?:\b)EroCrawler(?:\b)" 3; + "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)Evil(?:\b)" 3; + "~*(?:\b)Exabot(?:\b)" 3; + "~*(?:\b)Express\ WebPictures(?:\b)" 3; + "~*(?:\b)ExtLinksBot(?:\b)" 3; + "~*(?:\b)Extractor(?:\b)" 3; + "~*(?:\b)ExtractorPro(?:\b)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; + "~*(?:\b)EyeNetIE(?:\b)" 3; + "~*(?:\b)Ezooms(?:\b)" 3; + "~*(?:\b)facebookscraper(?:\b)" 3; + "~*(?:\b)FDM(?:\b)" 3; + "~*(?:\b)FemtosearchBot(?:\b)" 3; + "~*(?:\b)FHscan(?:\b)" 3; + "~*(?:\b)Fimap(?:\b)" 3; + "~*(?:\b)Firefox/7.0(?:\b)" 3; + "~*(?:\b)FlashGet(?:\b)" 3; + "~*(?:\b)Flunky(?:\b)" 3; + "~*(?:\b)Foobot(?:\b)" 3; + "~*(?:\b)Freeuploader(?:\b)" 3; + "~*(?:\b)FrontPage(?:\b)" 3; + "~*(?:\b)FyberSpider(?:\b)" 3; + "~*(?:\b)Fyrebot(?:\b)" 3; + "~*(?:\b)GalaxyBot(?:\b)" 3; + "~*(?:\b)Genieo(?:\b)" 3; + "~*(?:\b)GermCrawler(?:\b)" 3; + "~*(?:\b)Getintent(?:\b)" 3; + "~*(?:\b)GetRight(?:\b)" 3; + "~*(?:\b)GetWeb(?:\b)" 3; + "~*(?:\b)Gigablast(?:\b)" 3; + "~*(?:\b)Gigabot(?:\b)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; + "~*(?:\b)Gotit(?:\b)" 3; + "~*(?:\b)GoZilla(?:\b)" 3; + "~*(?:\b)Go!Zilla(?:\b)" 3; + "~*(?:\b)Grabber(?:\b)" 3; + "~*(?:\b)GrabNet(?:\b)" 3; + "~*(?:\b)Grafula(?:\b)" 3; + "~*(?:\b)GrapeFX(?:\b)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b)" 3; + "~*(?:\b)GridBot(?:\b)" 3; + "~*(?:\b)GT::WWW(?:\b)" 3; + "~*(?:\b)Haansoft(?:\b)" 3; + "~*(?:\b)HaosouSpider(?:\b)" 3; + "~*(?:\b)Harvest(?:\b)" 3; + "~*(?:\b)Havij(?:\b)" 3; + "~*(?:\b)HEADMasterSEO(?:\b)" 3; + "~*(?:\b)Heritrix(?:\b)" 3; + "~*(?:\b)Hloader(?:\b)" 3; + "~*(?:\b)HMView(?:\b)" 3; + "~*(?:\b)HTMLparser(?:\b)" 3; + "~*(?:\b)HTTP::Lite(?:\b)" 3; + "~*(?:\b)HTTrack(?:\b)" 3; + "~*(?:\b)Humanlinks(?:\b)" 3; + "~*(?:\b)HybridBot(?:\b)" 3; + "~*(?:\b)Iblog(?:\b)" 3; + "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)Id-search(?:\b)" 3; + "~*(?:\b)IlseBot(?:\b)" 3; + "~*(?:\b)Image\ Fetch(?:\b)" 3; + "~*(?:\b)Image\ Sucker(?:\b)" 3; + "~*(?:\b)IndeedBot(?:\b)" 3; + "~*(?:\b)Indy\ Library(?:\b)" 3; + "~*(?:\b)InfoNaviRobot(?:\b)" 3; + "~*(?:\b)InfoTekies(?:\b)" 3; + "~*(?:\b)instabid(?:\b)" 3; + "~*(?:\b)Intelliseek(?:\b)" 3; + "~*(?:\b)InterGET(?:\b)" 3; + "~*(?:\b)Internet\ Ninja(?:\b)" 3; + "~*(?:\b)InternetSeer(?:\b)" 3; + "~*(?:\b)internetVista\ monitor(?:\b)" 3; + "~*(?:\b)ips-agent(?:\b)" 3; + "~*(?:\b)Iria(?:\b)" 3; + "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)Iskanie(?:\b)" 3; + "~*(?:\b)IstellaBot(?:\b)" 3; + "~*(?:\b)JamesBOT(?:\b)" 3; + "~*(?:\b)Jbrofuzz(?:\b)" 3; + "~*(?:\b)JennyBot(?:\b)" 3; + "~*(?:\b)JetCar(?:\b)" 3; + "~*(?:\b)Jetty(?:\b)" 3; + "~*(?:\b)JikeSpider(?:\b)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; + "~*(?:\b)Joomla(?:\b)" 3; + "~*(?:\b)Jorgee(?:\b)" 3; + "~*(?:\b)JustView(?:\b)" 3; + "~*(?:\b)Jyxobot(?:\b)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b)" 3; + "~*(?:\b)Keyword\ Density(?:\b)" 3; + "~*(?:\b)Kozmosbot(?:\b)" 3; + "~*(?:\b)Lanshanbot(?:\b)" 3; + "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)LeechFTP(?:\b)" 3; + "~*(?:\b)LeechGet(?:\b)" 3; + "~*(?:\b)LexiBot(?:\b)" 3; + "~*(?:\b)Lftp(?:\b)" 3; + "~*(?:\b)LibWeb(?:\b)" 3; + "~*(?:\b)Libwhisker(?:\b)" 3; + "~*(?:\b)Lightspeedsystems(?:\b)" 3; + "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkdexbot(?:\b)" 3; + "~*(?:\b)LinkextractorPro(?:\b)" 3; + "~*(?:\b)LinkpadBot(?:\b)" 3; + "~*(?:\b)LinkScan(?:\b)" 3; + "~*(?:\b)LinksManager(?:\b)" 3; + "~*(?:\b)LinkWalker(?:\b)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; + "~*(?:\b)Lipperhey(?:\b)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; + "~*(?:\b)Litemage_walker(?:\b)" 3; + "~*(?:\b)Lmspider(?:\b)" 3; + "~*(?:\b)LNSpiderguy(?:\b)" 3; + "~*(?:\b)Ltx71(?:\b)" 3; + "~*(?:\b)lwp-request(?:\b)" 3; + "~*(?:\b)LWP::Simple(?:\b)" 3; + "~*(?:\b)lwp-trivial(?:\b)" 3; + "~*(?:\b)Magnet(?:\b)" 3; + "~*(?:\b)Mag-Net(?:\b)" 3; + "~*(?:\b)magpie-crawler(?:\b)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b)" 3; + "~*(?:\b)Majestic12(?:\b)" 3; + "~*(?:\b)Majestic-SEO(?:\b)" 3; + "~*(?:\b)Majestic\ SEO(?:\b)" 3; + "~*(?:\b)MarkMonitor(?:\b)" 3; + "~*(?:\b)MarkWatch(?:\b)" 3; + "~*(?:\b)Masscan(?:\b)" 3; + "~*(?:\b)Mass\ Downloader(?:\b)" 3; + "~*(?:\b)Mata\ Hari(?:\b)" 3; + "~*(?:\b)MauiBot(?:\b)" 3; + "~*(?:\b)meanpathbot(?:\b)" 3; + "~*(?:\b)Meanpathbot(?:\b)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b)" 3; + "~*(?:\b)mediawords(?:\b)" 3; + "~*(?:\b)MegaIndex.ru(?:\b)" 3; + "~*(?:\b)Metauri(?:\b)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; + "~*(?:\b)MIDown\ tool(?:\b)" 3; + "~*(?:\b)MIIxpc(?:\b)" 3; + "~*(?:\b)Mister\ PiX(?:\b)" 3; + "~*(?:\b)MJ12bot(?:\b)" 3; + "~*(?:\b)Mojeek(?:\b)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; + "~*(?:\b)Mr.4x3(?:\b)" 3; + "~*(?:\b)MSFrontPage(?:\b)" 3; + "~*(?:\b)MSIECrawler(?:\b)" 3; + "~*(?:\b)Msrabot(?:\b)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; + "~*(?:\b)muhstik-scan(?:\b)" 3; + "~*(?:\b)Musobot(?:\b)" 3; + "~*(?:\b)Name\ Intelligence(?:\b)" 3; + "~*(?:\b)Nameprotect(?:\b)" 3; + "~*(?:\b)Navroad(?:\b)" 3; + "~*(?:\b)NearSite(?:\b)" 3; + "~*(?:\b)Needle(?:\b)" 3; + "~*(?:\b)Nessus(?:\b)" 3; + "~*(?:\b)NetAnts(?:\b)" 3; + "~*(?:\b)Netcraft(?:\b)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; + "~*(?:\b)NetLyzer(?:\b)" 3; + "~*(?:\b)NetMechanic(?:\b)" 3; + "~*(?:\b)NetSpider(?:\b)" 3; + "~*(?:\b)Nettrack(?:\b)" 3; + "~*(?:\b)Net\ Vampire(?:\b)" 3; + "~*(?:\b)Netvibes(?:\b)" 3; + "~*(?:\b)NetZIP(?:\b)" 3; + "~*(?:\b)NextGenSearchBot(?:\b)" 3; + "~*(?:\b)Nibbler(?:\b)" 3; + "~*(?:\b)NICErsPRO(?:\b)" 3; + "~*(?:\b)Niki-bot(?:\b)" 3; + "~*(?:\b)Nikto(?:\b)" 3; + "~*(?:\b)NimbleCrawler(?:\b)" 3; + "~*(?:\b)Nimbostratus(?:\b)" 3; + "~*(?:\b)Ninja(?:\b)" 3; + "~*(?:\b)Nmap(?:\b)" 3; + "~*(?:\b)NPbot(?:\b)" 3; + "~*(?:\b)Nutch(?:\b)" 3; + "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)Octopus(?:\b)" 3; + "~*(?:\b)Offline\ Explorer(?:\b)" 3; + "~*(?:\b)Offline\ Navigator(?:\b)" 3; + "~*(?:\b)OnCrawl(?:\b)" 3; + "~*(?:\b)Openfind(?:\b)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b)" 3; + "~*(?:\b)Openvas(?:\b)" 3; + "~*(?:\b)OpenVAS(?:\b)" 3; + "~*(?:\b)OrangeBot(?:\b)" 3; + "~*(?:\b)OrangeSpider(?:\b)" 3; + "~*(?:\b)OutclicksBot(?:\b)" 3; + "~*(?:\b)OutfoxBot(?:\b)" 3; + "~*(?:\b)PageAnalyzer(?:\b)" 3; + "~*(?:\b)Page\ Analyzer(?:\b)" 3; + "~*(?:\b)PageGrabber(?:\b)" 3; + "~*(?:\b)page\ scorer(?:\b)" 3; + "~*(?:\b)PageScorer(?:\b)" 3; + "~*(?:\b)Pandalytics(?:\b)" 3; + "~*(?:\b)Panscient(?:\b)" 3; + "~*(?:\b)Papa\ Foto(?:\b)" 3; + "~*(?:\b)Pavuk(?:\b)" 3; + "~*(?:\b)pcBrowser(?:\b)" 3; + "~*(?:\b)PECL::HTTP(?:\b)" 3; + "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)PHPCrawl(?:\b)" 3; + "~*(?:\b)Picscout(?:\b)" 3; + "~*(?:\b)Picsearch(?:\b)" 3; + "~*(?:\b)PictureFinder(?:\b)" 3; + "~*(?:\b)Pimonster(?:\b)" 3; + "~*(?:\b)Pi-Monster(?:\b)" 3; + "~*(?:\b)Pixray(?:\b)" 3; + "~*(?:\b)PleaseCrawl(?:\b)" 3; + "~*(?:\b)plumanalytics(?:\b)" 3; + "~*(?:\b)Pockey(?:\b)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; + "~*(?:\b)Probethenet(?:\b)" 3; + "~*(?:\b)ProPowerBot(?:\b)" 3; + "~*(?:\b)ProWebWalker(?:\b)" 3; + "~*(?:\b)Psbot(?:\b)" 3; + "~*(?:\b)Pump(?:\b)" 3; + "~*(?:\b)PxBroker(?:\b)" 3; + "~*(?:\b)PyCurl(?:\b)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; + "~*(?:\b)Quick-Crawler(?:\b)" 3; + "~*(?:\b)RankActive(?:\b)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b)" 3; + "~*(?:\b)RankFlex(?:\b)" 3; + "~*(?:\b)RankingBot(?:\b)" 3; + "~*(?:\b)RankingBot2(?:\b)" 3; + "~*(?:\b)Rankivabot(?:\b)" 3; + "~*(?:\b)RankurBot(?:\b)" 3; + "~*(?:\b)RealDownload(?:\b)" 3; + "~*(?:\b)Reaper(?:\b)" 3; + "~*(?:\b)RebelMouse(?:\b)" 3; + "~*(?:\b)Recorder(?:\b)" 3; + "~*(?:\b)RedesScrapy(?:\b)" 3; + "~*(?:\b)ReGet(?:\b)" 3; + "~*(?:\b)RepoMonkey(?:\b)" 3; + "~*(?:\b)Ripper(?:\b)" 3; + "~*(?:\b)RocketCrawler(?:\b)" 3; + "~*(?:\b)Rogerbot(?:\b)" 3; + "~*(?:\b)s1z.ru(?:\b)" 3; + "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)SBIder(?:\b)" 3; + "~*(?:\b)ScanAlert(?:\b)" 3; + "~*(?:\b)Scanbot(?:\b)" 3; + "~*(?:\b)scan.lol(?:\b)" 3; + "~*(?:\b)ScoutJet(?:\b)" 3; + "~*(?:\b)Scrapy(?:\b)" 3; + "~*(?:\b)Screaming(?:\b)" 3; + "~*(?:\b)ScreenerBot(?:\b)" 3; + "~*(?:\b)Searchestate(?:\b)" 3; + "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Semrush(?:\b)" 3; + "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SEOkicks(?:\b)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; + "~*(?:\b)Seomoz(?:\b)" 3; + "~*(?:\b)SEOprofiler(?:\b)" 3; + "~*(?:\b)seoscanners(?:\b)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b)" 3; + "~*(?:\b)SEOstats(?:\b)" 3; + "~*(?:\b)serpstatbot(?:\b)" 3; + "~*(?:\b)sexsearcher(?:\b)" 3; + "~*(?:\b)Shodan(?:\b)" 3; + "~*(?:\b)Siphon(?:\b)" 3; + "~*(?:\b)SISTRIX(?:\b)" 3; + "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteExplorer(?:\b)" 3; + "~*(?:\b)Siteimprove(?:\b)" 3; + "~*(?:\b)SiteLockSpider(?:\b)" 3; + "~*(?:\b)SiteSnagger(?:\b)" 3; + "~*(?:\b)SiteSucker(?:\b)" 3; + "~*(?:\b)Site\ Sucker(?:\b)" 3; + "~*(?:\b)Sitevigil(?:\b)" 3; + "~*(?:\b)SlySearch(?:\b)" 3; + "~*(?:\b)SmartDownload(?:\b)" 3; + "~*(?:\b)SMTBot(?:\b)" 3; + "~*(?:\b)Snake(?:\b)" 3; + "~*(?:\b)Snapbot(?:\b)" 3; + "~*(?:\b)Snoopy(?:\b)" 3; + "~*(?:\b)SocialRankIOBot(?:\b)" 3; + "~*(?:\b)Sociscraper(?:\b)" 3; + "~*(?:\b)sogouspider(?:\b)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; + "~*(?:\b)Sosospider(?:\b)" 3; + "~*(?:\b)Sottopop(?:\b)" 3; + "~*(?:\b)SpaceBison(?:\b)" 3; + "~*(?:\b)Spammen(?:\b)" 3; + "~*(?:\b)SpankBot(?:\b)" 3; + "~*(?:\b)Spanner(?:\b)" 3; + "~*(?:\b)sp_auditbot(?:\b)" 3; + "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spinn3r(?:\b)" 3; + "~*(?:\b)SputnikBot(?:\b)" 3; + "~*(?:\b)spyfu(?:\b)" 3; + "~*(?:\b)Sqlmap(?:\b)" 3; + "~*(?:\b)Sqlworm(?:\b)" 3; + "~*(?:\b)Sqworm(?:\b)" 3; + "~*(?:\b)Steeler(?:\b)" 3; + "~*(?:\b)Stripper(?:\b)" 3; + "~*(?:\b)Sucker(?:\b)" 3; + "~*(?:\b)Sucuri(?:\b)" 3; + "~*(?:\b)SuperBot(?:\b)" 3; + "~*(?:\b)SuperHTTP(?:\b)" 3; + "~*(?:\b)Surfbot(?:\b)" 3; + "~*(?:\b)SurveyBot(?:\b)" 3; + "~*(?:\b)Suzuran(?:\b)" 3; + "~*(?:\b)Swiftbot(?:\b)" 3; + "~*(?:\b)sysscan(?:\b)" 3; + "~*(?:\b)Szukacz(?:\b)" 3; + "~*(?:\b)T0PHackTeam(?:\b)" 3; + "~*(?:\b)T8Abot(?:\b)" 3; + "~*(?:\b)tAkeOut(?:\b)" 3; + "~*(?:\b)Teleport(?:\b)" 3; + "~*(?:\b)TeleportPro(?:\b)" 3; + "~*(?:\b)Telesoft(?:\b)" 3; + "~*(?:\b)Telesphoreo(?:\b)" 3; + "~*(?:\b)Telesphorep(?:\b)" 3; + "~*(?:\b)The\ Intraformant(?:\b)" 3; + "~*(?:\b)TheNomad(?:\b)" 3; + "~*(?:\b)Thumbor(?:\b)" 3; + "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)Titan(?:\b)" 3; + "~*(?:\b)Toata(?:\b)" 3; + "~*(?:\b)Toweyabot(?:\b)" 3; + "~*(?:\b)Tracemyfile(?:\b)" 3; + "~*(?:\b)Trendiction(?:\b)" 3; + "~*(?:\b)Trendictionbot(?:\b)" 3; + "~*(?:\b)trendiction.com(?:\b)" 3; + "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)True_Robot(?:\b)" 3; + "~*(?:\b)Turingos(?:\b)" 3; + "~*(?:\b)Turnitin(?:\b)" 3; + "~*(?:\b)TurnitinBot(?:\b)" 3; + "~*(?:\b)TwengaBot(?:\b)" 3; + "~*(?:\b)Twice(?:\b)" 3; + "~*(?:\b)Typhoeus(?:\b)" 3; + "~*(?:\b)UnisterBot(?:\b)" 3; + "~*(?:\b)Upflow(?:\b)" 3; + "~*(?:\b)URLy.Warning(?:\b)" 3; + "~*(?:\b)URLy\ Warning(?:\b)" 3; + "~*(?:\b)Vacuum(?:\b)" 3; + "~*(?:\b)Vagabondo(?:\b)" 3; + "~*(?:\b)VB\ Project(?:\b)" 3; + "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b)" 3; + "~*(?:\b)VidibleScraper(?:\b)" 3; + "~*(?:\b)Virusdie(?:\b)" 3; + "~*(?:\b)VoidEYE(?:\b)" 3; + "~*(?:\b)Voil(?:\b)" 3; + "~*(?:\b)Voltron(?:\b)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b)" 3; + "~*(?:\b)WallpapersHD(?:\b)" 3; + "~*(?:\b)WASALive-Bot(?:\b)" 3; + "~*(?:\b)WBSearchBot(?:\b)" 3; + "~*(?:\b)Webalta(?:\b)" 3; + "~*(?:\b)WebAuto(?:\b)" 3; + "~*(?:\b)Web\ Auto(?:\b)" 3; + "~*(?:\b)WebBandit(?:\b)" 3; + "~*(?:\b)WebCollage(?:\b)" 3; + "~*(?:\b)Web\ Collage(?:\b)" 3; + "~*(?:\b)WebCopier(?:\b)" 3; + "~*(?:\b)WEBDAV(?:\b)" 3; + "~*(?:\b)WebEnhancer(?:\b)" 3; + "~*(?:\b)Web\ Enhancer(?:\b)" 3; + "~*(?:\b)WebFetch(?:\b)" 3; + "~*(?:\b)Web\ Fetch(?:\b)" 3; + "~*(?:\b)WebFuck(?:\b)" 3; + "~*(?:\b)Web\ Fuck(?:\b)" 3; + "~*(?:\b)WebGo\ IS(?:\b)" 3; + "~*(?:\b)WebImageCollector(?:\b)" 3; + "~*(?:\b)WebLeacher(?:\b)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; + "~*(?:\b)webmeup-crawler(?:\b)" 3; + "~*(?:\b)WebPix(?:\b)" 3; + "~*(?:\b)Web\ Pix(?:\b)" 3; + "~*(?:\b)WebReaper(?:\b)" 3; + "~*(?:\b)WebSauger(?:\b)" 3; + "~*(?:\b)Web\ Sauger(?:\b)" 3; + "~*(?:\b)Webshag(?:\b)" 3; + "~*(?:\b)WebsiteExtractor(?:\b)" 3; + "~*(?:\b)WebsiteQuester(?:\b)" 3; + "~*(?:\b)Website\ Quester(?:\b)" 3; + "~*(?:\b)Webster(?:\b)" 3; + "~*(?:\b)WebStripper(?:\b)" 3; + "~*(?:\b)WebSucker(?:\b)" 3; + "~*(?:\b)Web\ Sucker(?:\b)" 3; + "~*(?:\b)WebWhacker(?:\b)" 3; + "~*(?:\b)WebZIP(?:\b)" 3; + "~*(?:\b)WeSEE(?:\b)" 3; + "~*(?:\b)Whack(?:\b)" 3; + "~*(?:\b)Whacker(?:\b)" 3; + "~*(?:\b)Whatweb(?:\b)" 3; + "~*(?:\b)Who.is\ Bot(?:\b)" 3; + "~*(?:\b)Widow(?:\b)" 3; + "~*(?:\b)WinHTTrack(?:\b)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; + "~*(?:\b)WISENutbot(?:\b)" 3; + "~*(?:\b)Wonderbot(?:\b)" 3; + "~*(?:\b)Woobot(?:\b)" 3; + "~*(?:\b)Wotbox(?:\b)" 3; + "~*(?:\b)Wprecon(?:\b)" 3; + "~*(?:\b)WPScan(?:\b)" 3; + "~*(?:\b)WWW-Collector-E(?:\b)" 3; + "~*(?:\b)WWW-Mechanize(?:\b)" 3; + "~*(?:\b)WWW::Mechanize(?:\b)" 3; + "~*(?:\b)WWWOFFLE(?:\b)" 3; + "~*(?:\b)x09Mozilla(?:\b)" 3; + "~*(?:\b)x22Mozilla(?:\b)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; + "~*(?:\b)Xenu(?:\b)" 3; + "~*(?:\b)xpymep1.exe(?:\b)" 3; + "~*(?:\b)YoudaoBot(?:\b)" 3; + "~*(?:\b)Zade(?:\b)" 3; + "~*(?:\b)Zauba(?:\b)" 3; + "~*(?:\b)zauba.io(?:\b)" 3; + "~*(?:\b)Zermelo(?:\b)" 3; + "~*(?:\b)Zeus(?:\b)" 3; + "~*(?:\b)zgrab(?:\b)" 3; + "~*(?:\b)Zitebot(?:\b)" 3; + "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZumBot(?:\b)" 3; + "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------- @@ -717,38 +717,38 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)adidxbot(?:\b|)" 0; - "~*(?:\b)AdsBot-Google(?:\b|)" 0; - "~*(?:\b)aolbuild(?:\b|)" 0; - "~*(?:\b)bingbot(?:\b|)" 0; - "~*(?:\b)bingpreview(?:\b|)" 0; - "~*(?:\b)DoCoMo(?:\b|)" 0; - "~*(?:\b)duckduckgo(?:\b|)" 0; - "~*(?:\b)facebookexternalhit(?:\b|)" 0; - "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; - "~*(?:\b)Googlebot(?:\b|)" 0; - "~*(?:\b)Googlebot-Image(?:\b|)" 0; - "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; - "~*(?:\b)Googlebot-News(?:\b|)" 0; - "~*(?:\b)Googlebot/Test(?:\b|)" 0; - "~*(?:\b)Googlebot-Video(?:\b|)" 0; - "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; - "~*(?:\b)Gravityscan(?:\b|)" 0; - "~*(?:\b)gsa-crawler(?:\b|)" 0; - "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; - "~*(?:\b)Kraken/0.1(?:\b|)" 0; - "~*(?:\b)LinkedInBot(?:\b|)" 0; - "~*(?:\b)Mediapartners-Google(?:\b|)" 0; - "~*(?:\b)msnbot(?:\b|)" 0; - "~*(?:\b)msnbot-media(?:\b|)" 0; - "~*(?:\b)SAMSUNG(?:\b|)" 0; - "~*(?:\b)Slackbot(?:\b|)" 0; - "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; - "~*(?:\b)slurp(?:\b|)" 0; - "~*(?:\b)teoma(?:\b|)" 0; - "~*(?:\b)TwitterBot(?:\b|)" 0; - "~*(?:\b)Wordpress(?:\b|)" 0; - "~*(?:\b)yahoo(?:\b|)" 0; + "~*(?:\b)adidxbot(?:\b)" 0; + "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bingbot(?:\b)" 0; + "~*(?:\b)bingpreview(?:\b)" 0; + "~*(?:\b)DoCoMo(?:\b)" 0; + "~*(?:\b)duckduckgo(?:\b)" 0; + "~*(?:\b)facebookexternalhit(?:\b)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b)" 0; + "~*(?:\b)Googlebot(?:\b)" 0; + "~*(?:\b)Googlebot-Image(?:\b)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b)" 0; + "~*(?:\b)Googlebot-News(?:\b)" 0; + "~*(?:\b)Googlebot/Test(?:\b)" 0; + "~*(?:\b)Googlebot-Video(?:\b)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; + "~*(?:\b)Gravityscan(?:\b)" 0; + "~*(?:\b)gsa-crawler(?:\b)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b)" 0; + "~*(?:\b)Kraken/0.1(?:\b)" 0; + "~*(?:\b)LinkedInBot(?:\b)" 0; + "~*(?:\b)Mediapartners-Google(?:\b)" 0; + "~*(?:\b)msnbot(?:\b)" 0; + "~*(?:\b)msnbot-media(?:\b)" 0; + "~*(?:\b)SAMSUNG(?:\b)" 0; + "~*(?:\b)Slackbot(?:\b)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 0; + "~*(?:\b)slurp(?:\b)" 0; + "~*(?:\b)teoma(?:\b)" 0; + "~*(?:\b)TwitterBot(?:\b)" 0; + "~*(?:\b)Wordpress(?:\b)" 0; + "~*(?:\b)yahoo(?:\b)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------------------- @@ -760,12 +760,12 @@ map $http_user_agent $bad_bot { #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)jetmon(?:\b|)" 1; - "~*(?:\b)libwww-perl(?:\b|)" 1; - "~*(?:\b)Lynx(?:\b|)" 1; - "~*(?:\b)munin(?:\b|)" 1; - "~*(?:\b)Presto(?:\b|)" 1; - "~*(?:\b)Wget/1.15(?:\b|)" 1; + "~*(?:\b)jetmon(?:\b)" 1; + "~*(?:\b)libwww-perl(?:\b)" 1; + "~*(?:\b)Lynx(?:\b)" 1; + "~*(?:\b)munin(?:\b)" 1; + "~*(?:\b)Presto(?:\b)" 1; + "~*(?:\b)Wget/1.15(?:\b)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ------------------------------------------------------- @@ -778,23 +778,23 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)Alexa(?:\b|)" 2; - "~*(?:\b)archive.org(?:\b|)" 2; - "~*(?:\b)Baidu(?:\b|)" 2; - "~*(?:\b)BUbiNG(?:\b|)" 2; - "~*(?:\b)FlipboardProxy(?:\b|)" 2; - "~*(?:\b)ia_archiver(?:\b|)" 2; - "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; - "~*(?:\b)Proximic(?:\b|)" 2; - "~*(?:\b)R6_CommentReader(?:\b|)" 2; - "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; - "~*(?:\b)RED/1(?:\b|)" 2; - "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; - "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; - "~*(?:\b)Spaidu(?:\b|)" 2; - "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; - "~*(?:\b)YandexBot(?:\b|)" 2; - "~*(?:\b)YandexImages(?:\b|)" 2; + "~*(?:\b)Alexa(?:\b)" 2; + "~*(?:\b)archive.org(?:\b)" 2; + "~*(?:\b)Baidu(?:\b)" 2; + "~*(?:\b)BUbiNG(?:\b)" 2; + "~*(?:\b)FlipboardProxy(?:\b)" 2; + "~*(?:\b)ia_archiver(?:\b)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b)" 2; + "~*(?:\b)Proximic(?:\b)" 2; + "~*(?:\b)R6_CommentReader(?:\b)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b)" 2; + "~*(?:\b)RED/1(?:\b)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b)" 2; + "~*(?:\b)Spaidu(?:\b)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b)" 2; + "~*(?:\b)YandexBot(?:\b)" 2; + "~*(?:\b)YandexImages(?:\b)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index b9d24324a..0f27f0dd9 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 1; +104.154.113.151 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index ea117da9f..0e0848852 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 0; +104.154.113.151 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index f26e52add..a1999739a 100644 --- a/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1709 -### Updated: Mon Jul 1 10:37:04 SAST 2019 +### Version: V4.2019.07.1712 +### Updated: Mon Jul 1 11:24:52 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### @@ -147,562 +147,562 @@ map $http_user_agent $bad_bot { # -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b|)" 3; - "~*(?:\b)404checker(?:\b|)" 3; - "~*(?:\b)404enemy(?:\b|)" 3; - "~*(?:\b)80legs(?:\b|)" 3; - "~*(?:\b)Abonti(?:\b|)" 3; - "~*(?:\b)Aboundex(?:\b|)" 3; - "~*(?:\b)Aboundexbot(?:\b|)" 3; - "~*(?:\b)Acunetix(?:\b|)" 3; - "~*(?:\b)ADmantX(?:\b|)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; - "~*(?:\b)AhrefsBot(?:\b|)" 3; - "~*(?:\b)AIBOT(?:\b|)" 3; - "~*(?:\b)AiHitBot(?:\b|)" 3; - "~*(?:\b)Aipbot(?:\b|)" 3; - "~*(?:\b)Alexibot(?:\b|)" 3; - "~*(?:\b)Alligator(?:\b|)" 3; - "~*(?:\b)AllSubmitter(?:\b|)" 3; - "~*(?:\b)AlphaBot(?:\b|)" 3; - "~*(?:\b)Anarchie(?:\b|)" 3; - "~*(?:\b)Apexoo(?:\b|)" 3; - "~*(?:\b)archive.org_bot(?:\b|)" 3; - "~*(?:\b)ASPSeek(?:\b|)" 3; - "~*(?:\b)Asterias(?:\b|)" 3; - "~*(?:\b)Attach(?:\b|)" 3; - "~*(?:\b)autoemailspider(?:\b|)" 3; - "~*(?:\b)BackDoorBot(?:\b|)" 3; - "~*(?:\b)Backlink-Ceck(?:\b|)" 3; - "~*(?:\b)backlink-check(?:\b|)" 3; - "~*(?:\b)BacklinkCrawler(?:\b|)" 3; - "~*(?:\b)BackStreet(?:\b|)" 3; - "~*(?:\b)BackWeb(?:\b|)" 3; - "~*(?:\b)Badass(?:\b|)" 3; - "~*(?:\b)Bandit(?:\b|)" 3; - "~*(?:\b)Barkrowler(?:\b|)" 3; - "~*(?:\b)BatchFTP(?:\b|)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; - "~*(?:\b)BBBike(?:\b|)" 3; - "~*(?:\b)BDCbot(?:\b|)" 3; - "~*(?:\b)BDFetch(?:\b|)" 3; - "~*(?:\b)BetaBot(?:\b|)" 3; - "~*(?:\b)Bigfoot(?:\b|)" 3; - "~*(?:\b)Bitacle(?:\b|)" 3; - "~*(?:\b)Blackboard(?:\b|)" 3; - "~*(?:\b)Black\ Hole(?:\b|)" 3; - "~*(?:\b)BlackWidow(?:\b|)" 3; - "~*(?:\b)BLEXBot(?:\b|)" 3; - "~*(?:\b)Blow(?:\b|)" 3; - "~*(?:\b)BlowFish(?:\b|)" 3; - "~*(?:\b)Boardreader(?:\b|)" 3; - "~*(?:\b)Bolt(?:\b|)" 3; - "~*(?:\b)BotALot(?:\b|)" 3; - "~*(?:\b)Brandprotect(?:\b|)" 3; - "~*(?:\b)Brandwatch(?:\b|)" 3; - "~*(?:\b)Buddy(?:\b|)" 3; - "~*(?:\b)BuiltBotTough(?:\b|)" 3; - "~*(?:\b)BuiltWith(?:\b|)" 3; - "~*(?:\b)Bullseye(?:\b|)" 3; - "~*(?:\b)BunnySlippers(?:\b|)" 3; - "~*(?:\b)BuzzSumo(?:\b|)" 3; - "~*(?:\b)Calculon(?:\b|)" 3; - "~*(?:\b)CATExplorador(?:\b|)" 3; - "~*(?:\b)CazoodleBot(?:\b|)" 3; - "~*(?:\b)CCBot(?:\b|)" 3; - "~*(?:\b)Cegbfeieh(?:\b|)" 3; - "~*(?:\b)CheeseBot(?:\b|)" 3; - "~*(?:\b)CherryPicker(?:\b|)" 3; - "~*(?:\b)CheTeam(?:\b|)" 3; - "~*(?:\b)ChinaClaw(?:\b|)" 3; - "~*(?:\b)Chlooe(?:\b|)" 3; - "~*(?:\b)Claritybot(?:\b|)" 3; - "~*(?:\b)Cliqzbot(?:\b|)" 3; - "~*(?:\b)Cloud\ mapping(?:\b|)" 3; - "~*(?:\b)coccocbot-web(?:\b|)" 3; - "~*(?:\b)Cogentbot(?:\b|)" 3; - "~*(?:\b)cognitiveseo(?:\b|)" 3; - "~*(?:\b)Collector(?:\b|)" 3; - "~*(?:\b)com.plumanalytics(?:\b|)" 3; - "~*(?:\b)Copier(?:\b|)" 3; - "~*(?:\b)CopyRightCheck(?:\b|)" 3; - "~*(?:\b)Copyscape(?:\b|)" 3; - "~*(?:\b)Cosmos(?:\b|)" 3; - "~*(?:\b)Craftbot(?:\b|)" 3; - "~*(?:\b)crawler4j(?:\b|)" 3; - "~*(?:\b)crawler.feedback(?:\b|)" 3; - "~*(?:\b)crawl.sogou.com(?:\b|)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; - "~*(?:\b)Crescent(?:\b|)" 3; - "~*(?:\b)CrunchBot(?:\b|)" 3; - "~*(?:\b)CSHttp(?:\b|)" 3; - "~*(?:\b)Curious(?:\b|)" 3; - "~*(?:\b)Custo(?:\b|)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; - "~*(?:\b)DataCha0s(?:\b|)" 3; - "~*(?:\b)DBLBot(?:\b|)" 3; - "~*(?:\b)demandbase-bot(?:\b|)" 3; - "~*(?:\b)Demon(?:\b|)" 3; - "~*(?:\b)Deusu(?:\b|)" 3; - "~*(?:\b)Devil(?:\b|)" 3; - "~*(?:\b)Digincore(?:\b|)" 3; - "~*(?:\b)DigitalPebble(?:\b|)" 3; - "~*(?:\b)DIIbot(?:\b|)" 3; - "~*(?:\b)Dirbuster(?:\b|)" 3; - "~*(?:\b)Disco(?:\b|)" 3; - "~*(?:\b)Discobot(?:\b|)" 3; - "~*(?:\b)Discoverybot(?:\b|)" 3; - "~*(?:\b)Dispatch(?:\b|)" 3; - "~*(?:\b)DittoSpyder(?:\b|)" 3; - "~*(?:\b)DnyzBot(?:\b|)" 3; - "~*(?:\b)DomainAppender(?:\b|)" 3; - "~*(?:\b)DomainCrawler(?:\b|)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; - "~*(?:\b)DomainStatsBot(?:\b|)" 3; - "~*(?:\b)Dotbot(?:\b|)" 3; - "~*(?:\b)Download\ Wonder(?:\b|)" 3; - "~*(?:\b)Dragonfly(?:\b|)" 3; - "~*(?:\b)Drip(?:\b|)" 3; - "~*(?:\b)DSearch(?:\b|)" 3; - "~*(?:\b)DTS\ Agent(?:\b|)" 3; - "~*(?:\b)EasyDL(?:\b|)" 3; - "~*(?:\b)Ebingbong(?:\b|)" 3; - "~*(?:\b)eCatch(?:\b|)" 3; - "~*(?:\b)ECCP/1.0(?:\b|)" 3; - "~*(?:\b)Ecxi(?:\b|)" 3; - "~*(?:\b)EirGrabber(?:\b|)" 3; - "~*(?:\b)EMail\ Siphon(?:\b|)" 3; - "~*(?:\b)EMail\ Wolf(?:\b|)" 3; - "~*(?:\b)EroCrawler(?:\b|)" 3; - "~*(?:\b)evc-batch(?:\b|)" 3; - "~*(?:\b)Evil(?:\b|)" 3; - "~*(?:\b)Exabot(?:\b|)" 3; - "~*(?:\b)Express\ WebPictures(?:\b|)" 3; - "~*(?:\b)ExtLinksBot(?:\b|)" 3; - "~*(?:\b)Extractor(?:\b|)" 3; - "~*(?:\b)ExtractorPro(?:\b|)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; - "~*(?:\b)EyeNetIE(?:\b|)" 3; - "~*(?:\b)Ezooms(?:\b|)" 3; - "~*(?:\b)facebookscraper(?:\b|)" 3; - "~*(?:\b)FDM(?:\b|)" 3; - "~*(?:\b)FemtosearchBot(?:\b|)" 3; - "~*(?:\b)FHscan(?:\b|)" 3; - "~*(?:\b)Fimap(?:\b|)" 3; - "~*(?:\b)Firefox/7.0(?:\b|)" 3; - "~*(?:\b)FlashGet(?:\b|)" 3; - "~*(?:\b)Flunky(?:\b|)" 3; - "~*(?:\b)Foobot(?:\b|)" 3; - "~*(?:\b)Freeuploader(?:\b|)" 3; - "~*(?:\b)FrontPage(?:\b|)" 3; - "~*(?:\b)FyberSpider(?:\b|)" 3; - "~*(?:\b)Fyrebot(?:\b|)" 3; - "~*(?:\b)GalaxyBot(?:\b|)" 3; - "~*(?:\b)Genieo(?:\b|)" 3; - "~*(?:\b)GermCrawler(?:\b|)" 3; - "~*(?:\b)Getintent(?:\b|)" 3; - "~*(?:\b)GetRight(?:\b|)" 3; - "~*(?:\b)GetWeb(?:\b|)" 3; - "~*(?:\b)Gigablast(?:\b|)" 3; - "~*(?:\b)Gigabot(?:\b|)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; - "~*(?:\b)Gotit(?:\b|)" 3; - "~*(?:\b)GoZilla(?:\b|)" 3; - "~*(?:\b)Go!Zilla(?:\b|)" 3; - "~*(?:\b)Grabber(?:\b|)" 3; - "~*(?:\b)GrabNet(?:\b|)" 3; - "~*(?:\b)Grafula(?:\b|)" 3; - "~*(?:\b)GrapeFX(?:\b|)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; - "~*(?:\b)GridBot(?:\b|)" 3; - "~*(?:\b)GT::WWW(?:\b|)" 3; - "~*(?:\b)Haansoft(?:\b|)" 3; - "~*(?:\b)HaosouSpider(?:\b|)" 3; - "~*(?:\b)Harvest(?:\b|)" 3; - "~*(?:\b)Havij(?:\b|)" 3; - "~*(?:\b)HEADMasterSEO(?:\b|)" 3; - "~*(?:\b)Heritrix(?:\b|)" 3; - "~*(?:\b)Hloader(?:\b|)" 3; - "~*(?:\b)HMView(?:\b|)" 3; - "~*(?:\b)HTMLparser(?:\b|)" 3; - "~*(?:\b)HTTP::Lite(?:\b|)" 3; - "~*(?:\b)HTTrack(?:\b|)" 3; - "~*(?:\b)Humanlinks(?:\b|)" 3; - "~*(?:\b)HybridBot(?:\b|)" 3; - "~*(?:\b)Iblog(?:\b|)" 3; - "~*(?:\b)IDBot(?:\b|)" 3; - "~*(?:\b)Id-search(?:\b|)" 3; - "~*(?:\b)IlseBot(?:\b|)" 3; - "~*(?:\b)Image\ Fetch(?:\b|)" 3; - "~*(?:\b)Image\ Sucker(?:\b|)" 3; - "~*(?:\b)IndeedBot(?:\b|)" 3; - "~*(?:\b)Indy\ Library(?:\b|)" 3; - "~*(?:\b)InfoNaviRobot(?:\b|)" 3; - "~*(?:\b)InfoTekies(?:\b|)" 3; - "~*(?:\b)instabid(?:\b|)" 3; - "~*(?:\b)Intelliseek(?:\b|)" 3; - "~*(?:\b)InterGET(?:\b|)" 3; - "~*(?:\b)Internet\ Ninja(?:\b|)" 3; - "~*(?:\b)InternetSeer(?:\b|)" 3; - "~*(?:\b)internetVista\ monitor(?:\b|)" 3; - "~*(?:\b)ips-agent(?:\b|)" 3; - "~*(?:\b)Iria(?:\b|)" 3; - "~*(?:\b)IRLbot(?:\b|)" 3; - "~*(?:\b)Iskanie(?:\b|)" 3; - "~*(?:\b)IstellaBot(?:\b|)" 3; - "~*(?:\b)JamesBOT(?:\b|)" 3; - "~*(?:\b)Jbrofuzz(?:\b|)" 3; - "~*(?:\b)JennyBot(?:\b|)" 3; - "~*(?:\b)JetCar(?:\b|)" 3; - "~*(?:\b)Jetty(?:\b|)" 3; - "~*(?:\b)JikeSpider(?:\b|)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; - "~*(?:\b)Joomla(?:\b|)" 3; - "~*(?:\b)Jorgee(?:\b|)" 3; - "~*(?:\b)JustView(?:\b|)" 3; - "~*(?:\b)Jyxobot(?:\b|)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; - "~*(?:\b)Keyword\ Density(?:\b|)" 3; - "~*(?:\b)Kozmosbot(?:\b|)" 3; - "~*(?:\b)Lanshanbot(?:\b|)" 3; - "~*(?:\b)Larbin(?:\b|)" 3; - "~*(?:\b)LeechFTP(?:\b|)" 3; - "~*(?:\b)LeechGet(?:\b|)" 3; - "~*(?:\b)LexiBot(?:\b|)" 3; - "~*(?:\b)Lftp(?:\b|)" 3; - "~*(?:\b)LibWeb(?:\b|)" 3; - "~*(?:\b)Libwhisker(?:\b|)" 3; - "~*(?:\b)Lightspeedsystems(?:\b|)" 3; - "~*(?:\b)Likse(?:\b|)" 3; - "~*(?:\b)Linkdexbot(?:\b|)" 3; - "~*(?:\b)LinkextractorPro(?:\b|)" 3; - "~*(?:\b)LinkpadBot(?:\b|)" 3; - "~*(?:\b)LinkScan(?:\b|)" 3; - "~*(?:\b)LinksManager(?:\b|)" 3; - "~*(?:\b)LinkWalker(?:\b|)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; - "~*(?:\b)Lipperhey(?:\b|)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; - "~*(?:\b)Litemage_walker(?:\b|)" 3; - "~*(?:\b)Lmspider(?:\b|)" 3; - "~*(?:\b)LNSpiderguy(?:\b|)" 3; - "~*(?:\b)Ltx71(?:\b|)" 3; - "~*(?:\b)lwp-request(?:\b|)" 3; - "~*(?:\b)LWP::Simple(?:\b|)" 3; - "~*(?:\b)lwp-trivial(?:\b|)" 3; - "~*(?:\b)Magnet(?:\b|)" 3; - "~*(?:\b)Mag-Net(?:\b|)" 3; - "~*(?:\b)magpie-crawler(?:\b|)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; - "~*(?:\b)Majestic12(?:\b|)" 3; - "~*(?:\b)Majestic-SEO(?:\b|)" 3; - "~*(?:\b)Majestic\ SEO(?:\b|)" 3; - "~*(?:\b)MarkMonitor(?:\b|)" 3; - "~*(?:\b)MarkWatch(?:\b|)" 3; - "~*(?:\b)Masscan(?:\b|)" 3; - "~*(?:\b)Mass\ Downloader(?:\b|)" 3; - "~*(?:\b)Mata\ Hari(?:\b|)" 3; - "~*(?:\b)MauiBot(?:\b|)" 3; - "~*(?:\b)meanpathbot(?:\b|)" 3; - "~*(?:\b)Meanpathbot(?:\b|)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; - "~*(?:\b)mediawords(?:\b|)" 3; - "~*(?:\b)MegaIndex.ru(?:\b|)" 3; - "~*(?:\b)Metauri(?:\b|)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; - "~*(?:\b)MIDown\ tool(?:\b|)" 3; - "~*(?:\b)MIIxpc(?:\b|)" 3; - "~*(?:\b)Mister\ PiX(?:\b|)" 3; - "~*(?:\b)MJ12bot(?:\b|)" 3; - "~*(?:\b)Mojeek(?:\b|)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; - "~*(?:\b)Mr.4x3(?:\b|)" 3; - "~*(?:\b)MSFrontPage(?:\b|)" 3; - "~*(?:\b)MSIECrawler(?:\b|)" 3; - "~*(?:\b)Msrabot(?:\b|)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; - "~*(?:\b)muhstik-scan(?:\b|)" 3; - "~*(?:\b)Musobot(?:\b|)" 3; - "~*(?:\b)Name\ Intelligence(?:\b|)" 3; - "~*(?:\b)Nameprotect(?:\b|)" 3; - "~*(?:\b)Navroad(?:\b|)" 3; - "~*(?:\b)NearSite(?:\b|)" 3; - "~*(?:\b)Needle(?:\b|)" 3; - "~*(?:\b)Nessus(?:\b|)" 3; - "~*(?:\b)NetAnts(?:\b|)" 3; - "~*(?:\b)Netcraft(?:\b|)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; - "~*(?:\b)NetLyzer(?:\b|)" 3; - "~*(?:\b)NetMechanic(?:\b|)" 3; - "~*(?:\b)NetSpider(?:\b|)" 3; - "~*(?:\b)Nettrack(?:\b|)" 3; - "~*(?:\b)Net\ Vampire(?:\b|)" 3; - "~*(?:\b)Netvibes(?:\b|)" 3; - "~*(?:\b)NetZIP(?:\b|)" 3; - "~*(?:\b)NextGenSearchBot(?:\b|)" 3; - "~*(?:\b)Nibbler(?:\b|)" 3; - "~*(?:\b)NICErsPRO(?:\b|)" 3; - "~*(?:\b)Niki-bot(?:\b|)" 3; - "~*(?:\b)Nikto(?:\b|)" 3; - "~*(?:\b)NimbleCrawler(?:\b|)" 3; - "~*(?:\b)Nimbostratus(?:\b|)" 3; - "~*(?:\b)Ninja(?:\b|)" 3; - "~*(?:\b)Nmap(?:\b|)" 3; - "~*(?:\b)NPbot(?:\b|)" 3; - "~*(?:\b)Nutch(?:\b|)" 3; - "~*(?:\b)oBot(?:\b|)" 3; - "~*(?:\b)Octopus(?:\b|)" 3; - "~*(?:\b)Offline\ Explorer(?:\b|)" 3; - "~*(?:\b)Offline\ Navigator(?:\b|)" 3; - "~*(?:\b)OnCrawl(?:\b|)" 3; - "~*(?:\b)Openfind(?:\b|)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; - "~*(?:\b)Openvas(?:\b|)" 3; - "~*(?:\b)OpenVAS(?:\b|)" 3; - "~*(?:\b)OrangeBot(?:\b|)" 3; - "~*(?:\b)OrangeSpider(?:\b|)" 3; - "~*(?:\b)OutclicksBot(?:\b|)" 3; - "~*(?:\b)OutfoxBot(?:\b|)" 3; - "~*(?:\b)PageAnalyzer(?:\b|)" 3; - "~*(?:\b)Page\ Analyzer(?:\b|)" 3; - "~*(?:\b)PageGrabber(?:\b|)" 3; - "~*(?:\b)page\ scorer(?:\b|)" 3; - "~*(?:\b)PageScorer(?:\b|)" 3; - "~*(?:\b)Pandalytics(?:\b|)" 3; - "~*(?:\b)Panscient(?:\b|)" 3; - "~*(?:\b)Papa\ Foto(?:\b|)" 3; - "~*(?:\b)Pavuk(?:\b|)" 3; - "~*(?:\b)pcBrowser(?:\b|)" 3; - "~*(?:\b)PECL::HTTP(?:\b|)" 3; - "~*(?:\b)PeoplePal(?:\b|)" 3; - "~*(?:\b)PHPCrawl(?:\b|)" 3; - "~*(?:\b)Picscout(?:\b|)" 3; - "~*(?:\b)Picsearch(?:\b|)" 3; - "~*(?:\b)PictureFinder(?:\b|)" 3; - "~*(?:\b)Pimonster(?:\b|)" 3; - "~*(?:\b)Pi-Monster(?:\b|)" 3; - "~*(?:\b)Pixray(?:\b|)" 3; - "~*(?:\b)PleaseCrawl(?:\b|)" 3; - "~*(?:\b)plumanalytics(?:\b|)" 3; - "~*(?:\b)Pockey(?:\b|)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; - "~*(?:\b)Probethenet(?:\b|)" 3; - "~*(?:\b)ProPowerBot(?:\b|)" 3; - "~*(?:\b)ProWebWalker(?:\b|)" 3; - "~*(?:\b)Psbot(?:\b|)" 3; - "~*(?:\b)Pump(?:\b|)" 3; - "~*(?:\b)PxBroker(?:\b|)" 3; - "~*(?:\b)PyCurl(?:\b|)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; - "~*(?:\b)Quick-Crawler(?:\b|)" 3; - "~*(?:\b)RankActive(?:\b|)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; - "~*(?:\b)RankFlex(?:\b|)" 3; - "~*(?:\b)RankingBot(?:\b|)" 3; - "~*(?:\b)RankingBot2(?:\b|)" 3; - "~*(?:\b)Rankivabot(?:\b|)" 3; - "~*(?:\b)RankurBot(?:\b|)" 3; - "~*(?:\b)RealDownload(?:\b|)" 3; - "~*(?:\b)Reaper(?:\b|)" 3; - "~*(?:\b)RebelMouse(?:\b|)" 3; - "~*(?:\b)Recorder(?:\b|)" 3; - "~*(?:\b)RedesScrapy(?:\b|)" 3; - "~*(?:\b)ReGet(?:\b|)" 3; - "~*(?:\b)RepoMonkey(?:\b|)" 3; - "~*(?:\b)Ripper(?:\b|)" 3; - "~*(?:\b)RocketCrawler(?:\b|)" 3; - "~*(?:\b)Rogerbot(?:\b|)" 3; - "~*(?:\b)s1z.ru(?:\b|)" 3; - "~*(?:\b)SalesIntelligent(?:\b|)" 3; - "~*(?:\b)SBIder(?:\b|)" 3; - "~*(?:\b)ScanAlert(?:\b|)" 3; - "~*(?:\b)Scanbot(?:\b|)" 3; - "~*(?:\b)scan.lol(?:\b|)" 3; - "~*(?:\b)ScoutJet(?:\b|)" 3; - "~*(?:\b)Scrapy(?:\b|)" 3; - "~*(?:\b)Screaming(?:\b|)" 3; - "~*(?:\b)ScreenerBot(?:\b|)" 3; - "~*(?:\b)Searchestate(?:\b|)" 3; - "~*(?:\b)SearchmetricsBot(?:\b|)" 3; - "~*(?:\b)Semrush(?:\b|)" 3; - "~*(?:\b)SemrushBot(?:\b|)" 3; - "~*(?:\b)SEOkicks(?:\b|)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; - "~*(?:\b)Seomoz(?:\b|)" 3; - "~*(?:\b)SEOprofiler(?:\b|)" 3; - "~*(?:\b)seoscanners(?:\b|)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; - "~*(?:\b)SEOstats(?:\b|)" 3; - "~*(?:\b)serpstatbot(?:\b|)" 3; - "~*(?:\b)sexsearcher(?:\b|)" 3; - "~*(?:\b)Shodan(?:\b|)" 3; - "~*(?:\b)Siphon(?:\b|)" 3; - "~*(?:\b)SISTRIX(?:\b|)" 3; - "~*(?:\b)Sitebeam(?:\b|)" 3; - "~*(?:\b)SiteExplorer(?:\b|)" 3; - "~*(?:\b)Siteimprove(?:\b|)" 3; - "~*(?:\b)SiteLockSpider(?:\b|)" 3; - "~*(?:\b)SiteSnagger(?:\b|)" 3; - "~*(?:\b)SiteSucker(?:\b|)" 3; - "~*(?:\b)Site\ Sucker(?:\b|)" 3; - "~*(?:\b)Sitevigil(?:\b|)" 3; - "~*(?:\b)SlySearch(?:\b|)" 3; - "~*(?:\b)SmartDownload(?:\b|)" 3; - "~*(?:\b)SMTBot(?:\b|)" 3; - "~*(?:\b)Snake(?:\b|)" 3; - "~*(?:\b)Snapbot(?:\b|)" 3; - "~*(?:\b)Snoopy(?:\b|)" 3; - "~*(?:\b)SocialRankIOBot(?:\b|)" 3; - "~*(?:\b)Sociscraper(?:\b|)" 3; - "~*(?:\b)sogouspider(?:\b|)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; - "~*(?:\b)Sosospider(?:\b|)" 3; - "~*(?:\b)Sottopop(?:\b|)" 3; - "~*(?:\b)SpaceBison(?:\b|)" 3; - "~*(?:\b)Spammen(?:\b|)" 3; - "~*(?:\b)SpankBot(?:\b|)" 3; - "~*(?:\b)Spanner(?:\b|)" 3; - "~*(?:\b)sp_auditbot(?:\b|)" 3; - "~*(?:\b)Spbot(?:\b|)" 3; - "~*(?:\b)Spinn3r(?:\b|)" 3; - "~*(?:\b)SputnikBot(?:\b|)" 3; - "~*(?:\b)spyfu(?:\b|)" 3; - "~*(?:\b)Sqlmap(?:\b|)" 3; - "~*(?:\b)Sqlworm(?:\b|)" 3; - "~*(?:\b)Sqworm(?:\b|)" 3; - "~*(?:\b)Steeler(?:\b|)" 3; - "~*(?:\b)Stripper(?:\b|)" 3; - "~*(?:\b)Sucker(?:\b|)" 3; - "~*(?:\b)Sucuri(?:\b|)" 3; - "~*(?:\b)SuperBot(?:\b|)" 3; - "~*(?:\b)SuperHTTP(?:\b|)" 3; - "~*(?:\b)Surfbot(?:\b|)" 3; - "~*(?:\b)SurveyBot(?:\b|)" 3; - "~*(?:\b)Suzuran(?:\b|)" 3; - "~*(?:\b)Swiftbot(?:\b|)" 3; - "~*(?:\b)sysscan(?:\b|)" 3; - "~*(?:\b)Szukacz(?:\b|)" 3; - "~*(?:\b)T0PHackTeam(?:\b|)" 3; - "~*(?:\b)T8Abot(?:\b|)" 3; - "~*(?:\b)tAkeOut(?:\b|)" 3; - "~*(?:\b)Teleport(?:\b|)" 3; - "~*(?:\b)TeleportPro(?:\b|)" 3; - "~*(?:\b)Telesoft(?:\b|)" 3; - "~*(?:\b)Telesphoreo(?:\b|)" 3; - "~*(?:\b)Telesphorep(?:\b|)" 3; - "~*(?:\b)The\ Intraformant(?:\b|)" 3; - "~*(?:\b)TheNomad(?:\b|)" 3; - "~*(?:\b)Thumbor(?:\b|)" 3; - "~*(?:\b)TightTwatBot(?:\b|)" 3; - "~*(?:\b)Titan(?:\b|)" 3; - "~*(?:\b)Toata(?:\b|)" 3; - "~*(?:\b)Toweyabot(?:\b|)" 3; - "~*(?:\b)Tracemyfile(?:\b|)" 3; - "~*(?:\b)Trendiction(?:\b|)" 3; - "~*(?:\b)Trendictionbot(?:\b|)" 3; - "~*(?:\b)trendiction.com(?:\b|)" 3; - "~*(?:\b)trendiction.de(?:\b|)" 3; - "~*(?:\b)True_Robot(?:\b|)" 3; - "~*(?:\b)Turingos(?:\b|)" 3; - "~*(?:\b)Turnitin(?:\b|)" 3; - "~*(?:\b)TurnitinBot(?:\b|)" 3; - "~*(?:\b)TwengaBot(?:\b|)" 3; - "~*(?:\b)Twice(?:\b|)" 3; - "~*(?:\b)Typhoeus(?:\b|)" 3; - "~*(?:\b)UnisterBot(?:\b|)" 3; - "~*(?:\b)Upflow(?:\b|)" 3; - "~*(?:\b)URLy.Warning(?:\b|)" 3; - "~*(?:\b)URLy\ Warning(?:\b|)" 3; - "~*(?:\b)Vacuum(?:\b|)" 3; - "~*(?:\b)Vagabondo(?:\b|)" 3; - "~*(?:\b)VB\ Project(?:\b|)" 3; - "~*(?:\b)VCI(?:\b|)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; - "~*(?:\b)VidibleScraper(?:\b|)" 3; - "~*(?:\b)Virusdie(?:\b|)" 3; - "~*(?:\b)VoidEYE(?:\b|)" 3; - "~*(?:\b)Voil(?:\b|)" 3; - "~*(?:\b)Voltron(?:\b|)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; - "~*(?:\b)WallpapersHD(?:\b|)" 3; - "~*(?:\b)WASALive-Bot(?:\b|)" 3; - "~*(?:\b)WBSearchBot(?:\b|)" 3; - "~*(?:\b)Webalta(?:\b|)" 3; - "~*(?:\b)WebAuto(?:\b|)" 3; - "~*(?:\b)Web\ Auto(?:\b|)" 3; - "~*(?:\b)WebBandit(?:\b|)" 3; - "~*(?:\b)WebCollage(?:\b|)" 3; - "~*(?:\b)Web\ Collage(?:\b|)" 3; - "~*(?:\b)WebCopier(?:\b|)" 3; - "~*(?:\b)WEBDAV(?:\b|)" 3; - "~*(?:\b)WebEnhancer(?:\b|)" 3; - "~*(?:\b)Web\ Enhancer(?:\b|)" 3; - "~*(?:\b)WebFetch(?:\b|)" 3; - "~*(?:\b)Web\ Fetch(?:\b|)" 3; - "~*(?:\b)WebFuck(?:\b|)" 3; - "~*(?:\b)Web\ Fuck(?:\b|)" 3; - "~*(?:\b)WebGo\ IS(?:\b|)" 3; - "~*(?:\b)WebImageCollector(?:\b|)" 3; - "~*(?:\b)WebLeacher(?:\b|)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; - "~*(?:\b)webmeup-crawler(?:\b|)" 3; - "~*(?:\b)WebPix(?:\b|)" 3; - "~*(?:\b)Web\ Pix(?:\b|)" 3; - "~*(?:\b)WebReaper(?:\b|)" 3; - "~*(?:\b)WebSauger(?:\b|)" 3; - "~*(?:\b)Web\ Sauger(?:\b|)" 3; - "~*(?:\b)Webshag(?:\b|)" 3; - "~*(?:\b)WebsiteExtractor(?:\b|)" 3; - "~*(?:\b)WebsiteQuester(?:\b|)" 3; - "~*(?:\b)Website\ Quester(?:\b|)" 3; - "~*(?:\b)Webster(?:\b|)" 3; - "~*(?:\b)WebStripper(?:\b|)" 3; - "~*(?:\b)WebSucker(?:\b|)" 3; - "~*(?:\b)Web\ Sucker(?:\b|)" 3; - "~*(?:\b)WebWhacker(?:\b|)" 3; - "~*(?:\b)WebZIP(?:\b|)" 3; - "~*(?:\b)WeSEE(?:\b|)" 3; - "~*(?:\b)Whack(?:\b|)" 3; - "~*(?:\b)Whacker(?:\b|)" 3; - "~*(?:\b)Whatweb(?:\b|)" 3; - "~*(?:\b)Who.is\ Bot(?:\b|)" 3; - "~*(?:\b)Widow(?:\b|)" 3; - "~*(?:\b)WinHTTrack(?:\b|)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; - "~*(?:\b)WISENutbot(?:\b|)" 3; - "~*(?:\b)Wonderbot(?:\b|)" 3; - "~*(?:\b)Woobot(?:\b|)" 3; - "~*(?:\b)Wotbox(?:\b|)" 3; - "~*(?:\b)Wprecon(?:\b|)" 3; - "~*(?:\b)WPScan(?:\b|)" 3; - "~*(?:\b)WWW-Collector-E(?:\b|)" 3; - "~*(?:\b)WWW-Mechanize(?:\b|)" 3; - "~*(?:\b)WWW::Mechanize(?:\b|)" 3; - "~*(?:\b)WWWOFFLE(?:\b|)" 3; - "~*(?:\b)x09Mozilla(?:\b|)" 3; - "~*(?:\b)x22Mozilla(?:\b|)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; - "~*(?:\b)Xenu(?:\b|)" 3; - "~*(?:\b)xpymep1.exe(?:\b|)" 3; - "~*(?:\b)YoudaoBot(?:\b|)" 3; - "~*(?:\b)Zade(?:\b|)" 3; - "~*(?:\b)Zauba(?:\b|)" 3; - "~*(?:\b)zauba.io(?:\b|)" 3; - "~*(?:\b)Zermelo(?:\b|)" 3; - "~*(?:\b)Zeus(?:\b|)" 3; - "~*(?:\b)zgrab(?:\b|)" 3; - "~*(?:\b)Zitebot(?:\b|)" 3; - "~*(?:\b)ZmEu(?:\b|)" 3; - "~*(?:\b)ZumBot(?:\b|)" 3; - "~*(?:\b)ZyBorg(?:\b|)" 3; + "~*(?:\b)360Spider(?:\b)" 3; + "~*(?:\b)404checker(?:\b)" 3; + "~*(?:\b)404enemy(?:\b)" 3; + "~*(?:\b)80legs(?:\b)" 3; + "~*(?:\b)Abonti(?:\b)" 3; + "~*(?:\b)Aboundex(?:\b)" 3; + "~*(?:\b)Aboundexbot(?:\b)" 3; + "~*(?:\b)Acunetix(?:\b)" 3; + "~*(?:\b)ADmantX(?:\b)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; + "~*(?:\b)AhrefsBot(?:\b)" 3; + "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)AiHitBot(?:\b)" 3; + "~*(?:\b)Aipbot(?:\b)" 3; + "~*(?:\b)Alexibot(?:\b)" 3; + "~*(?:\b)Alligator(?:\b)" 3; + "~*(?:\b)AllSubmitter(?:\b)" 3; + "~*(?:\b)AlphaBot(?:\b)" 3; + "~*(?:\b)Anarchie(?:\b)" 3; + "~*(?:\b)Apexoo(?:\b)" 3; + "~*(?:\b)archive.org_bot(?:\b)" 3; + "~*(?:\b)ASPSeek(?:\b)" 3; + "~*(?:\b)Asterias(?:\b)" 3; + "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)BackDoorBot(?:\b)" 3; + "~*(?:\b)Backlink-Ceck(?:\b)" 3; + "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)BacklinkCrawler(?:\b)" 3; + "~*(?:\b)BackStreet(?:\b)" 3; + "~*(?:\b)BackWeb(?:\b)" 3; + "~*(?:\b)Badass(?:\b)" 3; + "~*(?:\b)Bandit(?:\b)" 3; + "~*(?:\b)Barkrowler(?:\b)" 3; + "~*(?:\b)BatchFTP(?:\b)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; + "~*(?:\b)BBBike(?:\b)" 3; + "~*(?:\b)BDCbot(?:\b)" 3; + "~*(?:\b)BDFetch(?:\b)" 3; + "~*(?:\b)BetaBot(?:\b)" 3; + "~*(?:\b)Bigfoot(?:\b)" 3; + "~*(?:\b)Bitacle(?:\b)" 3; + "~*(?:\b)Blackboard(?:\b)" 3; + "~*(?:\b)Black\ Hole(?:\b)" 3; + "~*(?:\b)BlackWidow(?:\b)" 3; + "~*(?:\b)BLEXBot(?:\b)" 3; + "~*(?:\b)Blow(?:\b)" 3; + "~*(?:\b)BlowFish(?:\b)" 3; + "~*(?:\b)Boardreader(?:\b)" 3; + "~*(?:\b)Bolt(?:\b)" 3; + "~*(?:\b)BotALot(?:\b)" 3; + "~*(?:\b)Brandprotect(?:\b)" 3; + "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buddy(?:\b)" 3; + "~*(?:\b)BuiltBotTough(?:\b)" 3; + "~*(?:\b)BuiltWith(?:\b)" 3; + "~*(?:\b)Bullseye(?:\b)" 3; + "~*(?:\b)BunnySlippers(?:\b)" 3; + "~*(?:\b)BuzzSumo(?:\b)" 3; + "~*(?:\b)Calculon(?:\b)" 3; + "~*(?:\b)CATExplorador(?:\b)" 3; + "~*(?:\b)CazoodleBot(?:\b)" 3; + "~*(?:\b)CCBot(?:\b)" 3; + "~*(?:\b)Cegbfeieh(?:\b)" 3; + "~*(?:\b)CheeseBot(?:\b)" 3; + "~*(?:\b)CherryPicker(?:\b)" 3; + "~*(?:\b)CheTeam(?:\b)" 3; + "~*(?:\b)ChinaClaw(?:\b)" 3; + "~*(?:\b)Chlooe(?:\b)" 3; + "~*(?:\b)Claritybot(?:\b)" 3; + "~*(?:\b)Cliqzbot(?:\b)" 3; + "~*(?:\b)Cloud\ mapping(?:\b)" 3; + "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cogentbot(?:\b)" 3; + "~*(?:\b)cognitiveseo(?:\b)" 3; + "~*(?:\b)Collector(?:\b)" 3; + "~*(?:\b)com.plumanalytics(?:\b)" 3; + "~*(?:\b)Copier(?:\b)" 3; + "~*(?:\b)CopyRightCheck(?:\b)" 3; + "~*(?:\b)Copyscape(?:\b)" 3; + "~*(?:\b)Cosmos(?:\b)" 3; + "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)crawler4j(?:\b)" 3; + "~*(?:\b)crawler.feedback(?:\b)" 3; + "~*(?:\b)crawl.sogou.com(?:\b)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b)" 3; + "~*(?:\b)Crescent(?:\b)" 3; + "~*(?:\b)CrunchBot(?:\b)" 3; + "~*(?:\b)CSHttp(?:\b)" 3; + "~*(?:\b)Curious(?:\b)" 3; + "~*(?:\b)Custo(?:\b)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; + "~*(?:\b)DataCha0s(?:\b)" 3; + "~*(?:\b)DBLBot(?:\b)" 3; + "~*(?:\b)demandbase-bot(?:\b)" 3; + "~*(?:\b)Demon(?:\b)" 3; + "~*(?:\b)Deusu(?:\b)" 3; + "~*(?:\b)Devil(?:\b)" 3; + "~*(?:\b)Digincore(?:\b)" 3; + "~*(?:\b)DigitalPebble(?:\b)" 3; + "~*(?:\b)DIIbot(?:\b)" 3; + "~*(?:\b)Dirbuster(?:\b)" 3; + "~*(?:\b)Disco(?:\b)" 3; + "~*(?:\b)Discobot(?:\b)" 3; + "~*(?:\b)Discoverybot(?:\b)" 3; + "~*(?:\b)Dispatch(?:\b)" 3; + "~*(?:\b)DittoSpyder(?:\b)" 3; + "~*(?:\b)DnyzBot(?:\b)" 3; + "~*(?:\b)DomainAppender(?:\b)" 3; + "~*(?:\b)DomainCrawler(?:\b)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)DomainStatsBot(?:\b)" 3; + "~*(?:\b)Dotbot(?:\b)" 3; + "~*(?:\b)Download\ Wonder(?:\b)" 3; + "~*(?:\b)Dragonfly(?:\b)" 3; + "~*(?:\b)Drip(?:\b)" 3; + "~*(?:\b)DSearch(?:\b)" 3; + "~*(?:\b)DTS\ Agent(?:\b)" 3; + "~*(?:\b)EasyDL(?:\b)" 3; + "~*(?:\b)Ebingbong(?:\b)" 3; + "~*(?:\b)eCatch(?:\b)" 3; + "~*(?:\b)ECCP/1.0(?:\b)" 3; + "~*(?:\b)Ecxi(?:\b)" 3; + "~*(?:\b)EirGrabber(?:\b)" 3; + "~*(?:\b)EMail\ Siphon(?:\b)" 3; + "~*(?:\b)EMail\ Wolf(?:\b)" 3; + "~*(?:\b)EroCrawler(?:\b)" 3; + "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)Evil(?:\b)" 3; + "~*(?:\b)Exabot(?:\b)" 3; + "~*(?:\b)Express\ WebPictures(?:\b)" 3; + "~*(?:\b)ExtLinksBot(?:\b)" 3; + "~*(?:\b)Extractor(?:\b)" 3; + "~*(?:\b)ExtractorPro(?:\b)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; + "~*(?:\b)EyeNetIE(?:\b)" 3; + "~*(?:\b)Ezooms(?:\b)" 3; + "~*(?:\b)facebookscraper(?:\b)" 3; + "~*(?:\b)FDM(?:\b)" 3; + "~*(?:\b)FemtosearchBot(?:\b)" 3; + "~*(?:\b)FHscan(?:\b)" 3; + "~*(?:\b)Fimap(?:\b)" 3; + "~*(?:\b)Firefox/7.0(?:\b)" 3; + "~*(?:\b)FlashGet(?:\b)" 3; + "~*(?:\b)Flunky(?:\b)" 3; + "~*(?:\b)Foobot(?:\b)" 3; + "~*(?:\b)Freeuploader(?:\b)" 3; + "~*(?:\b)FrontPage(?:\b)" 3; + "~*(?:\b)FyberSpider(?:\b)" 3; + "~*(?:\b)Fyrebot(?:\b)" 3; + "~*(?:\b)GalaxyBot(?:\b)" 3; + "~*(?:\b)Genieo(?:\b)" 3; + "~*(?:\b)GermCrawler(?:\b)" 3; + "~*(?:\b)Getintent(?:\b)" 3; + "~*(?:\b)GetRight(?:\b)" 3; + "~*(?:\b)GetWeb(?:\b)" 3; + "~*(?:\b)Gigablast(?:\b)" 3; + "~*(?:\b)Gigabot(?:\b)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; + "~*(?:\b)Gotit(?:\b)" 3; + "~*(?:\b)GoZilla(?:\b)" 3; + "~*(?:\b)Go!Zilla(?:\b)" 3; + "~*(?:\b)Grabber(?:\b)" 3; + "~*(?:\b)GrabNet(?:\b)" 3; + "~*(?:\b)Grafula(?:\b)" 3; + "~*(?:\b)GrapeFX(?:\b)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b)" 3; + "~*(?:\b)GridBot(?:\b)" 3; + "~*(?:\b)GT::WWW(?:\b)" 3; + "~*(?:\b)Haansoft(?:\b)" 3; + "~*(?:\b)HaosouSpider(?:\b)" 3; + "~*(?:\b)Harvest(?:\b)" 3; + "~*(?:\b)Havij(?:\b)" 3; + "~*(?:\b)HEADMasterSEO(?:\b)" 3; + "~*(?:\b)Heritrix(?:\b)" 3; + "~*(?:\b)Hloader(?:\b)" 3; + "~*(?:\b)HMView(?:\b)" 3; + "~*(?:\b)HTMLparser(?:\b)" 3; + "~*(?:\b)HTTP::Lite(?:\b)" 3; + "~*(?:\b)HTTrack(?:\b)" 3; + "~*(?:\b)Humanlinks(?:\b)" 3; + "~*(?:\b)HybridBot(?:\b)" 3; + "~*(?:\b)Iblog(?:\b)" 3; + "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)Id-search(?:\b)" 3; + "~*(?:\b)IlseBot(?:\b)" 3; + "~*(?:\b)Image\ Fetch(?:\b)" 3; + "~*(?:\b)Image\ Sucker(?:\b)" 3; + "~*(?:\b)IndeedBot(?:\b)" 3; + "~*(?:\b)Indy\ Library(?:\b)" 3; + "~*(?:\b)InfoNaviRobot(?:\b)" 3; + "~*(?:\b)InfoTekies(?:\b)" 3; + "~*(?:\b)instabid(?:\b)" 3; + "~*(?:\b)Intelliseek(?:\b)" 3; + "~*(?:\b)InterGET(?:\b)" 3; + "~*(?:\b)Internet\ Ninja(?:\b)" 3; + "~*(?:\b)InternetSeer(?:\b)" 3; + "~*(?:\b)internetVista\ monitor(?:\b)" 3; + "~*(?:\b)ips-agent(?:\b)" 3; + "~*(?:\b)Iria(?:\b)" 3; + "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)Iskanie(?:\b)" 3; + "~*(?:\b)IstellaBot(?:\b)" 3; + "~*(?:\b)JamesBOT(?:\b)" 3; + "~*(?:\b)Jbrofuzz(?:\b)" 3; + "~*(?:\b)JennyBot(?:\b)" 3; + "~*(?:\b)JetCar(?:\b)" 3; + "~*(?:\b)Jetty(?:\b)" 3; + "~*(?:\b)JikeSpider(?:\b)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; + "~*(?:\b)Joomla(?:\b)" 3; + "~*(?:\b)Jorgee(?:\b)" 3; + "~*(?:\b)JustView(?:\b)" 3; + "~*(?:\b)Jyxobot(?:\b)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b)" 3; + "~*(?:\b)Keyword\ Density(?:\b)" 3; + "~*(?:\b)Kozmosbot(?:\b)" 3; + "~*(?:\b)Lanshanbot(?:\b)" 3; + "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)LeechFTP(?:\b)" 3; + "~*(?:\b)LeechGet(?:\b)" 3; + "~*(?:\b)LexiBot(?:\b)" 3; + "~*(?:\b)Lftp(?:\b)" 3; + "~*(?:\b)LibWeb(?:\b)" 3; + "~*(?:\b)Libwhisker(?:\b)" 3; + "~*(?:\b)Lightspeedsystems(?:\b)" 3; + "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkdexbot(?:\b)" 3; + "~*(?:\b)LinkextractorPro(?:\b)" 3; + "~*(?:\b)LinkpadBot(?:\b)" 3; + "~*(?:\b)LinkScan(?:\b)" 3; + "~*(?:\b)LinksManager(?:\b)" 3; + "~*(?:\b)LinkWalker(?:\b)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; + "~*(?:\b)Lipperhey(?:\b)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; + "~*(?:\b)Litemage_walker(?:\b)" 3; + "~*(?:\b)Lmspider(?:\b)" 3; + "~*(?:\b)LNSpiderguy(?:\b)" 3; + "~*(?:\b)Ltx71(?:\b)" 3; + "~*(?:\b)lwp-request(?:\b)" 3; + "~*(?:\b)LWP::Simple(?:\b)" 3; + "~*(?:\b)lwp-trivial(?:\b)" 3; + "~*(?:\b)Magnet(?:\b)" 3; + "~*(?:\b)Mag-Net(?:\b)" 3; + "~*(?:\b)magpie-crawler(?:\b)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b)" 3; + "~*(?:\b)Majestic12(?:\b)" 3; + "~*(?:\b)Majestic-SEO(?:\b)" 3; + "~*(?:\b)Majestic\ SEO(?:\b)" 3; + "~*(?:\b)MarkMonitor(?:\b)" 3; + "~*(?:\b)MarkWatch(?:\b)" 3; + "~*(?:\b)Masscan(?:\b)" 3; + "~*(?:\b)Mass\ Downloader(?:\b)" 3; + "~*(?:\b)Mata\ Hari(?:\b)" 3; + "~*(?:\b)MauiBot(?:\b)" 3; + "~*(?:\b)meanpathbot(?:\b)" 3; + "~*(?:\b)Meanpathbot(?:\b)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b)" 3; + "~*(?:\b)mediawords(?:\b)" 3; + "~*(?:\b)MegaIndex.ru(?:\b)" 3; + "~*(?:\b)Metauri(?:\b)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; + "~*(?:\b)MIDown\ tool(?:\b)" 3; + "~*(?:\b)MIIxpc(?:\b)" 3; + "~*(?:\b)Mister\ PiX(?:\b)" 3; + "~*(?:\b)MJ12bot(?:\b)" 3; + "~*(?:\b)Mojeek(?:\b)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; + "~*(?:\b)Mr.4x3(?:\b)" 3; + "~*(?:\b)MSFrontPage(?:\b)" 3; + "~*(?:\b)MSIECrawler(?:\b)" 3; + "~*(?:\b)Msrabot(?:\b)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; + "~*(?:\b)muhstik-scan(?:\b)" 3; + "~*(?:\b)Musobot(?:\b)" 3; + "~*(?:\b)Name\ Intelligence(?:\b)" 3; + "~*(?:\b)Nameprotect(?:\b)" 3; + "~*(?:\b)Navroad(?:\b)" 3; + "~*(?:\b)NearSite(?:\b)" 3; + "~*(?:\b)Needle(?:\b)" 3; + "~*(?:\b)Nessus(?:\b)" 3; + "~*(?:\b)NetAnts(?:\b)" 3; + "~*(?:\b)Netcraft(?:\b)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; + "~*(?:\b)NetLyzer(?:\b)" 3; + "~*(?:\b)NetMechanic(?:\b)" 3; + "~*(?:\b)NetSpider(?:\b)" 3; + "~*(?:\b)Nettrack(?:\b)" 3; + "~*(?:\b)Net\ Vampire(?:\b)" 3; + "~*(?:\b)Netvibes(?:\b)" 3; + "~*(?:\b)NetZIP(?:\b)" 3; + "~*(?:\b)NextGenSearchBot(?:\b)" 3; + "~*(?:\b)Nibbler(?:\b)" 3; + "~*(?:\b)NICErsPRO(?:\b)" 3; + "~*(?:\b)Niki-bot(?:\b)" 3; + "~*(?:\b)Nikto(?:\b)" 3; + "~*(?:\b)NimbleCrawler(?:\b)" 3; + "~*(?:\b)Nimbostratus(?:\b)" 3; + "~*(?:\b)Ninja(?:\b)" 3; + "~*(?:\b)Nmap(?:\b)" 3; + "~*(?:\b)NPbot(?:\b)" 3; + "~*(?:\b)Nutch(?:\b)" 3; + "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)Octopus(?:\b)" 3; + "~*(?:\b)Offline\ Explorer(?:\b)" 3; + "~*(?:\b)Offline\ Navigator(?:\b)" 3; + "~*(?:\b)OnCrawl(?:\b)" 3; + "~*(?:\b)Openfind(?:\b)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b)" 3; + "~*(?:\b)Openvas(?:\b)" 3; + "~*(?:\b)OpenVAS(?:\b)" 3; + "~*(?:\b)OrangeBot(?:\b)" 3; + "~*(?:\b)OrangeSpider(?:\b)" 3; + "~*(?:\b)OutclicksBot(?:\b)" 3; + "~*(?:\b)OutfoxBot(?:\b)" 3; + "~*(?:\b)PageAnalyzer(?:\b)" 3; + "~*(?:\b)Page\ Analyzer(?:\b)" 3; + "~*(?:\b)PageGrabber(?:\b)" 3; + "~*(?:\b)page\ scorer(?:\b)" 3; + "~*(?:\b)PageScorer(?:\b)" 3; + "~*(?:\b)Pandalytics(?:\b)" 3; + "~*(?:\b)Panscient(?:\b)" 3; + "~*(?:\b)Papa\ Foto(?:\b)" 3; + "~*(?:\b)Pavuk(?:\b)" 3; + "~*(?:\b)pcBrowser(?:\b)" 3; + "~*(?:\b)PECL::HTTP(?:\b)" 3; + "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)PHPCrawl(?:\b)" 3; + "~*(?:\b)Picscout(?:\b)" 3; + "~*(?:\b)Picsearch(?:\b)" 3; + "~*(?:\b)PictureFinder(?:\b)" 3; + "~*(?:\b)Pimonster(?:\b)" 3; + "~*(?:\b)Pi-Monster(?:\b)" 3; + "~*(?:\b)Pixray(?:\b)" 3; + "~*(?:\b)PleaseCrawl(?:\b)" 3; + "~*(?:\b)plumanalytics(?:\b)" 3; + "~*(?:\b)Pockey(?:\b)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; + "~*(?:\b)Probethenet(?:\b)" 3; + "~*(?:\b)ProPowerBot(?:\b)" 3; + "~*(?:\b)ProWebWalker(?:\b)" 3; + "~*(?:\b)Psbot(?:\b)" 3; + "~*(?:\b)Pump(?:\b)" 3; + "~*(?:\b)PxBroker(?:\b)" 3; + "~*(?:\b)PyCurl(?:\b)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; + "~*(?:\b)Quick-Crawler(?:\b)" 3; + "~*(?:\b)RankActive(?:\b)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b)" 3; + "~*(?:\b)RankFlex(?:\b)" 3; + "~*(?:\b)RankingBot(?:\b)" 3; + "~*(?:\b)RankingBot2(?:\b)" 3; + "~*(?:\b)Rankivabot(?:\b)" 3; + "~*(?:\b)RankurBot(?:\b)" 3; + "~*(?:\b)RealDownload(?:\b)" 3; + "~*(?:\b)Reaper(?:\b)" 3; + "~*(?:\b)RebelMouse(?:\b)" 3; + "~*(?:\b)Recorder(?:\b)" 3; + "~*(?:\b)RedesScrapy(?:\b)" 3; + "~*(?:\b)ReGet(?:\b)" 3; + "~*(?:\b)RepoMonkey(?:\b)" 3; + "~*(?:\b)Ripper(?:\b)" 3; + "~*(?:\b)RocketCrawler(?:\b)" 3; + "~*(?:\b)Rogerbot(?:\b)" 3; + "~*(?:\b)s1z.ru(?:\b)" 3; + "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)SBIder(?:\b)" 3; + "~*(?:\b)ScanAlert(?:\b)" 3; + "~*(?:\b)Scanbot(?:\b)" 3; + "~*(?:\b)scan.lol(?:\b)" 3; + "~*(?:\b)ScoutJet(?:\b)" 3; + "~*(?:\b)Scrapy(?:\b)" 3; + "~*(?:\b)Screaming(?:\b)" 3; + "~*(?:\b)ScreenerBot(?:\b)" 3; + "~*(?:\b)Searchestate(?:\b)" 3; + "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Semrush(?:\b)" 3; + "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SEOkicks(?:\b)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; + "~*(?:\b)Seomoz(?:\b)" 3; + "~*(?:\b)SEOprofiler(?:\b)" 3; + "~*(?:\b)seoscanners(?:\b)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b)" 3; + "~*(?:\b)SEOstats(?:\b)" 3; + "~*(?:\b)serpstatbot(?:\b)" 3; + "~*(?:\b)sexsearcher(?:\b)" 3; + "~*(?:\b)Shodan(?:\b)" 3; + "~*(?:\b)Siphon(?:\b)" 3; + "~*(?:\b)SISTRIX(?:\b)" 3; + "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteExplorer(?:\b)" 3; + "~*(?:\b)Siteimprove(?:\b)" 3; + "~*(?:\b)SiteLockSpider(?:\b)" 3; + "~*(?:\b)SiteSnagger(?:\b)" 3; + "~*(?:\b)SiteSucker(?:\b)" 3; + "~*(?:\b)Site\ Sucker(?:\b)" 3; + "~*(?:\b)Sitevigil(?:\b)" 3; + "~*(?:\b)SlySearch(?:\b)" 3; + "~*(?:\b)SmartDownload(?:\b)" 3; + "~*(?:\b)SMTBot(?:\b)" 3; + "~*(?:\b)Snake(?:\b)" 3; + "~*(?:\b)Snapbot(?:\b)" 3; + "~*(?:\b)Snoopy(?:\b)" 3; + "~*(?:\b)SocialRankIOBot(?:\b)" 3; + "~*(?:\b)Sociscraper(?:\b)" 3; + "~*(?:\b)sogouspider(?:\b)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; + "~*(?:\b)Sosospider(?:\b)" 3; + "~*(?:\b)Sottopop(?:\b)" 3; + "~*(?:\b)SpaceBison(?:\b)" 3; + "~*(?:\b)Spammen(?:\b)" 3; + "~*(?:\b)SpankBot(?:\b)" 3; + "~*(?:\b)Spanner(?:\b)" 3; + "~*(?:\b)sp_auditbot(?:\b)" 3; + "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spinn3r(?:\b)" 3; + "~*(?:\b)SputnikBot(?:\b)" 3; + "~*(?:\b)spyfu(?:\b)" 3; + "~*(?:\b)Sqlmap(?:\b)" 3; + "~*(?:\b)Sqlworm(?:\b)" 3; + "~*(?:\b)Sqworm(?:\b)" 3; + "~*(?:\b)Steeler(?:\b)" 3; + "~*(?:\b)Stripper(?:\b)" 3; + "~*(?:\b)Sucker(?:\b)" 3; + "~*(?:\b)Sucuri(?:\b)" 3; + "~*(?:\b)SuperBot(?:\b)" 3; + "~*(?:\b)SuperHTTP(?:\b)" 3; + "~*(?:\b)Surfbot(?:\b)" 3; + "~*(?:\b)SurveyBot(?:\b)" 3; + "~*(?:\b)Suzuran(?:\b)" 3; + "~*(?:\b)Swiftbot(?:\b)" 3; + "~*(?:\b)sysscan(?:\b)" 3; + "~*(?:\b)Szukacz(?:\b)" 3; + "~*(?:\b)T0PHackTeam(?:\b)" 3; + "~*(?:\b)T8Abot(?:\b)" 3; + "~*(?:\b)tAkeOut(?:\b)" 3; + "~*(?:\b)Teleport(?:\b)" 3; + "~*(?:\b)TeleportPro(?:\b)" 3; + "~*(?:\b)Telesoft(?:\b)" 3; + "~*(?:\b)Telesphoreo(?:\b)" 3; + "~*(?:\b)Telesphorep(?:\b)" 3; + "~*(?:\b)The\ Intraformant(?:\b)" 3; + "~*(?:\b)TheNomad(?:\b)" 3; + "~*(?:\b)Thumbor(?:\b)" 3; + "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)Titan(?:\b)" 3; + "~*(?:\b)Toata(?:\b)" 3; + "~*(?:\b)Toweyabot(?:\b)" 3; + "~*(?:\b)Tracemyfile(?:\b)" 3; + "~*(?:\b)Trendiction(?:\b)" 3; + "~*(?:\b)Trendictionbot(?:\b)" 3; + "~*(?:\b)trendiction.com(?:\b)" 3; + "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)True_Robot(?:\b)" 3; + "~*(?:\b)Turingos(?:\b)" 3; + "~*(?:\b)Turnitin(?:\b)" 3; + "~*(?:\b)TurnitinBot(?:\b)" 3; + "~*(?:\b)TwengaBot(?:\b)" 3; + "~*(?:\b)Twice(?:\b)" 3; + "~*(?:\b)Typhoeus(?:\b)" 3; + "~*(?:\b)UnisterBot(?:\b)" 3; + "~*(?:\b)Upflow(?:\b)" 3; + "~*(?:\b)URLy.Warning(?:\b)" 3; + "~*(?:\b)URLy\ Warning(?:\b)" 3; + "~*(?:\b)Vacuum(?:\b)" 3; + "~*(?:\b)Vagabondo(?:\b)" 3; + "~*(?:\b)VB\ Project(?:\b)" 3; + "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b)" 3; + "~*(?:\b)VidibleScraper(?:\b)" 3; + "~*(?:\b)Virusdie(?:\b)" 3; + "~*(?:\b)VoidEYE(?:\b)" 3; + "~*(?:\b)Voil(?:\b)" 3; + "~*(?:\b)Voltron(?:\b)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b)" 3; + "~*(?:\b)WallpapersHD(?:\b)" 3; + "~*(?:\b)WASALive-Bot(?:\b)" 3; + "~*(?:\b)WBSearchBot(?:\b)" 3; + "~*(?:\b)Webalta(?:\b)" 3; + "~*(?:\b)WebAuto(?:\b)" 3; + "~*(?:\b)Web\ Auto(?:\b)" 3; + "~*(?:\b)WebBandit(?:\b)" 3; + "~*(?:\b)WebCollage(?:\b)" 3; + "~*(?:\b)Web\ Collage(?:\b)" 3; + "~*(?:\b)WebCopier(?:\b)" 3; + "~*(?:\b)WEBDAV(?:\b)" 3; + "~*(?:\b)WebEnhancer(?:\b)" 3; + "~*(?:\b)Web\ Enhancer(?:\b)" 3; + "~*(?:\b)WebFetch(?:\b)" 3; + "~*(?:\b)Web\ Fetch(?:\b)" 3; + "~*(?:\b)WebFuck(?:\b)" 3; + "~*(?:\b)Web\ Fuck(?:\b)" 3; + "~*(?:\b)WebGo\ IS(?:\b)" 3; + "~*(?:\b)WebImageCollector(?:\b)" 3; + "~*(?:\b)WebLeacher(?:\b)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; + "~*(?:\b)webmeup-crawler(?:\b)" 3; + "~*(?:\b)WebPix(?:\b)" 3; + "~*(?:\b)Web\ Pix(?:\b)" 3; + "~*(?:\b)WebReaper(?:\b)" 3; + "~*(?:\b)WebSauger(?:\b)" 3; + "~*(?:\b)Web\ Sauger(?:\b)" 3; + "~*(?:\b)Webshag(?:\b)" 3; + "~*(?:\b)WebsiteExtractor(?:\b)" 3; + "~*(?:\b)WebsiteQuester(?:\b)" 3; + "~*(?:\b)Website\ Quester(?:\b)" 3; + "~*(?:\b)Webster(?:\b)" 3; + "~*(?:\b)WebStripper(?:\b)" 3; + "~*(?:\b)WebSucker(?:\b)" 3; + "~*(?:\b)Web\ Sucker(?:\b)" 3; + "~*(?:\b)WebWhacker(?:\b)" 3; + "~*(?:\b)WebZIP(?:\b)" 3; + "~*(?:\b)WeSEE(?:\b)" 3; + "~*(?:\b)Whack(?:\b)" 3; + "~*(?:\b)Whacker(?:\b)" 3; + "~*(?:\b)Whatweb(?:\b)" 3; + "~*(?:\b)Who.is\ Bot(?:\b)" 3; + "~*(?:\b)Widow(?:\b)" 3; + "~*(?:\b)WinHTTrack(?:\b)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; + "~*(?:\b)WISENutbot(?:\b)" 3; + "~*(?:\b)Wonderbot(?:\b)" 3; + "~*(?:\b)Woobot(?:\b)" 3; + "~*(?:\b)Wotbox(?:\b)" 3; + "~*(?:\b)Wprecon(?:\b)" 3; + "~*(?:\b)WPScan(?:\b)" 3; + "~*(?:\b)WWW-Collector-E(?:\b)" 3; + "~*(?:\b)WWW-Mechanize(?:\b)" 3; + "~*(?:\b)WWW::Mechanize(?:\b)" 3; + "~*(?:\b)WWWOFFLE(?:\b)" 3; + "~*(?:\b)x09Mozilla(?:\b)" 3; + "~*(?:\b)x22Mozilla(?:\b)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; + "~*(?:\b)Xenu(?:\b)" 3; + "~*(?:\b)xpymep1.exe(?:\b)" 3; + "~*(?:\b)YoudaoBot(?:\b)" 3; + "~*(?:\b)Zade(?:\b)" 3; + "~*(?:\b)Zauba(?:\b)" 3; + "~*(?:\b)zauba.io(?:\b)" 3; + "~*(?:\b)Zermelo(?:\b)" 3; + "~*(?:\b)Zeus(?:\b)" 3; + "~*(?:\b)zgrab(?:\b)" 3; + "~*(?:\b)Zitebot(?:\b)" 3; + "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZumBot(?:\b)" 3; + "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------- @@ -717,38 +717,38 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)adidxbot(?:\b|)" 0; - "~*(?:\b)AdsBot-Google(?:\b|)" 0; - "~*(?:\b)aolbuild(?:\b|)" 0; - "~*(?:\b)bingbot(?:\b|)" 0; - "~*(?:\b)bingpreview(?:\b|)" 0; - "~*(?:\b)DoCoMo(?:\b|)" 0; - "~*(?:\b)duckduckgo(?:\b|)" 0; - "~*(?:\b)facebookexternalhit(?:\b|)" 0; - "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; - "~*(?:\b)Googlebot(?:\b|)" 0; - "~*(?:\b)Googlebot-Image(?:\b|)" 0; - "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; - "~*(?:\b)Googlebot-News(?:\b|)" 0; - "~*(?:\b)Googlebot/Test(?:\b|)" 0; - "~*(?:\b)Googlebot-Video(?:\b|)" 0; - "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; - "~*(?:\b)Gravityscan(?:\b|)" 0; - "~*(?:\b)gsa-crawler(?:\b|)" 0; - "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; - "~*(?:\b)Kraken/0.1(?:\b|)" 0; - "~*(?:\b)LinkedInBot(?:\b|)" 0; - "~*(?:\b)Mediapartners-Google(?:\b|)" 0; - "~*(?:\b)msnbot(?:\b|)" 0; - "~*(?:\b)msnbot-media(?:\b|)" 0; - "~*(?:\b)SAMSUNG(?:\b|)" 0; - "~*(?:\b)Slackbot(?:\b|)" 0; - "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; - "~*(?:\b)slurp(?:\b|)" 0; - "~*(?:\b)teoma(?:\b|)" 0; - "~*(?:\b)TwitterBot(?:\b|)" 0; - "~*(?:\b)Wordpress(?:\b|)" 0; - "~*(?:\b)yahoo(?:\b|)" 0; + "~*(?:\b)adidxbot(?:\b)" 0; + "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bingbot(?:\b)" 0; + "~*(?:\b)bingpreview(?:\b)" 0; + "~*(?:\b)DoCoMo(?:\b)" 0; + "~*(?:\b)duckduckgo(?:\b)" 0; + "~*(?:\b)facebookexternalhit(?:\b)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b)" 0; + "~*(?:\b)Googlebot(?:\b)" 0; + "~*(?:\b)Googlebot-Image(?:\b)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b)" 0; + "~*(?:\b)Googlebot-News(?:\b)" 0; + "~*(?:\b)Googlebot/Test(?:\b)" 0; + "~*(?:\b)Googlebot-Video(?:\b)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; + "~*(?:\b)Gravityscan(?:\b)" 0; + "~*(?:\b)gsa-crawler(?:\b)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b)" 0; + "~*(?:\b)Kraken/0.1(?:\b)" 0; + "~*(?:\b)LinkedInBot(?:\b)" 0; + "~*(?:\b)Mediapartners-Google(?:\b)" 0; + "~*(?:\b)msnbot(?:\b)" 0; + "~*(?:\b)msnbot-media(?:\b)" 0; + "~*(?:\b)SAMSUNG(?:\b)" 0; + "~*(?:\b)Slackbot(?:\b)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 0; + "~*(?:\b)slurp(?:\b)" 0; + "~*(?:\b)teoma(?:\b)" 0; + "~*(?:\b)TwitterBot(?:\b)" 0; + "~*(?:\b)Wordpress(?:\b)" 0; + "~*(?:\b)yahoo(?:\b)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------------------- @@ -760,12 +760,12 @@ map $http_user_agent $bad_bot { #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)jetmon(?:\b|)" 1; - "~*(?:\b)libwww-perl(?:\b|)" 1; - "~*(?:\b)Lynx(?:\b|)" 1; - "~*(?:\b)munin(?:\b|)" 1; - "~*(?:\b)Presto(?:\b|)" 1; - "~*(?:\b)Wget/1.15(?:\b|)" 1; + "~*(?:\b)jetmon(?:\b)" 1; + "~*(?:\b)libwww-perl(?:\b)" 1; + "~*(?:\b)Lynx(?:\b)" 1; + "~*(?:\b)munin(?:\b)" 1; + "~*(?:\b)Presto(?:\b)" 1; + "~*(?:\b)Wget/1.15(?:\b)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ------------------------------------------------------- @@ -778,23 +778,23 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)Alexa(?:\b|)" 2; - "~*(?:\b)archive.org(?:\b|)" 2; - "~*(?:\b)Baidu(?:\b|)" 2; - "~*(?:\b)BUbiNG(?:\b|)" 2; - "~*(?:\b)FlipboardProxy(?:\b|)" 2; - "~*(?:\b)ia_archiver(?:\b|)" 2; - "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; - "~*(?:\b)Proximic(?:\b|)" 2; - "~*(?:\b)R6_CommentReader(?:\b|)" 2; - "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; - "~*(?:\b)RED/1(?:\b|)" 2; - "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; - "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; - "~*(?:\b)Spaidu(?:\b|)" 2; - "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; - "~*(?:\b)YandexBot(?:\b|)" 2; - "~*(?:\b)YandexImages(?:\b|)" 2; + "~*(?:\b)Alexa(?:\b)" 2; + "~*(?:\b)archive.org(?:\b)" 2; + "~*(?:\b)Baidu(?:\b)" 2; + "~*(?:\b)BUbiNG(?:\b)" 2; + "~*(?:\b)FlipboardProxy(?:\b)" 2; + "~*(?:\b)ia_archiver(?:\b)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b)" 2; + "~*(?:\b)Proximic(?:\b)" 2; + "~*(?:\b)R6_CommentReader(?:\b)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b)" 2; + "~*(?:\b)RED/1(?:\b)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b)" 2; + "~*(?:\b)Spaidu(?:\b)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b)" 2; + "~*(?:\b)YandexBot(?:\b)" 2; + "~*(?:\b)YandexImages(?:\b)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index b9d24324a..0f27f0dd9 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 1; +104.154.113.151 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index ea117da9f..0e0848852 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 0; +104.154.113.151 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index f26e52add..a1999739a 100644 --- a/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1709 -### Updated: Mon Jul 1 10:37:04 SAST 2019 +### Version: V4.2019.07.1712 +### Updated: Mon Jul 1 11:24:52 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### @@ -147,562 +147,562 @@ map $http_user_agent $bad_bot { # -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b|)" 3; - "~*(?:\b)404checker(?:\b|)" 3; - "~*(?:\b)404enemy(?:\b|)" 3; - "~*(?:\b)80legs(?:\b|)" 3; - "~*(?:\b)Abonti(?:\b|)" 3; - "~*(?:\b)Aboundex(?:\b|)" 3; - "~*(?:\b)Aboundexbot(?:\b|)" 3; - "~*(?:\b)Acunetix(?:\b|)" 3; - "~*(?:\b)ADmantX(?:\b|)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; - "~*(?:\b)AhrefsBot(?:\b|)" 3; - "~*(?:\b)AIBOT(?:\b|)" 3; - "~*(?:\b)AiHitBot(?:\b|)" 3; - "~*(?:\b)Aipbot(?:\b|)" 3; - "~*(?:\b)Alexibot(?:\b|)" 3; - "~*(?:\b)Alligator(?:\b|)" 3; - "~*(?:\b)AllSubmitter(?:\b|)" 3; - "~*(?:\b)AlphaBot(?:\b|)" 3; - "~*(?:\b)Anarchie(?:\b|)" 3; - "~*(?:\b)Apexoo(?:\b|)" 3; - "~*(?:\b)archive.org_bot(?:\b|)" 3; - "~*(?:\b)ASPSeek(?:\b|)" 3; - "~*(?:\b)Asterias(?:\b|)" 3; - "~*(?:\b)Attach(?:\b|)" 3; - "~*(?:\b)autoemailspider(?:\b|)" 3; - "~*(?:\b)BackDoorBot(?:\b|)" 3; - "~*(?:\b)Backlink-Ceck(?:\b|)" 3; - "~*(?:\b)backlink-check(?:\b|)" 3; - "~*(?:\b)BacklinkCrawler(?:\b|)" 3; - "~*(?:\b)BackStreet(?:\b|)" 3; - "~*(?:\b)BackWeb(?:\b|)" 3; - "~*(?:\b)Badass(?:\b|)" 3; - "~*(?:\b)Bandit(?:\b|)" 3; - "~*(?:\b)Barkrowler(?:\b|)" 3; - "~*(?:\b)BatchFTP(?:\b|)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; - "~*(?:\b)BBBike(?:\b|)" 3; - "~*(?:\b)BDCbot(?:\b|)" 3; - "~*(?:\b)BDFetch(?:\b|)" 3; - "~*(?:\b)BetaBot(?:\b|)" 3; - "~*(?:\b)Bigfoot(?:\b|)" 3; - "~*(?:\b)Bitacle(?:\b|)" 3; - "~*(?:\b)Blackboard(?:\b|)" 3; - "~*(?:\b)Black\ Hole(?:\b|)" 3; - "~*(?:\b)BlackWidow(?:\b|)" 3; - "~*(?:\b)BLEXBot(?:\b|)" 3; - "~*(?:\b)Blow(?:\b|)" 3; - "~*(?:\b)BlowFish(?:\b|)" 3; - "~*(?:\b)Boardreader(?:\b|)" 3; - "~*(?:\b)Bolt(?:\b|)" 3; - "~*(?:\b)BotALot(?:\b|)" 3; - "~*(?:\b)Brandprotect(?:\b|)" 3; - "~*(?:\b)Brandwatch(?:\b|)" 3; - "~*(?:\b)Buddy(?:\b|)" 3; - "~*(?:\b)BuiltBotTough(?:\b|)" 3; - "~*(?:\b)BuiltWith(?:\b|)" 3; - "~*(?:\b)Bullseye(?:\b|)" 3; - "~*(?:\b)BunnySlippers(?:\b|)" 3; - "~*(?:\b)BuzzSumo(?:\b|)" 3; - "~*(?:\b)Calculon(?:\b|)" 3; - "~*(?:\b)CATExplorador(?:\b|)" 3; - "~*(?:\b)CazoodleBot(?:\b|)" 3; - "~*(?:\b)CCBot(?:\b|)" 3; - "~*(?:\b)Cegbfeieh(?:\b|)" 3; - "~*(?:\b)CheeseBot(?:\b|)" 3; - "~*(?:\b)CherryPicker(?:\b|)" 3; - "~*(?:\b)CheTeam(?:\b|)" 3; - "~*(?:\b)ChinaClaw(?:\b|)" 3; - "~*(?:\b)Chlooe(?:\b|)" 3; - "~*(?:\b)Claritybot(?:\b|)" 3; - "~*(?:\b)Cliqzbot(?:\b|)" 3; - "~*(?:\b)Cloud\ mapping(?:\b|)" 3; - "~*(?:\b)coccocbot-web(?:\b|)" 3; - "~*(?:\b)Cogentbot(?:\b|)" 3; - "~*(?:\b)cognitiveseo(?:\b|)" 3; - "~*(?:\b)Collector(?:\b|)" 3; - "~*(?:\b)com.plumanalytics(?:\b|)" 3; - "~*(?:\b)Copier(?:\b|)" 3; - "~*(?:\b)CopyRightCheck(?:\b|)" 3; - "~*(?:\b)Copyscape(?:\b|)" 3; - "~*(?:\b)Cosmos(?:\b|)" 3; - "~*(?:\b)Craftbot(?:\b|)" 3; - "~*(?:\b)crawler4j(?:\b|)" 3; - "~*(?:\b)crawler.feedback(?:\b|)" 3; - "~*(?:\b)crawl.sogou.com(?:\b|)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; - "~*(?:\b)Crescent(?:\b|)" 3; - "~*(?:\b)CrunchBot(?:\b|)" 3; - "~*(?:\b)CSHttp(?:\b|)" 3; - "~*(?:\b)Curious(?:\b|)" 3; - "~*(?:\b)Custo(?:\b|)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; - "~*(?:\b)DataCha0s(?:\b|)" 3; - "~*(?:\b)DBLBot(?:\b|)" 3; - "~*(?:\b)demandbase-bot(?:\b|)" 3; - "~*(?:\b)Demon(?:\b|)" 3; - "~*(?:\b)Deusu(?:\b|)" 3; - "~*(?:\b)Devil(?:\b|)" 3; - "~*(?:\b)Digincore(?:\b|)" 3; - "~*(?:\b)DigitalPebble(?:\b|)" 3; - "~*(?:\b)DIIbot(?:\b|)" 3; - "~*(?:\b)Dirbuster(?:\b|)" 3; - "~*(?:\b)Disco(?:\b|)" 3; - "~*(?:\b)Discobot(?:\b|)" 3; - "~*(?:\b)Discoverybot(?:\b|)" 3; - "~*(?:\b)Dispatch(?:\b|)" 3; - "~*(?:\b)DittoSpyder(?:\b|)" 3; - "~*(?:\b)DnyzBot(?:\b|)" 3; - "~*(?:\b)DomainAppender(?:\b|)" 3; - "~*(?:\b)DomainCrawler(?:\b|)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; - "~*(?:\b)DomainStatsBot(?:\b|)" 3; - "~*(?:\b)Dotbot(?:\b|)" 3; - "~*(?:\b)Download\ Wonder(?:\b|)" 3; - "~*(?:\b)Dragonfly(?:\b|)" 3; - "~*(?:\b)Drip(?:\b|)" 3; - "~*(?:\b)DSearch(?:\b|)" 3; - "~*(?:\b)DTS\ Agent(?:\b|)" 3; - "~*(?:\b)EasyDL(?:\b|)" 3; - "~*(?:\b)Ebingbong(?:\b|)" 3; - "~*(?:\b)eCatch(?:\b|)" 3; - "~*(?:\b)ECCP/1.0(?:\b|)" 3; - "~*(?:\b)Ecxi(?:\b|)" 3; - "~*(?:\b)EirGrabber(?:\b|)" 3; - "~*(?:\b)EMail\ Siphon(?:\b|)" 3; - "~*(?:\b)EMail\ Wolf(?:\b|)" 3; - "~*(?:\b)EroCrawler(?:\b|)" 3; - "~*(?:\b)evc-batch(?:\b|)" 3; - "~*(?:\b)Evil(?:\b|)" 3; - "~*(?:\b)Exabot(?:\b|)" 3; - "~*(?:\b)Express\ WebPictures(?:\b|)" 3; - "~*(?:\b)ExtLinksBot(?:\b|)" 3; - "~*(?:\b)Extractor(?:\b|)" 3; - "~*(?:\b)ExtractorPro(?:\b|)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; - "~*(?:\b)EyeNetIE(?:\b|)" 3; - "~*(?:\b)Ezooms(?:\b|)" 3; - "~*(?:\b)facebookscraper(?:\b|)" 3; - "~*(?:\b)FDM(?:\b|)" 3; - "~*(?:\b)FemtosearchBot(?:\b|)" 3; - "~*(?:\b)FHscan(?:\b|)" 3; - "~*(?:\b)Fimap(?:\b|)" 3; - "~*(?:\b)Firefox/7.0(?:\b|)" 3; - "~*(?:\b)FlashGet(?:\b|)" 3; - "~*(?:\b)Flunky(?:\b|)" 3; - "~*(?:\b)Foobot(?:\b|)" 3; - "~*(?:\b)Freeuploader(?:\b|)" 3; - "~*(?:\b)FrontPage(?:\b|)" 3; - "~*(?:\b)FyberSpider(?:\b|)" 3; - "~*(?:\b)Fyrebot(?:\b|)" 3; - "~*(?:\b)GalaxyBot(?:\b|)" 3; - "~*(?:\b)Genieo(?:\b|)" 3; - "~*(?:\b)GermCrawler(?:\b|)" 3; - "~*(?:\b)Getintent(?:\b|)" 3; - "~*(?:\b)GetRight(?:\b|)" 3; - "~*(?:\b)GetWeb(?:\b|)" 3; - "~*(?:\b)Gigablast(?:\b|)" 3; - "~*(?:\b)Gigabot(?:\b|)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; - "~*(?:\b)Gotit(?:\b|)" 3; - "~*(?:\b)GoZilla(?:\b|)" 3; - "~*(?:\b)Go!Zilla(?:\b|)" 3; - "~*(?:\b)Grabber(?:\b|)" 3; - "~*(?:\b)GrabNet(?:\b|)" 3; - "~*(?:\b)Grafula(?:\b|)" 3; - "~*(?:\b)GrapeFX(?:\b|)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; - "~*(?:\b)GridBot(?:\b|)" 3; - "~*(?:\b)GT::WWW(?:\b|)" 3; - "~*(?:\b)Haansoft(?:\b|)" 3; - "~*(?:\b)HaosouSpider(?:\b|)" 3; - "~*(?:\b)Harvest(?:\b|)" 3; - "~*(?:\b)Havij(?:\b|)" 3; - "~*(?:\b)HEADMasterSEO(?:\b|)" 3; - "~*(?:\b)Heritrix(?:\b|)" 3; - "~*(?:\b)Hloader(?:\b|)" 3; - "~*(?:\b)HMView(?:\b|)" 3; - "~*(?:\b)HTMLparser(?:\b|)" 3; - "~*(?:\b)HTTP::Lite(?:\b|)" 3; - "~*(?:\b)HTTrack(?:\b|)" 3; - "~*(?:\b)Humanlinks(?:\b|)" 3; - "~*(?:\b)HybridBot(?:\b|)" 3; - "~*(?:\b)Iblog(?:\b|)" 3; - "~*(?:\b)IDBot(?:\b|)" 3; - "~*(?:\b)Id-search(?:\b|)" 3; - "~*(?:\b)IlseBot(?:\b|)" 3; - "~*(?:\b)Image\ Fetch(?:\b|)" 3; - "~*(?:\b)Image\ Sucker(?:\b|)" 3; - "~*(?:\b)IndeedBot(?:\b|)" 3; - "~*(?:\b)Indy\ Library(?:\b|)" 3; - "~*(?:\b)InfoNaviRobot(?:\b|)" 3; - "~*(?:\b)InfoTekies(?:\b|)" 3; - "~*(?:\b)instabid(?:\b|)" 3; - "~*(?:\b)Intelliseek(?:\b|)" 3; - "~*(?:\b)InterGET(?:\b|)" 3; - "~*(?:\b)Internet\ Ninja(?:\b|)" 3; - "~*(?:\b)InternetSeer(?:\b|)" 3; - "~*(?:\b)internetVista\ monitor(?:\b|)" 3; - "~*(?:\b)ips-agent(?:\b|)" 3; - "~*(?:\b)Iria(?:\b|)" 3; - "~*(?:\b)IRLbot(?:\b|)" 3; - "~*(?:\b)Iskanie(?:\b|)" 3; - "~*(?:\b)IstellaBot(?:\b|)" 3; - "~*(?:\b)JamesBOT(?:\b|)" 3; - "~*(?:\b)Jbrofuzz(?:\b|)" 3; - "~*(?:\b)JennyBot(?:\b|)" 3; - "~*(?:\b)JetCar(?:\b|)" 3; - "~*(?:\b)Jetty(?:\b|)" 3; - "~*(?:\b)JikeSpider(?:\b|)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; - "~*(?:\b)Joomla(?:\b|)" 3; - "~*(?:\b)Jorgee(?:\b|)" 3; - "~*(?:\b)JustView(?:\b|)" 3; - "~*(?:\b)Jyxobot(?:\b|)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; - "~*(?:\b)Keyword\ Density(?:\b|)" 3; - "~*(?:\b)Kozmosbot(?:\b|)" 3; - "~*(?:\b)Lanshanbot(?:\b|)" 3; - "~*(?:\b)Larbin(?:\b|)" 3; - "~*(?:\b)LeechFTP(?:\b|)" 3; - "~*(?:\b)LeechGet(?:\b|)" 3; - "~*(?:\b)LexiBot(?:\b|)" 3; - "~*(?:\b)Lftp(?:\b|)" 3; - "~*(?:\b)LibWeb(?:\b|)" 3; - "~*(?:\b)Libwhisker(?:\b|)" 3; - "~*(?:\b)Lightspeedsystems(?:\b|)" 3; - "~*(?:\b)Likse(?:\b|)" 3; - "~*(?:\b)Linkdexbot(?:\b|)" 3; - "~*(?:\b)LinkextractorPro(?:\b|)" 3; - "~*(?:\b)LinkpadBot(?:\b|)" 3; - "~*(?:\b)LinkScan(?:\b|)" 3; - "~*(?:\b)LinksManager(?:\b|)" 3; - "~*(?:\b)LinkWalker(?:\b|)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; - "~*(?:\b)Lipperhey(?:\b|)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; - "~*(?:\b)Litemage_walker(?:\b|)" 3; - "~*(?:\b)Lmspider(?:\b|)" 3; - "~*(?:\b)LNSpiderguy(?:\b|)" 3; - "~*(?:\b)Ltx71(?:\b|)" 3; - "~*(?:\b)lwp-request(?:\b|)" 3; - "~*(?:\b)LWP::Simple(?:\b|)" 3; - "~*(?:\b)lwp-trivial(?:\b|)" 3; - "~*(?:\b)Magnet(?:\b|)" 3; - "~*(?:\b)Mag-Net(?:\b|)" 3; - "~*(?:\b)magpie-crawler(?:\b|)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; - "~*(?:\b)Majestic12(?:\b|)" 3; - "~*(?:\b)Majestic-SEO(?:\b|)" 3; - "~*(?:\b)Majestic\ SEO(?:\b|)" 3; - "~*(?:\b)MarkMonitor(?:\b|)" 3; - "~*(?:\b)MarkWatch(?:\b|)" 3; - "~*(?:\b)Masscan(?:\b|)" 3; - "~*(?:\b)Mass\ Downloader(?:\b|)" 3; - "~*(?:\b)Mata\ Hari(?:\b|)" 3; - "~*(?:\b)MauiBot(?:\b|)" 3; - "~*(?:\b)meanpathbot(?:\b|)" 3; - "~*(?:\b)Meanpathbot(?:\b|)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; - "~*(?:\b)mediawords(?:\b|)" 3; - "~*(?:\b)MegaIndex.ru(?:\b|)" 3; - "~*(?:\b)Metauri(?:\b|)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; - "~*(?:\b)MIDown\ tool(?:\b|)" 3; - "~*(?:\b)MIIxpc(?:\b|)" 3; - "~*(?:\b)Mister\ PiX(?:\b|)" 3; - "~*(?:\b)MJ12bot(?:\b|)" 3; - "~*(?:\b)Mojeek(?:\b|)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; - "~*(?:\b)Mr.4x3(?:\b|)" 3; - "~*(?:\b)MSFrontPage(?:\b|)" 3; - "~*(?:\b)MSIECrawler(?:\b|)" 3; - "~*(?:\b)Msrabot(?:\b|)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; - "~*(?:\b)muhstik-scan(?:\b|)" 3; - "~*(?:\b)Musobot(?:\b|)" 3; - "~*(?:\b)Name\ Intelligence(?:\b|)" 3; - "~*(?:\b)Nameprotect(?:\b|)" 3; - "~*(?:\b)Navroad(?:\b|)" 3; - "~*(?:\b)NearSite(?:\b|)" 3; - "~*(?:\b)Needle(?:\b|)" 3; - "~*(?:\b)Nessus(?:\b|)" 3; - "~*(?:\b)NetAnts(?:\b|)" 3; - "~*(?:\b)Netcraft(?:\b|)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; - "~*(?:\b)NetLyzer(?:\b|)" 3; - "~*(?:\b)NetMechanic(?:\b|)" 3; - "~*(?:\b)NetSpider(?:\b|)" 3; - "~*(?:\b)Nettrack(?:\b|)" 3; - "~*(?:\b)Net\ Vampire(?:\b|)" 3; - "~*(?:\b)Netvibes(?:\b|)" 3; - "~*(?:\b)NetZIP(?:\b|)" 3; - "~*(?:\b)NextGenSearchBot(?:\b|)" 3; - "~*(?:\b)Nibbler(?:\b|)" 3; - "~*(?:\b)NICErsPRO(?:\b|)" 3; - "~*(?:\b)Niki-bot(?:\b|)" 3; - "~*(?:\b)Nikto(?:\b|)" 3; - "~*(?:\b)NimbleCrawler(?:\b|)" 3; - "~*(?:\b)Nimbostratus(?:\b|)" 3; - "~*(?:\b)Ninja(?:\b|)" 3; - "~*(?:\b)Nmap(?:\b|)" 3; - "~*(?:\b)NPbot(?:\b|)" 3; - "~*(?:\b)Nutch(?:\b|)" 3; - "~*(?:\b)oBot(?:\b|)" 3; - "~*(?:\b)Octopus(?:\b|)" 3; - "~*(?:\b)Offline\ Explorer(?:\b|)" 3; - "~*(?:\b)Offline\ Navigator(?:\b|)" 3; - "~*(?:\b)OnCrawl(?:\b|)" 3; - "~*(?:\b)Openfind(?:\b|)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; - "~*(?:\b)Openvas(?:\b|)" 3; - "~*(?:\b)OpenVAS(?:\b|)" 3; - "~*(?:\b)OrangeBot(?:\b|)" 3; - "~*(?:\b)OrangeSpider(?:\b|)" 3; - "~*(?:\b)OutclicksBot(?:\b|)" 3; - "~*(?:\b)OutfoxBot(?:\b|)" 3; - "~*(?:\b)PageAnalyzer(?:\b|)" 3; - "~*(?:\b)Page\ Analyzer(?:\b|)" 3; - "~*(?:\b)PageGrabber(?:\b|)" 3; - "~*(?:\b)page\ scorer(?:\b|)" 3; - "~*(?:\b)PageScorer(?:\b|)" 3; - "~*(?:\b)Pandalytics(?:\b|)" 3; - "~*(?:\b)Panscient(?:\b|)" 3; - "~*(?:\b)Papa\ Foto(?:\b|)" 3; - "~*(?:\b)Pavuk(?:\b|)" 3; - "~*(?:\b)pcBrowser(?:\b|)" 3; - "~*(?:\b)PECL::HTTP(?:\b|)" 3; - "~*(?:\b)PeoplePal(?:\b|)" 3; - "~*(?:\b)PHPCrawl(?:\b|)" 3; - "~*(?:\b)Picscout(?:\b|)" 3; - "~*(?:\b)Picsearch(?:\b|)" 3; - "~*(?:\b)PictureFinder(?:\b|)" 3; - "~*(?:\b)Pimonster(?:\b|)" 3; - "~*(?:\b)Pi-Monster(?:\b|)" 3; - "~*(?:\b)Pixray(?:\b|)" 3; - "~*(?:\b)PleaseCrawl(?:\b|)" 3; - "~*(?:\b)plumanalytics(?:\b|)" 3; - "~*(?:\b)Pockey(?:\b|)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; - "~*(?:\b)Probethenet(?:\b|)" 3; - "~*(?:\b)ProPowerBot(?:\b|)" 3; - "~*(?:\b)ProWebWalker(?:\b|)" 3; - "~*(?:\b)Psbot(?:\b|)" 3; - "~*(?:\b)Pump(?:\b|)" 3; - "~*(?:\b)PxBroker(?:\b|)" 3; - "~*(?:\b)PyCurl(?:\b|)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; - "~*(?:\b)Quick-Crawler(?:\b|)" 3; - "~*(?:\b)RankActive(?:\b|)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; - "~*(?:\b)RankFlex(?:\b|)" 3; - "~*(?:\b)RankingBot(?:\b|)" 3; - "~*(?:\b)RankingBot2(?:\b|)" 3; - "~*(?:\b)Rankivabot(?:\b|)" 3; - "~*(?:\b)RankurBot(?:\b|)" 3; - "~*(?:\b)RealDownload(?:\b|)" 3; - "~*(?:\b)Reaper(?:\b|)" 3; - "~*(?:\b)RebelMouse(?:\b|)" 3; - "~*(?:\b)Recorder(?:\b|)" 3; - "~*(?:\b)RedesScrapy(?:\b|)" 3; - "~*(?:\b)ReGet(?:\b|)" 3; - "~*(?:\b)RepoMonkey(?:\b|)" 3; - "~*(?:\b)Ripper(?:\b|)" 3; - "~*(?:\b)RocketCrawler(?:\b|)" 3; - "~*(?:\b)Rogerbot(?:\b|)" 3; - "~*(?:\b)s1z.ru(?:\b|)" 3; - "~*(?:\b)SalesIntelligent(?:\b|)" 3; - "~*(?:\b)SBIder(?:\b|)" 3; - "~*(?:\b)ScanAlert(?:\b|)" 3; - "~*(?:\b)Scanbot(?:\b|)" 3; - "~*(?:\b)scan.lol(?:\b|)" 3; - "~*(?:\b)ScoutJet(?:\b|)" 3; - "~*(?:\b)Scrapy(?:\b|)" 3; - "~*(?:\b)Screaming(?:\b|)" 3; - "~*(?:\b)ScreenerBot(?:\b|)" 3; - "~*(?:\b)Searchestate(?:\b|)" 3; - "~*(?:\b)SearchmetricsBot(?:\b|)" 3; - "~*(?:\b)Semrush(?:\b|)" 3; - "~*(?:\b)SemrushBot(?:\b|)" 3; - "~*(?:\b)SEOkicks(?:\b|)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; - "~*(?:\b)Seomoz(?:\b|)" 3; - "~*(?:\b)SEOprofiler(?:\b|)" 3; - "~*(?:\b)seoscanners(?:\b|)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; - "~*(?:\b)SEOstats(?:\b|)" 3; - "~*(?:\b)serpstatbot(?:\b|)" 3; - "~*(?:\b)sexsearcher(?:\b|)" 3; - "~*(?:\b)Shodan(?:\b|)" 3; - "~*(?:\b)Siphon(?:\b|)" 3; - "~*(?:\b)SISTRIX(?:\b|)" 3; - "~*(?:\b)Sitebeam(?:\b|)" 3; - "~*(?:\b)SiteExplorer(?:\b|)" 3; - "~*(?:\b)Siteimprove(?:\b|)" 3; - "~*(?:\b)SiteLockSpider(?:\b|)" 3; - "~*(?:\b)SiteSnagger(?:\b|)" 3; - "~*(?:\b)SiteSucker(?:\b|)" 3; - "~*(?:\b)Site\ Sucker(?:\b|)" 3; - "~*(?:\b)Sitevigil(?:\b|)" 3; - "~*(?:\b)SlySearch(?:\b|)" 3; - "~*(?:\b)SmartDownload(?:\b|)" 3; - "~*(?:\b)SMTBot(?:\b|)" 3; - "~*(?:\b)Snake(?:\b|)" 3; - "~*(?:\b)Snapbot(?:\b|)" 3; - "~*(?:\b)Snoopy(?:\b|)" 3; - "~*(?:\b)SocialRankIOBot(?:\b|)" 3; - "~*(?:\b)Sociscraper(?:\b|)" 3; - "~*(?:\b)sogouspider(?:\b|)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; - "~*(?:\b)Sosospider(?:\b|)" 3; - "~*(?:\b)Sottopop(?:\b|)" 3; - "~*(?:\b)SpaceBison(?:\b|)" 3; - "~*(?:\b)Spammen(?:\b|)" 3; - "~*(?:\b)SpankBot(?:\b|)" 3; - "~*(?:\b)Spanner(?:\b|)" 3; - "~*(?:\b)sp_auditbot(?:\b|)" 3; - "~*(?:\b)Spbot(?:\b|)" 3; - "~*(?:\b)Spinn3r(?:\b|)" 3; - "~*(?:\b)SputnikBot(?:\b|)" 3; - "~*(?:\b)spyfu(?:\b|)" 3; - "~*(?:\b)Sqlmap(?:\b|)" 3; - "~*(?:\b)Sqlworm(?:\b|)" 3; - "~*(?:\b)Sqworm(?:\b|)" 3; - "~*(?:\b)Steeler(?:\b|)" 3; - "~*(?:\b)Stripper(?:\b|)" 3; - "~*(?:\b)Sucker(?:\b|)" 3; - "~*(?:\b)Sucuri(?:\b|)" 3; - "~*(?:\b)SuperBot(?:\b|)" 3; - "~*(?:\b)SuperHTTP(?:\b|)" 3; - "~*(?:\b)Surfbot(?:\b|)" 3; - "~*(?:\b)SurveyBot(?:\b|)" 3; - "~*(?:\b)Suzuran(?:\b|)" 3; - "~*(?:\b)Swiftbot(?:\b|)" 3; - "~*(?:\b)sysscan(?:\b|)" 3; - "~*(?:\b)Szukacz(?:\b|)" 3; - "~*(?:\b)T0PHackTeam(?:\b|)" 3; - "~*(?:\b)T8Abot(?:\b|)" 3; - "~*(?:\b)tAkeOut(?:\b|)" 3; - "~*(?:\b)Teleport(?:\b|)" 3; - "~*(?:\b)TeleportPro(?:\b|)" 3; - "~*(?:\b)Telesoft(?:\b|)" 3; - "~*(?:\b)Telesphoreo(?:\b|)" 3; - "~*(?:\b)Telesphorep(?:\b|)" 3; - "~*(?:\b)The\ Intraformant(?:\b|)" 3; - "~*(?:\b)TheNomad(?:\b|)" 3; - "~*(?:\b)Thumbor(?:\b|)" 3; - "~*(?:\b)TightTwatBot(?:\b|)" 3; - "~*(?:\b)Titan(?:\b|)" 3; - "~*(?:\b)Toata(?:\b|)" 3; - "~*(?:\b)Toweyabot(?:\b|)" 3; - "~*(?:\b)Tracemyfile(?:\b|)" 3; - "~*(?:\b)Trendiction(?:\b|)" 3; - "~*(?:\b)Trendictionbot(?:\b|)" 3; - "~*(?:\b)trendiction.com(?:\b|)" 3; - "~*(?:\b)trendiction.de(?:\b|)" 3; - "~*(?:\b)True_Robot(?:\b|)" 3; - "~*(?:\b)Turingos(?:\b|)" 3; - "~*(?:\b)Turnitin(?:\b|)" 3; - "~*(?:\b)TurnitinBot(?:\b|)" 3; - "~*(?:\b)TwengaBot(?:\b|)" 3; - "~*(?:\b)Twice(?:\b|)" 3; - "~*(?:\b)Typhoeus(?:\b|)" 3; - "~*(?:\b)UnisterBot(?:\b|)" 3; - "~*(?:\b)Upflow(?:\b|)" 3; - "~*(?:\b)URLy.Warning(?:\b|)" 3; - "~*(?:\b)URLy\ Warning(?:\b|)" 3; - "~*(?:\b)Vacuum(?:\b|)" 3; - "~*(?:\b)Vagabondo(?:\b|)" 3; - "~*(?:\b)VB\ Project(?:\b|)" 3; - "~*(?:\b)VCI(?:\b|)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; - "~*(?:\b)VidibleScraper(?:\b|)" 3; - "~*(?:\b)Virusdie(?:\b|)" 3; - "~*(?:\b)VoidEYE(?:\b|)" 3; - "~*(?:\b)Voil(?:\b|)" 3; - "~*(?:\b)Voltron(?:\b|)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; - "~*(?:\b)WallpapersHD(?:\b|)" 3; - "~*(?:\b)WASALive-Bot(?:\b|)" 3; - "~*(?:\b)WBSearchBot(?:\b|)" 3; - "~*(?:\b)Webalta(?:\b|)" 3; - "~*(?:\b)WebAuto(?:\b|)" 3; - "~*(?:\b)Web\ Auto(?:\b|)" 3; - "~*(?:\b)WebBandit(?:\b|)" 3; - "~*(?:\b)WebCollage(?:\b|)" 3; - "~*(?:\b)Web\ Collage(?:\b|)" 3; - "~*(?:\b)WebCopier(?:\b|)" 3; - "~*(?:\b)WEBDAV(?:\b|)" 3; - "~*(?:\b)WebEnhancer(?:\b|)" 3; - "~*(?:\b)Web\ Enhancer(?:\b|)" 3; - "~*(?:\b)WebFetch(?:\b|)" 3; - "~*(?:\b)Web\ Fetch(?:\b|)" 3; - "~*(?:\b)WebFuck(?:\b|)" 3; - "~*(?:\b)Web\ Fuck(?:\b|)" 3; - "~*(?:\b)WebGo\ IS(?:\b|)" 3; - "~*(?:\b)WebImageCollector(?:\b|)" 3; - "~*(?:\b)WebLeacher(?:\b|)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; - "~*(?:\b)webmeup-crawler(?:\b|)" 3; - "~*(?:\b)WebPix(?:\b|)" 3; - "~*(?:\b)Web\ Pix(?:\b|)" 3; - "~*(?:\b)WebReaper(?:\b|)" 3; - "~*(?:\b)WebSauger(?:\b|)" 3; - "~*(?:\b)Web\ Sauger(?:\b|)" 3; - "~*(?:\b)Webshag(?:\b|)" 3; - "~*(?:\b)WebsiteExtractor(?:\b|)" 3; - "~*(?:\b)WebsiteQuester(?:\b|)" 3; - "~*(?:\b)Website\ Quester(?:\b|)" 3; - "~*(?:\b)Webster(?:\b|)" 3; - "~*(?:\b)WebStripper(?:\b|)" 3; - "~*(?:\b)WebSucker(?:\b|)" 3; - "~*(?:\b)Web\ Sucker(?:\b|)" 3; - "~*(?:\b)WebWhacker(?:\b|)" 3; - "~*(?:\b)WebZIP(?:\b|)" 3; - "~*(?:\b)WeSEE(?:\b|)" 3; - "~*(?:\b)Whack(?:\b|)" 3; - "~*(?:\b)Whacker(?:\b|)" 3; - "~*(?:\b)Whatweb(?:\b|)" 3; - "~*(?:\b)Who.is\ Bot(?:\b|)" 3; - "~*(?:\b)Widow(?:\b|)" 3; - "~*(?:\b)WinHTTrack(?:\b|)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; - "~*(?:\b)WISENutbot(?:\b|)" 3; - "~*(?:\b)Wonderbot(?:\b|)" 3; - "~*(?:\b)Woobot(?:\b|)" 3; - "~*(?:\b)Wotbox(?:\b|)" 3; - "~*(?:\b)Wprecon(?:\b|)" 3; - "~*(?:\b)WPScan(?:\b|)" 3; - "~*(?:\b)WWW-Collector-E(?:\b|)" 3; - "~*(?:\b)WWW-Mechanize(?:\b|)" 3; - "~*(?:\b)WWW::Mechanize(?:\b|)" 3; - "~*(?:\b)WWWOFFLE(?:\b|)" 3; - "~*(?:\b)x09Mozilla(?:\b|)" 3; - "~*(?:\b)x22Mozilla(?:\b|)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; - "~*(?:\b)Xenu(?:\b|)" 3; - "~*(?:\b)xpymep1.exe(?:\b|)" 3; - "~*(?:\b)YoudaoBot(?:\b|)" 3; - "~*(?:\b)Zade(?:\b|)" 3; - "~*(?:\b)Zauba(?:\b|)" 3; - "~*(?:\b)zauba.io(?:\b|)" 3; - "~*(?:\b)Zermelo(?:\b|)" 3; - "~*(?:\b)Zeus(?:\b|)" 3; - "~*(?:\b)zgrab(?:\b|)" 3; - "~*(?:\b)Zitebot(?:\b|)" 3; - "~*(?:\b)ZmEu(?:\b|)" 3; - "~*(?:\b)ZumBot(?:\b|)" 3; - "~*(?:\b)ZyBorg(?:\b|)" 3; + "~*(?:\b)360Spider(?:\b)" 3; + "~*(?:\b)404checker(?:\b)" 3; + "~*(?:\b)404enemy(?:\b)" 3; + "~*(?:\b)80legs(?:\b)" 3; + "~*(?:\b)Abonti(?:\b)" 3; + "~*(?:\b)Aboundex(?:\b)" 3; + "~*(?:\b)Aboundexbot(?:\b)" 3; + "~*(?:\b)Acunetix(?:\b)" 3; + "~*(?:\b)ADmantX(?:\b)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; + "~*(?:\b)AhrefsBot(?:\b)" 3; + "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)AiHitBot(?:\b)" 3; + "~*(?:\b)Aipbot(?:\b)" 3; + "~*(?:\b)Alexibot(?:\b)" 3; + "~*(?:\b)Alligator(?:\b)" 3; + "~*(?:\b)AllSubmitter(?:\b)" 3; + "~*(?:\b)AlphaBot(?:\b)" 3; + "~*(?:\b)Anarchie(?:\b)" 3; + "~*(?:\b)Apexoo(?:\b)" 3; + "~*(?:\b)archive.org_bot(?:\b)" 3; + "~*(?:\b)ASPSeek(?:\b)" 3; + "~*(?:\b)Asterias(?:\b)" 3; + "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)BackDoorBot(?:\b)" 3; + "~*(?:\b)Backlink-Ceck(?:\b)" 3; + "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)BacklinkCrawler(?:\b)" 3; + "~*(?:\b)BackStreet(?:\b)" 3; + "~*(?:\b)BackWeb(?:\b)" 3; + "~*(?:\b)Badass(?:\b)" 3; + "~*(?:\b)Bandit(?:\b)" 3; + "~*(?:\b)Barkrowler(?:\b)" 3; + "~*(?:\b)BatchFTP(?:\b)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; + "~*(?:\b)BBBike(?:\b)" 3; + "~*(?:\b)BDCbot(?:\b)" 3; + "~*(?:\b)BDFetch(?:\b)" 3; + "~*(?:\b)BetaBot(?:\b)" 3; + "~*(?:\b)Bigfoot(?:\b)" 3; + "~*(?:\b)Bitacle(?:\b)" 3; + "~*(?:\b)Blackboard(?:\b)" 3; + "~*(?:\b)Black\ Hole(?:\b)" 3; + "~*(?:\b)BlackWidow(?:\b)" 3; + "~*(?:\b)BLEXBot(?:\b)" 3; + "~*(?:\b)Blow(?:\b)" 3; + "~*(?:\b)BlowFish(?:\b)" 3; + "~*(?:\b)Boardreader(?:\b)" 3; + "~*(?:\b)Bolt(?:\b)" 3; + "~*(?:\b)BotALot(?:\b)" 3; + "~*(?:\b)Brandprotect(?:\b)" 3; + "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buddy(?:\b)" 3; + "~*(?:\b)BuiltBotTough(?:\b)" 3; + "~*(?:\b)BuiltWith(?:\b)" 3; + "~*(?:\b)Bullseye(?:\b)" 3; + "~*(?:\b)BunnySlippers(?:\b)" 3; + "~*(?:\b)BuzzSumo(?:\b)" 3; + "~*(?:\b)Calculon(?:\b)" 3; + "~*(?:\b)CATExplorador(?:\b)" 3; + "~*(?:\b)CazoodleBot(?:\b)" 3; + "~*(?:\b)CCBot(?:\b)" 3; + "~*(?:\b)Cegbfeieh(?:\b)" 3; + "~*(?:\b)CheeseBot(?:\b)" 3; + "~*(?:\b)CherryPicker(?:\b)" 3; + "~*(?:\b)CheTeam(?:\b)" 3; + "~*(?:\b)ChinaClaw(?:\b)" 3; + "~*(?:\b)Chlooe(?:\b)" 3; + "~*(?:\b)Claritybot(?:\b)" 3; + "~*(?:\b)Cliqzbot(?:\b)" 3; + "~*(?:\b)Cloud\ mapping(?:\b)" 3; + "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cogentbot(?:\b)" 3; + "~*(?:\b)cognitiveseo(?:\b)" 3; + "~*(?:\b)Collector(?:\b)" 3; + "~*(?:\b)com.plumanalytics(?:\b)" 3; + "~*(?:\b)Copier(?:\b)" 3; + "~*(?:\b)CopyRightCheck(?:\b)" 3; + "~*(?:\b)Copyscape(?:\b)" 3; + "~*(?:\b)Cosmos(?:\b)" 3; + "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)crawler4j(?:\b)" 3; + "~*(?:\b)crawler.feedback(?:\b)" 3; + "~*(?:\b)crawl.sogou.com(?:\b)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b)" 3; + "~*(?:\b)Crescent(?:\b)" 3; + "~*(?:\b)CrunchBot(?:\b)" 3; + "~*(?:\b)CSHttp(?:\b)" 3; + "~*(?:\b)Curious(?:\b)" 3; + "~*(?:\b)Custo(?:\b)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; + "~*(?:\b)DataCha0s(?:\b)" 3; + "~*(?:\b)DBLBot(?:\b)" 3; + "~*(?:\b)demandbase-bot(?:\b)" 3; + "~*(?:\b)Demon(?:\b)" 3; + "~*(?:\b)Deusu(?:\b)" 3; + "~*(?:\b)Devil(?:\b)" 3; + "~*(?:\b)Digincore(?:\b)" 3; + "~*(?:\b)DigitalPebble(?:\b)" 3; + "~*(?:\b)DIIbot(?:\b)" 3; + "~*(?:\b)Dirbuster(?:\b)" 3; + "~*(?:\b)Disco(?:\b)" 3; + "~*(?:\b)Discobot(?:\b)" 3; + "~*(?:\b)Discoverybot(?:\b)" 3; + "~*(?:\b)Dispatch(?:\b)" 3; + "~*(?:\b)DittoSpyder(?:\b)" 3; + "~*(?:\b)DnyzBot(?:\b)" 3; + "~*(?:\b)DomainAppender(?:\b)" 3; + "~*(?:\b)DomainCrawler(?:\b)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)DomainStatsBot(?:\b)" 3; + "~*(?:\b)Dotbot(?:\b)" 3; + "~*(?:\b)Download\ Wonder(?:\b)" 3; + "~*(?:\b)Dragonfly(?:\b)" 3; + "~*(?:\b)Drip(?:\b)" 3; + "~*(?:\b)DSearch(?:\b)" 3; + "~*(?:\b)DTS\ Agent(?:\b)" 3; + "~*(?:\b)EasyDL(?:\b)" 3; + "~*(?:\b)Ebingbong(?:\b)" 3; + "~*(?:\b)eCatch(?:\b)" 3; + "~*(?:\b)ECCP/1.0(?:\b)" 3; + "~*(?:\b)Ecxi(?:\b)" 3; + "~*(?:\b)EirGrabber(?:\b)" 3; + "~*(?:\b)EMail\ Siphon(?:\b)" 3; + "~*(?:\b)EMail\ Wolf(?:\b)" 3; + "~*(?:\b)EroCrawler(?:\b)" 3; + "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)Evil(?:\b)" 3; + "~*(?:\b)Exabot(?:\b)" 3; + "~*(?:\b)Express\ WebPictures(?:\b)" 3; + "~*(?:\b)ExtLinksBot(?:\b)" 3; + "~*(?:\b)Extractor(?:\b)" 3; + "~*(?:\b)ExtractorPro(?:\b)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; + "~*(?:\b)EyeNetIE(?:\b)" 3; + "~*(?:\b)Ezooms(?:\b)" 3; + "~*(?:\b)facebookscraper(?:\b)" 3; + "~*(?:\b)FDM(?:\b)" 3; + "~*(?:\b)FemtosearchBot(?:\b)" 3; + "~*(?:\b)FHscan(?:\b)" 3; + "~*(?:\b)Fimap(?:\b)" 3; + "~*(?:\b)Firefox/7.0(?:\b)" 3; + "~*(?:\b)FlashGet(?:\b)" 3; + "~*(?:\b)Flunky(?:\b)" 3; + "~*(?:\b)Foobot(?:\b)" 3; + "~*(?:\b)Freeuploader(?:\b)" 3; + "~*(?:\b)FrontPage(?:\b)" 3; + "~*(?:\b)FyberSpider(?:\b)" 3; + "~*(?:\b)Fyrebot(?:\b)" 3; + "~*(?:\b)GalaxyBot(?:\b)" 3; + "~*(?:\b)Genieo(?:\b)" 3; + "~*(?:\b)GermCrawler(?:\b)" 3; + "~*(?:\b)Getintent(?:\b)" 3; + "~*(?:\b)GetRight(?:\b)" 3; + "~*(?:\b)GetWeb(?:\b)" 3; + "~*(?:\b)Gigablast(?:\b)" 3; + "~*(?:\b)Gigabot(?:\b)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; + "~*(?:\b)Gotit(?:\b)" 3; + "~*(?:\b)GoZilla(?:\b)" 3; + "~*(?:\b)Go!Zilla(?:\b)" 3; + "~*(?:\b)Grabber(?:\b)" 3; + "~*(?:\b)GrabNet(?:\b)" 3; + "~*(?:\b)Grafula(?:\b)" 3; + "~*(?:\b)GrapeFX(?:\b)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b)" 3; + "~*(?:\b)GridBot(?:\b)" 3; + "~*(?:\b)GT::WWW(?:\b)" 3; + "~*(?:\b)Haansoft(?:\b)" 3; + "~*(?:\b)HaosouSpider(?:\b)" 3; + "~*(?:\b)Harvest(?:\b)" 3; + "~*(?:\b)Havij(?:\b)" 3; + "~*(?:\b)HEADMasterSEO(?:\b)" 3; + "~*(?:\b)Heritrix(?:\b)" 3; + "~*(?:\b)Hloader(?:\b)" 3; + "~*(?:\b)HMView(?:\b)" 3; + "~*(?:\b)HTMLparser(?:\b)" 3; + "~*(?:\b)HTTP::Lite(?:\b)" 3; + "~*(?:\b)HTTrack(?:\b)" 3; + "~*(?:\b)Humanlinks(?:\b)" 3; + "~*(?:\b)HybridBot(?:\b)" 3; + "~*(?:\b)Iblog(?:\b)" 3; + "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)Id-search(?:\b)" 3; + "~*(?:\b)IlseBot(?:\b)" 3; + "~*(?:\b)Image\ Fetch(?:\b)" 3; + "~*(?:\b)Image\ Sucker(?:\b)" 3; + "~*(?:\b)IndeedBot(?:\b)" 3; + "~*(?:\b)Indy\ Library(?:\b)" 3; + "~*(?:\b)InfoNaviRobot(?:\b)" 3; + "~*(?:\b)InfoTekies(?:\b)" 3; + "~*(?:\b)instabid(?:\b)" 3; + "~*(?:\b)Intelliseek(?:\b)" 3; + "~*(?:\b)InterGET(?:\b)" 3; + "~*(?:\b)Internet\ Ninja(?:\b)" 3; + "~*(?:\b)InternetSeer(?:\b)" 3; + "~*(?:\b)internetVista\ monitor(?:\b)" 3; + "~*(?:\b)ips-agent(?:\b)" 3; + "~*(?:\b)Iria(?:\b)" 3; + "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)Iskanie(?:\b)" 3; + "~*(?:\b)IstellaBot(?:\b)" 3; + "~*(?:\b)JamesBOT(?:\b)" 3; + "~*(?:\b)Jbrofuzz(?:\b)" 3; + "~*(?:\b)JennyBot(?:\b)" 3; + "~*(?:\b)JetCar(?:\b)" 3; + "~*(?:\b)Jetty(?:\b)" 3; + "~*(?:\b)JikeSpider(?:\b)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; + "~*(?:\b)Joomla(?:\b)" 3; + "~*(?:\b)Jorgee(?:\b)" 3; + "~*(?:\b)JustView(?:\b)" 3; + "~*(?:\b)Jyxobot(?:\b)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b)" 3; + "~*(?:\b)Keyword\ Density(?:\b)" 3; + "~*(?:\b)Kozmosbot(?:\b)" 3; + "~*(?:\b)Lanshanbot(?:\b)" 3; + "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)LeechFTP(?:\b)" 3; + "~*(?:\b)LeechGet(?:\b)" 3; + "~*(?:\b)LexiBot(?:\b)" 3; + "~*(?:\b)Lftp(?:\b)" 3; + "~*(?:\b)LibWeb(?:\b)" 3; + "~*(?:\b)Libwhisker(?:\b)" 3; + "~*(?:\b)Lightspeedsystems(?:\b)" 3; + "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkdexbot(?:\b)" 3; + "~*(?:\b)LinkextractorPro(?:\b)" 3; + "~*(?:\b)LinkpadBot(?:\b)" 3; + "~*(?:\b)LinkScan(?:\b)" 3; + "~*(?:\b)LinksManager(?:\b)" 3; + "~*(?:\b)LinkWalker(?:\b)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; + "~*(?:\b)Lipperhey(?:\b)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; + "~*(?:\b)Litemage_walker(?:\b)" 3; + "~*(?:\b)Lmspider(?:\b)" 3; + "~*(?:\b)LNSpiderguy(?:\b)" 3; + "~*(?:\b)Ltx71(?:\b)" 3; + "~*(?:\b)lwp-request(?:\b)" 3; + "~*(?:\b)LWP::Simple(?:\b)" 3; + "~*(?:\b)lwp-trivial(?:\b)" 3; + "~*(?:\b)Magnet(?:\b)" 3; + "~*(?:\b)Mag-Net(?:\b)" 3; + "~*(?:\b)magpie-crawler(?:\b)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b)" 3; + "~*(?:\b)Majestic12(?:\b)" 3; + "~*(?:\b)Majestic-SEO(?:\b)" 3; + "~*(?:\b)Majestic\ SEO(?:\b)" 3; + "~*(?:\b)MarkMonitor(?:\b)" 3; + "~*(?:\b)MarkWatch(?:\b)" 3; + "~*(?:\b)Masscan(?:\b)" 3; + "~*(?:\b)Mass\ Downloader(?:\b)" 3; + "~*(?:\b)Mata\ Hari(?:\b)" 3; + "~*(?:\b)MauiBot(?:\b)" 3; + "~*(?:\b)meanpathbot(?:\b)" 3; + "~*(?:\b)Meanpathbot(?:\b)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b)" 3; + "~*(?:\b)mediawords(?:\b)" 3; + "~*(?:\b)MegaIndex.ru(?:\b)" 3; + "~*(?:\b)Metauri(?:\b)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; + "~*(?:\b)MIDown\ tool(?:\b)" 3; + "~*(?:\b)MIIxpc(?:\b)" 3; + "~*(?:\b)Mister\ PiX(?:\b)" 3; + "~*(?:\b)MJ12bot(?:\b)" 3; + "~*(?:\b)Mojeek(?:\b)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; + "~*(?:\b)Mr.4x3(?:\b)" 3; + "~*(?:\b)MSFrontPage(?:\b)" 3; + "~*(?:\b)MSIECrawler(?:\b)" 3; + "~*(?:\b)Msrabot(?:\b)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; + "~*(?:\b)muhstik-scan(?:\b)" 3; + "~*(?:\b)Musobot(?:\b)" 3; + "~*(?:\b)Name\ Intelligence(?:\b)" 3; + "~*(?:\b)Nameprotect(?:\b)" 3; + "~*(?:\b)Navroad(?:\b)" 3; + "~*(?:\b)NearSite(?:\b)" 3; + "~*(?:\b)Needle(?:\b)" 3; + "~*(?:\b)Nessus(?:\b)" 3; + "~*(?:\b)NetAnts(?:\b)" 3; + "~*(?:\b)Netcraft(?:\b)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; + "~*(?:\b)NetLyzer(?:\b)" 3; + "~*(?:\b)NetMechanic(?:\b)" 3; + "~*(?:\b)NetSpider(?:\b)" 3; + "~*(?:\b)Nettrack(?:\b)" 3; + "~*(?:\b)Net\ Vampire(?:\b)" 3; + "~*(?:\b)Netvibes(?:\b)" 3; + "~*(?:\b)NetZIP(?:\b)" 3; + "~*(?:\b)NextGenSearchBot(?:\b)" 3; + "~*(?:\b)Nibbler(?:\b)" 3; + "~*(?:\b)NICErsPRO(?:\b)" 3; + "~*(?:\b)Niki-bot(?:\b)" 3; + "~*(?:\b)Nikto(?:\b)" 3; + "~*(?:\b)NimbleCrawler(?:\b)" 3; + "~*(?:\b)Nimbostratus(?:\b)" 3; + "~*(?:\b)Ninja(?:\b)" 3; + "~*(?:\b)Nmap(?:\b)" 3; + "~*(?:\b)NPbot(?:\b)" 3; + "~*(?:\b)Nutch(?:\b)" 3; + "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)Octopus(?:\b)" 3; + "~*(?:\b)Offline\ Explorer(?:\b)" 3; + "~*(?:\b)Offline\ Navigator(?:\b)" 3; + "~*(?:\b)OnCrawl(?:\b)" 3; + "~*(?:\b)Openfind(?:\b)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b)" 3; + "~*(?:\b)Openvas(?:\b)" 3; + "~*(?:\b)OpenVAS(?:\b)" 3; + "~*(?:\b)OrangeBot(?:\b)" 3; + "~*(?:\b)OrangeSpider(?:\b)" 3; + "~*(?:\b)OutclicksBot(?:\b)" 3; + "~*(?:\b)OutfoxBot(?:\b)" 3; + "~*(?:\b)PageAnalyzer(?:\b)" 3; + "~*(?:\b)Page\ Analyzer(?:\b)" 3; + "~*(?:\b)PageGrabber(?:\b)" 3; + "~*(?:\b)page\ scorer(?:\b)" 3; + "~*(?:\b)PageScorer(?:\b)" 3; + "~*(?:\b)Pandalytics(?:\b)" 3; + "~*(?:\b)Panscient(?:\b)" 3; + "~*(?:\b)Papa\ Foto(?:\b)" 3; + "~*(?:\b)Pavuk(?:\b)" 3; + "~*(?:\b)pcBrowser(?:\b)" 3; + "~*(?:\b)PECL::HTTP(?:\b)" 3; + "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)PHPCrawl(?:\b)" 3; + "~*(?:\b)Picscout(?:\b)" 3; + "~*(?:\b)Picsearch(?:\b)" 3; + "~*(?:\b)PictureFinder(?:\b)" 3; + "~*(?:\b)Pimonster(?:\b)" 3; + "~*(?:\b)Pi-Monster(?:\b)" 3; + "~*(?:\b)Pixray(?:\b)" 3; + "~*(?:\b)PleaseCrawl(?:\b)" 3; + "~*(?:\b)plumanalytics(?:\b)" 3; + "~*(?:\b)Pockey(?:\b)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; + "~*(?:\b)Probethenet(?:\b)" 3; + "~*(?:\b)ProPowerBot(?:\b)" 3; + "~*(?:\b)ProWebWalker(?:\b)" 3; + "~*(?:\b)Psbot(?:\b)" 3; + "~*(?:\b)Pump(?:\b)" 3; + "~*(?:\b)PxBroker(?:\b)" 3; + "~*(?:\b)PyCurl(?:\b)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; + "~*(?:\b)Quick-Crawler(?:\b)" 3; + "~*(?:\b)RankActive(?:\b)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b)" 3; + "~*(?:\b)RankFlex(?:\b)" 3; + "~*(?:\b)RankingBot(?:\b)" 3; + "~*(?:\b)RankingBot2(?:\b)" 3; + "~*(?:\b)Rankivabot(?:\b)" 3; + "~*(?:\b)RankurBot(?:\b)" 3; + "~*(?:\b)RealDownload(?:\b)" 3; + "~*(?:\b)Reaper(?:\b)" 3; + "~*(?:\b)RebelMouse(?:\b)" 3; + "~*(?:\b)Recorder(?:\b)" 3; + "~*(?:\b)RedesScrapy(?:\b)" 3; + "~*(?:\b)ReGet(?:\b)" 3; + "~*(?:\b)RepoMonkey(?:\b)" 3; + "~*(?:\b)Ripper(?:\b)" 3; + "~*(?:\b)RocketCrawler(?:\b)" 3; + "~*(?:\b)Rogerbot(?:\b)" 3; + "~*(?:\b)s1z.ru(?:\b)" 3; + "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)SBIder(?:\b)" 3; + "~*(?:\b)ScanAlert(?:\b)" 3; + "~*(?:\b)Scanbot(?:\b)" 3; + "~*(?:\b)scan.lol(?:\b)" 3; + "~*(?:\b)ScoutJet(?:\b)" 3; + "~*(?:\b)Scrapy(?:\b)" 3; + "~*(?:\b)Screaming(?:\b)" 3; + "~*(?:\b)ScreenerBot(?:\b)" 3; + "~*(?:\b)Searchestate(?:\b)" 3; + "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Semrush(?:\b)" 3; + "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SEOkicks(?:\b)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; + "~*(?:\b)Seomoz(?:\b)" 3; + "~*(?:\b)SEOprofiler(?:\b)" 3; + "~*(?:\b)seoscanners(?:\b)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b)" 3; + "~*(?:\b)SEOstats(?:\b)" 3; + "~*(?:\b)serpstatbot(?:\b)" 3; + "~*(?:\b)sexsearcher(?:\b)" 3; + "~*(?:\b)Shodan(?:\b)" 3; + "~*(?:\b)Siphon(?:\b)" 3; + "~*(?:\b)SISTRIX(?:\b)" 3; + "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteExplorer(?:\b)" 3; + "~*(?:\b)Siteimprove(?:\b)" 3; + "~*(?:\b)SiteLockSpider(?:\b)" 3; + "~*(?:\b)SiteSnagger(?:\b)" 3; + "~*(?:\b)SiteSucker(?:\b)" 3; + "~*(?:\b)Site\ Sucker(?:\b)" 3; + "~*(?:\b)Sitevigil(?:\b)" 3; + "~*(?:\b)SlySearch(?:\b)" 3; + "~*(?:\b)SmartDownload(?:\b)" 3; + "~*(?:\b)SMTBot(?:\b)" 3; + "~*(?:\b)Snake(?:\b)" 3; + "~*(?:\b)Snapbot(?:\b)" 3; + "~*(?:\b)Snoopy(?:\b)" 3; + "~*(?:\b)SocialRankIOBot(?:\b)" 3; + "~*(?:\b)Sociscraper(?:\b)" 3; + "~*(?:\b)sogouspider(?:\b)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; + "~*(?:\b)Sosospider(?:\b)" 3; + "~*(?:\b)Sottopop(?:\b)" 3; + "~*(?:\b)SpaceBison(?:\b)" 3; + "~*(?:\b)Spammen(?:\b)" 3; + "~*(?:\b)SpankBot(?:\b)" 3; + "~*(?:\b)Spanner(?:\b)" 3; + "~*(?:\b)sp_auditbot(?:\b)" 3; + "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spinn3r(?:\b)" 3; + "~*(?:\b)SputnikBot(?:\b)" 3; + "~*(?:\b)spyfu(?:\b)" 3; + "~*(?:\b)Sqlmap(?:\b)" 3; + "~*(?:\b)Sqlworm(?:\b)" 3; + "~*(?:\b)Sqworm(?:\b)" 3; + "~*(?:\b)Steeler(?:\b)" 3; + "~*(?:\b)Stripper(?:\b)" 3; + "~*(?:\b)Sucker(?:\b)" 3; + "~*(?:\b)Sucuri(?:\b)" 3; + "~*(?:\b)SuperBot(?:\b)" 3; + "~*(?:\b)SuperHTTP(?:\b)" 3; + "~*(?:\b)Surfbot(?:\b)" 3; + "~*(?:\b)SurveyBot(?:\b)" 3; + "~*(?:\b)Suzuran(?:\b)" 3; + "~*(?:\b)Swiftbot(?:\b)" 3; + "~*(?:\b)sysscan(?:\b)" 3; + "~*(?:\b)Szukacz(?:\b)" 3; + "~*(?:\b)T0PHackTeam(?:\b)" 3; + "~*(?:\b)T8Abot(?:\b)" 3; + "~*(?:\b)tAkeOut(?:\b)" 3; + "~*(?:\b)Teleport(?:\b)" 3; + "~*(?:\b)TeleportPro(?:\b)" 3; + "~*(?:\b)Telesoft(?:\b)" 3; + "~*(?:\b)Telesphoreo(?:\b)" 3; + "~*(?:\b)Telesphorep(?:\b)" 3; + "~*(?:\b)The\ Intraformant(?:\b)" 3; + "~*(?:\b)TheNomad(?:\b)" 3; + "~*(?:\b)Thumbor(?:\b)" 3; + "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)Titan(?:\b)" 3; + "~*(?:\b)Toata(?:\b)" 3; + "~*(?:\b)Toweyabot(?:\b)" 3; + "~*(?:\b)Tracemyfile(?:\b)" 3; + "~*(?:\b)Trendiction(?:\b)" 3; + "~*(?:\b)Trendictionbot(?:\b)" 3; + "~*(?:\b)trendiction.com(?:\b)" 3; + "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)True_Robot(?:\b)" 3; + "~*(?:\b)Turingos(?:\b)" 3; + "~*(?:\b)Turnitin(?:\b)" 3; + "~*(?:\b)TurnitinBot(?:\b)" 3; + "~*(?:\b)TwengaBot(?:\b)" 3; + "~*(?:\b)Twice(?:\b)" 3; + "~*(?:\b)Typhoeus(?:\b)" 3; + "~*(?:\b)UnisterBot(?:\b)" 3; + "~*(?:\b)Upflow(?:\b)" 3; + "~*(?:\b)URLy.Warning(?:\b)" 3; + "~*(?:\b)URLy\ Warning(?:\b)" 3; + "~*(?:\b)Vacuum(?:\b)" 3; + "~*(?:\b)Vagabondo(?:\b)" 3; + "~*(?:\b)VB\ Project(?:\b)" 3; + "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b)" 3; + "~*(?:\b)VidibleScraper(?:\b)" 3; + "~*(?:\b)Virusdie(?:\b)" 3; + "~*(?:\b)VoidEYE(?:\b)" 3; + "~*(?:\b)Voil(?:\b)" 3; + "~*(?:\b)Voltron(?:\b)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b)" 3; + "~*(?:\b)WallpapersHD(?:\b)" 3; + "~*(?:\b)WASALive-Bot(?:\b)" 3; + "~*(?:\b)WBSearchBot(?:\b)" 3; + "~*(?:\b)Webalta(?:\b)" 3; + "~*(?:\b)WebAuto(?:\b)" 3; + "~*(?:\b)Web\ Auto(?:\b)" 3; + "~*(?:\b)WebBandit(?:\b)" 3; + "~*(?:\b)WebCollage(?:\b)" 3; + "~*(?:\b)Web\ Collage(?:\b)" 3; + "~*(?:\b)WebCopier(?:\b)" 3; + "~*(?:\b)WEBDAV(?:\b)" 3; + "~*(?:\b)WebEnhancer(?:\b)" 3; + "~*(?:\b)Web\ Enhancer(?:\b)" 3; + "~*(?:\b)WebFetch(?:\b)" 3; + "~*(?:\b)Web\ Fetch(?:\b)" 3; + "~*(?:\b)WebFuck(?:\b)" 3; + "~*(?:\b)Web\ Fuck(?:\b)" 3; + "~*(?:\b)WebGo\ IS(?:\b)" 3; + "~*(?:\b)WebImageCollector(?:\b)" 3; + "~*(?:\b)WebLeacher(?:\b)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; + "~*(?:\b)webmeup-crawler(?:\b)" 3; + "~*(?:\b)WebPix(?:\b)" 3; + "~*(?:\b)Web\ Pix(?:\b)" 3; + "~*(?:\b)WebReaper(?:\b)" 3; + "~*(?:\b)WebSauger(?:\b)" 3; + "~*(?:\b)Web\ Sauger(?:\b)" 3; + "~*(?:\b)Webshag(?:\b)" 3; + "~*(?:\b)WebsiteExtractor(?:\b)" 3; + "~*(?:\b)WebsiteQuester(?:\b)" 3; + "~*(?:\b)Website\ Quester(?:\b)" 3; + "~*(?:\b)Webster(?:\b)" 3; + "~*(?:\b)WebStripper(?:\b)" 3; + "~*(?:\b)WebSucker(?:\b)" 3; + "~*(?:\b)Web\ Sucker(?:\b)" 3; + "~*(?:\b)WebWhacker(?:\b)" 3; + "~*(?:\b)WebZIP(?:\b)" 3; + "~*(?:\b)WeSEE(?:\b)" 3; + "~*(?:\b)Whack(?:\b)" 3; + "~*(?:\b)Whacker(?:\b)" 3; + "~*(?:\b)Whatweb(?:\b)" 3; + "~*(?:\b)Who.is\ Bot(?:\b)" 3; + "~*(?:\b)Widow(?:\b)" 3; + "~*(?:\b)WinHTTrack(?:\b)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; + "~*(?:\b)WISENutbot(?:\b)" 3; + "~*(?:\b)Wonderbot(?:\b)" 3; + "~*(?:\b)Woobot(?:\b)" 3; + "~*(?:\b)Wotbox(?:\b)" 3; + "~*(?:\b)Wprecon(?:\b)" 3; + "~*(?:\b)WPScan(?:\b)" 3; + "~*(?:\b)WWW-Collector-E(?:\b)" 3; + "~*(?:\b)WWW-Mechanize(?:\b)" 3; + "~*(?:\b)WWW::Mechanize(?:\b)" 3; + "~*(?:\b)WWWOFFLE(?:\b)" 3; + "~*(?:\b)x09Mozilla(?:\b)" 3; + "~*(?:\b)x22Mozilla(?:\b)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; + "~*(?:\b)Xenu(?:\b)" 3; + "~*(?:\b)xpymep1.exe(?:\b)" 3; + "~*(?:\b)YoudaoBot(?:\b)" 3; + "~*(?:\b)Zade(?:\b)" 3; + "~*(?:\b)Zauba(?:\b)" 3; + "~*(?:\b)zauba.io(?:\b)" 3; + "~*(?:\b)Zermelo(?:\b)" 3; + "~*(?:\b)Zeus(?:\b)" 3; + "~*(?:\b)zgrab(?:\b)" 3; + "~*(?:\b)Zitebot(?:\b)" 3; + "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZumBot(?:\b)" 3; + "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------- @@ -717,38 +717,38 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)adidxbot(?:\b|)" 0; - "~*(?:\b)AdsBot-Google(?:\b|)" 0; - "~*(?:\b)aolbuild(?:\b|)" 0; - "~*(?:\b)bingbot(?:\b|)" 0; - "~*(?:\b)bingpreview(?:\b|)" 0; - "~*(?:\b)DoCoMo(?:\b|)" 0; - "~*(?:\b)duckduckgo(?:\b|)" 0; - "~*(?:\b)facebookexternalhit(?:\b|)" 0; - "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; - "~*(?:\b)Googlebot(?:\b|)" 0; - "~*(?:\b)Googlebot-Image(?:\b|)" 0; - "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; - "~*(?:\b)Googlebot-News(?:\b|)" 0; - "~*(?:\b)Googlebot/Test(?:\b|)" 0; - "~*(?:\b)Googlebot-Video(?:\b|)" 0; - "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; - "~*(?:\b)Gravityscan(?:\b|)" 0; - "~*(?:\b)gsa-crawler(?:\b|)" 0; - "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; - "~*(?:\b)Kraken/0.1(?:\b|)" 0; - "~*(?:\b)LinkedInBot(?:\b|)" 0; - "~*(?:\b)Mediapartners-Google(?:\b|)" 0; - "~*(?:\b)msnbot(?:\b|)" 0; - "~*(?:\b)msnbot-media(?:\b|)" 0; - "~*(?:\b)SAMSUNG(?:\b|)" 0; - "~*(?:\b)Slackbot(?:\b|)" 0; - "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; - "~*(?:\b)slurp(?:\b|)" 0; - "~*(?:\b)teoma(?:\b|)" 0; - "~*(?:\b)TwitterBot(?:\b|)" 0; - "~*(?:\b)Wordpress(?:\b|)" 0; - "~*(?:\b)yahoo(?:\b|)" 0; + "~*(?:\b)adidxbot(?:\b)" 0; + "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bingbot(?:\b)" 0; + "~*(?:\b)bingpreview(?:\b)" 0; + "~*(?:\b)DoCoMo(?:\b)" 0; + "~*(?:\b)duckduckgo(?:\b)" 0; + "~*(?:\b)facebookexternalhit(?:\b)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b)" 0; + "~*(?:\b)Googlebot(?:\b)" 0; + "~*(?:\b)Googlebot-Image(?:\b)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b)" 0; + "~*(?:\b)Googlebot-News(?:\b)" 0; + "~*(?:\b)Googlebot/Test(?:\b)" 0; + "~*(?:\b)Googlebot-Video(?:\b)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; + "~*(?:\b)Gravityscan(?:\b)" 0; + "~*(?:\b)gsa-crawler(?:\b)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b)" 0; + "~*(?:\b)Kraken/0.1(?:\b)" 0; + "~*(?:\b)LinkedInBot(?:\b)" 0; + "~*(?:\b)Mediapartners-Google(?:\b)" 0; + "~*(?:\b)msnbot(?:\b)" 0; + "~*(?:\b)msnbot-media(?:\b)" 0; + "~*(?:\b)SAMSUNG(?:\b)" 0; + "~*(?:\b)Slackbot(?:\b)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 0; + "~*(?:\b)slurp(?:\b)" 0; + "~*(?:\b)teoma(?:\b)" 0; + "~*(?:\b)TwitterBot(?:\b)" 0; + "~*(?:\b)Wordpress(?:\b)" 0; + "~*(?:\b)yahoo(?:\b)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------------------- @@ -760,12 +760,12 @@ map $http_user_agent $bad_bot { #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)jetmon(?:\b|)" 1; - "~*(?:\b)libwww-perl(?:\b|)" 1; - "~*(?:\b)Lynx(?:\b|)" 1; - "~*(?:\b)munin(?:\b|)" 1; - "~*(?:\b)Presto(?:\b|)" 1; - "~*(?:\b)Wget/1.15(?:\b|)" 1; + "~*(?:\b)jetmon(?:\b)" 1; + "~*(?:\b)libwww-perl(?:\b)" 1; + "~*(?:\b)Lynx(?:\b)" 1; + "~*(?:\b)munin(?:\b)" 1; + "~*(?:\b)Presto(?:\b)" 1; + "~*(?:\b)Wget/1.15(?:\b)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ------------------------------------------------------- @@ -778,23 +778,23 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)Alexa(?:\b|)" 2; - "~*(?:\b)archive.org(?:\b|)" 2; - "~*(?:\b)Baidu(?:\b|)" 2; - "~*(?:\b)BUbiNG(?:\b|)" 2; - "~*(?:\b)FlipboardProxy(?:\b|)" 2; - "~*(?:\b)ia_archiver(?:\b|)" 2; - "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; - "~*(?:\b)Proximic(?:\b|)" 2; - "~*(?:\b)R6_CommentReader(?:\b|)" 2; - "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; - "~*(?:\b)RED/1(?:\b|)" 2; - "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; - "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; - "~*(?:\b)Spaidu(?:\b|)" 2; - "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; - "~*(?:\b)YandexBot(?:\b|)" 2; - "~*(?:\b)YandexImages(?:\b|)" 2; + "~*(?:\b)Alexa(?:\b)" 2; + "~*(?:\b)archive.org(?:\b)" 2; + "~*(?:\b)Baidu(?:\b)" 2; + "~*(?:\b)BUbiNG(?:\b)" 2; + "~*(?:\b)FlipboardProxy(?:\b)" 2; + "~*(?:\b)ia_archiver(?:\b)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b)" 2; + "~*(?:\b)Proximic(?:\b)" 2; + "~*(?:\b)R6_CommentReader(?:\b)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b)" 2; + "~*(?:\b)RED/1(?:\b)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b)" 2; + "~*(?:\b)Spaidu(?:\b)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b)" 2; + "~*(?:\b)YandexBot(?:\b)" 2; + "~*(?:\b)YandexImages(?:\b)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index b9d24324a..0f27f0dd9 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 1; +104.154.113.151 1; 127.0.0.1 1; diff --git a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index ea117da9f..0e0848852 100644 --- a/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/.dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -34.68.144.114 0; +104.154.113.151 0; 127.0.0.1 0; diff --git a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index f26e52add..a1999739a 100644 --- a/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/.dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1709 -### Updated: Mon Jul 1 10:37:04 SAST 2019 +### Version: V4.2019.07.1712 +### Updated: Mon Jul 1 11:24:52 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### @@ -147,562 +147,562 @@ map $http_user_agent $bad_bot { # -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b|)" 3; - "~*(?:\b)404checker(?:\b|)" 3; - "~*(?:\b)404enemy(?:\b|)" 3; - "~*(?:\b)80legs(?:\b|)" 3; - "~*(?:\b)Abonti(?:\b|)" 3; - "~*(?:\b)Aboundex(?:\b|)" 3; - "~*(?:\b)Aboundexbot(?:\b|)" 3; - "~*(?:\b)Acunetix(?:\b|)" 3; - "~*(?:\b)ADmantX(?:\b|)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; - "~*(?:\b)AhrefsBot(?:\b|)" 3; - "~*(?:\b)AIBOT(?:\b|)" 3; - "~*(?:\b)AiHitBot(?:\b|)" 3; - "~*(?:\b)Aipbot(?:\b|)" 3; - "~*(?:\b)Alexibot(?:\b|)" 3; - "~*(?:\b)Alligator(?:\b|)" 3; - "~*(?:\b)AllSubmitter(?:\b|)" 3; - "~*(?:\b)AlphaBot(?:\b|)" 3; - "~*(?:\b)Anarchie(?:\b|)" 3; - "~*(?:\b)Apexoo(?:\b|)" 3; - "~*(?:\b)archive.org_bot(?:\b|)" 3; - "~*(?:\b)ASPSeek(?:\b|)" 3; - "~*(?:\b)Asterias(?:\b|)" 3; - "~*(?:\b)Attach(?:\b|)" 3; - "~*(?:\b)autoemailspider(?:\b|)" 3; - "~*(?:\b)BackDoorBot(?:\b|)" 3; - "~*(?:\b)Backlink-Ceck(?:\b|)" 3; - "~*(?:\b)backlink-check(?:\b|)" 3; - "~*(?:\b)BacklinkCrawler(?:\b|)" 3; - "~*(?:\b)BackStreet(?:\b|)" 3; - "~*(?:\b)BackWeb(?:\b|)" 3; - "~*(?:\b)Badass(?:\b|)" 3; - "~*(?:\b)Bandit(?:\b|)" 3; - "~*(?:\b)Barkrowler(?:\b|)" 3; - "~*(?:\b)BatchFTP(?:\b|)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; - "~*(?:\b)BBBike(?:\b|)" 3; - "~*(?:\b)BDCbot(?:\b|)" 3; - "~*(?:\b)BDFetch(?:\b|)" 3; - "~*(?:\b)BetaBot(?:\b|)" 3; - "~*(?:\b)Bigfoot(?:\b|)" 3; - "~*(?:\b)Bitacle(?:\b|)" 3; - "~*(?:\b)Blackboard(?:\b|)" 3; - "~*(?:\b)Black\ Hole(?:\b|)" 3; - "~*(?:\b)BlackWidow(?:\b|)" 3; - "~*(?:\b)BLEXBot(?:\b|)" 3; - "~*(?:\b)Blow(?:\b|)" 3; - "~*(?:\b)BlowFish(?:\b|)" 3; - "~*(?:\b)Boardreader(?:\b|)" 3; - "~*(?:\b)Bolt(?:\b|)" 3; - "~*(?:\b)BotALot(?:\b|)" 3; - "~*(?:\b)Brandprotect(?:\b|)" 3; - "~*(?:\b)Brandwatch(?:\b|)" 3; - "~*(?:\b)Buddy(?:\b|)" 3; - "~*(?:\b)BuiltBotTough(?:\b|)" 3; - "~*(?:\b)BuiltWith(?:\b|)" 3; - "~*(?:\b)Bullseye(?:\b|)" 3; - "~*(?:\b)BunnySlippers(?:\b|)" 3; - "~*(?:\b)BuzzSumo(?:\b|)" 3; - "~*(?:\b)Calculon(?:\b|)" 3; - "~*(?:\b)CATExplorador(?:\b|)" 3; - "~*(?:\b)CazoodleBot(?:\b|)" 3; - "~*(?:\b)CCBot(?:\b|)" 3; - "~*(?:\b)Cegbfeieh(?:\b|)" 3; - "~*(?:\b)CheeseBot(?:\b|)" 3; - "~*(?:\b)CherryPicker(?:\b|)" 3; - "~*(?:\b)CheTeam(?:\b|)" 3; - "~*(?:\b)ChinaClaw(?:\b|)" 3; - "~*(?:\b)Chlooe(?:\b|)" 3; - "~*(?:\b)Claritybot(?:\b|)" 3; - "~*(?:\b)Cliqzbot(?:\b|)" 3; - "~*(?:\b)Cloud\ mapping(?:\b|)" 3; - "~*(?:\b)coccocbot-web(?:\b|)" 3; - "~*(?:\b)Cogentbot(?:\b|)" 3; - "~*(?:\b)cognitiveseo(?:\b|)" 3; - "~*(?:\b)Collector(?:\b|)" 3; - "~*(?:\b)com.plumanalytics(?:\b|)" 3; - "~*(?:\b)Copier(?:\b|)" 3; - "~*(?:\b)CopyRightCheck(?:\b|)" 3; - "~*(?:\b)Copyscape(?:\b|)" 3; - "~*(?:\b)Cosmos(?:\b|)" 3; - "~*(?:\b)Craftbot(?:\b|)" 3; - "~*(?:\b)crawler4j(?:\b|)" 3; - "~*(?:\b)crawler.feedback(?:\b|)" 3; - "~*(?:\b)crawl.sogou.com(?:\b|)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; - "~*(?:\b)Crescent(?:\b|)" 3; - "~*(?:\b)CrunchBot(?:\b|)" 3; - "~*(?:\b)CSHttp(?:\b|)" 3; - "~*(?:\b)Curious(?:\b|)" 3; - "~*(?:\b)Custo(?:\b|)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; - "~*(?:\b)DataCha0s(?:\b|)" 3; - "~*(?:\b)DBLBot(?:\b|)" 3; - "~*(?:\b)demandbase-bot(?:\b|)" 3; - "~*(?:\b)Demon(?:\b|)" 3; - "~*(?:\b)Deusu(?:\b|)" 3; - "~*(?:\b)Devil(?:\b|)" 3; - "~*(?:\b)Digincore(?:\b|)" 3; - "~*(?:\b)DigitalPebble(?:\b|)" 3; - "~*(?:\b)DIIbot(?:\b|)" 3; - "~*(?:\b)Dirbuster(?:\b|)" 3; - "~*(?:\b)Disco(?:\b|)" 3; - "~*(?:\b)Discobot(?:\b|)" 3; - "~*(?:\b)Discoverybot(?:\b|)" 3; - "~*(?:\b)Dispatch(?:\b|)" 3; - "~*(?:\b)DittoSpyder(?:\b|)" 3; - "~*(?:\b)DnyzBot(?:\b|)" 3; - "~*(?:\b)DomainAppender(?:\b|)" 3; - "~*(?:\b)DomainCrawler(?:\b|)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; - "~*(?:\b)DomainStatsBot(?:\b|)" 3; - "~*(?:\b)Dotbot(?:\b|)" 3; - "~*(?:\b)Download\ Wonder(?:\b|)" 3; - "~*(?:\b)Dragonfly(?:\b|)" 3; - "~*(?:\b)Drip(?:\b|)" 3; - "~*(?:\b)DSearch(?:\b|)" 3; - "~*(?:\b)DTS\ Agent(?:\b|)" 3; - "~*(?:\b)EasyDL(?:\b|)" 3; - "~*(?:\b)Ebingbong(?:\b|)" 3; - "~*(?:\b)eCatch(?:\b|)" 3; - "~*(?:\b)ECCP/1.0(?:\b|)" 3; - "~*(?:\b)Ecxi(?:\b|)" 3; - "~*(?:\b)EirGrabber(?:\b|)" 3; - "~*(?:\b)EMail\ Siphon(?:\b|)" 3; - "~*(?:\b)EMail\ Wolf(?:\b|)" 3; - "~*(?:\b)EroCrawler(?:\b|)" 3; - "~*(?:\b)evc-batch(?:\b|)" 3; - "~*(?:\b)Evil(?:\b|)" 3; - "~*(?:\b)Exabot(?:\b|)" 3; - "~*(?:\b)Express\ WebPictures(?:\b|)" 3; - "~*(?:\b)ExtLinksBot(?:\b|)" 3; - "~*(?:\b)Extractor(?:\b|)" 3; - "~*(?:\b)ExtractorPro(?:\b|)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; - "~*(?:\b)EyeNetIE(?:\b|)" 3; - "~*(?:\b)Ezooms(?:\b|)" 3; - "~*(?:\b)facebookscraper(?:\b|)" 3; - "~*(?:\b)FDM(?:\b|)" 3; - "~*(?:\b)FemtosearchBot(?:\b|)" 3; - "~*(?:\b)FHscan(?:\b|)" 3; - "~*(?:\b)Fimap(?:\b|)" 3; - "~*(?:\b)Firefox/7.0(?:\b|)" 3; - "~*(?:\b)FlashGet(?:\b|)" 3; - "~*(?:\b)Flunky(?:\b|)" 3; - "~*(?:\b)Foobot(?:\b|)" 3; - "~*(?:\b)Freeuploader(?:\b|)" 3; - "~*(?:\b)FrontPage(?:\b|)" 3; - "~*(?:\b)FyberSpider(?:\b|)" 3; - "~*(?:\b)Fyrebot(?:\b|)" 3; - "~*(?:\b)GalaxyBot(?:\b|)" 3; - "~*(?:\b)Genieo(?:\b|)" 3; - "~*(?:\b)GermCrawler(?:\b|)" 3; - "~*(?:\b)Getintent(?:\b|)" 3; - "~*(?:\b)GetRight(?:\b|)" 3; - "~*(?:\b)GetWeb(?:\b|)" 3; - "~*(?:\b)Gigablast(?:\b|)" 3; - "~*(?:\b)Gigabot(?:\b|)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; - "~*(?:\b)Gotit(?:\b|)" 3; - "~*(?:\b)GoZilla(?:\b|)" 3; - "~*(?:\b)Go!Zilla(?:\b|)" 3; - "~*(?:\b)Grabber(?:\b|)" 3; - "~*(?:\b)GrabNet(?:\b|)" 3; - "~*(?:\b)Grafula(?:\b|)" 3; - "~*(?:\b)GrapeFX(?:\b|)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; - "~*(?:\b)GridBot(?:\b|)" 3; - "~*(?:\b)GT::WWW(?:\b|)" 3; - "~*(?:\b)Haansoft(?:\b|)" 3; - "~*(?:\b)HaosouSpider(?:\b|)" 3; - "~*(?:\b)Harvest(?:\b|)" 3; - "~*(?:\b)Havij(?:\b|)" 3; - "~*(?:\b)HEADMasterSEO(?:\b|)" 3; - "~*(?:\b)Heritrix(?:\b|)" 3; - "~*(?:\b)Hloader(?:\b|)" 3; - "~*(?:\b)HMView(?:\b|)" 3; - "~*(?:\b)HTMLparser(?:\b|)" 3; - "~*(?:\b)HTTP::Lite(?:\b|)" 3; - "~*(?:\b)HTTrack(?:\b|)" 3; - "~*(?:\b)Humanlinks(?:\b|)" 3; - "~*(?:\b)HybridBot(?:\b|)" 3; - "~*(?:\b)Iblog(?:\b|)" 3; - "~*(?:\b)IDBot(?:\b|)" 3; - "~*(?:\b)Id-search(?:\b|)" 3; - "~*(?:\b)IlseBot(?:\b|)" 3; - "~*(?:\b)Image\ Fetch(?:\b|)" 3; - "~*(?:\b)Image\ Sucker(?:\b|)" 3; - "~*(?:\b)IndeedBot(?:\b|)" 3; - "~*(?:\b)Indy\ Library(?:\b|)" 3; - "~*(?:\b)InfoNaviRobot(?:\b|)" 3; - "~*(?:\b)InfoTekies(?:\b|)" 3; - "~*(?:\b)instabid(?:\b|)" 3; - "~*(?:\b)Intelliseek(?:\b|)" 3; - "~*(?:\b)InterGET(?:\b|)" 3; - "~*(?:\b)Internet\ Ninja(?:\b|)" 3; - "~*(?:\b)InternetSeer(?:\b|)" 3; - "~*(?:\b)internetVista\ monitor(?:\b|)" 3; - "~*(?:\b)ips-agent(?:\b|)" 3; - "~*(?:\b)Iria(?:\b|)" 3; - "~*(?:\b)IRLbot(?:\b|)" 3; - "~*(?:\b)Iskanie(?:\b|)" 3; - "~*(?:\b)IstellaBot(?:\b|)" 3; - "~*(?:\b)JamesBOT(?:\b|)" 3; - "~*(?:\b)Jbrofuzz(?:\b|)" 3; - "~*(?:\b)JennyBot(?:\b|)" 3; - "~*(?:\b)JetCar(?:\b|)" 3; - "~*(?:\b)Jetty(?:\b|)" 3; - "~*(?:\b)JikeSpider(?:\b|)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; - "~*(?:\b)Joomla(?:\b|)" 3; - "~*(?:\b)Jorgee(?:\b|)" 3; - "~*(?:\b)JustView(?:\b|)" 3; - "~*(?:\b)Jyxobot(?:\b|)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; - "~*(?:\b)Keyword\ Density(?:\b|)" 3; - "~*(?:\b)Kozmosbot(?:\b|)" 3; - "~*(?:\b)Lanshanbot(?:\b|)" 3; - "~*(?:\b)Larbin(?:\b|)" 3; - "~*(?:\b)LeechFTP(?:\b|)" 3; - "~*(?:\b)LeechGet(?:\b|)" 3; - "~*(?:\b)LexiBot(?:\b|)" 3; - "~*(?:\b)Lftp(?:\b|)" 3; - "~*(?:\b)LibWeb(?:\b|)" 3; - "~*(?:\b)Libwhisker(?:\b|)" 3; - "~*(?:\b)Lightspeedsystems(?:\b|)" 3; - "~*(?:\b)Likse(?:\b|)" 3; - "~*(?:\b)Linkdexbot(?:\b|)" 3; - "~*(?:\b)LinkextractorPro(?:\b|)" 3; - "~*(?:\b)LinkpadBot(?:\b|)" 3; - "~*(?:\b)LinkScan(?:\b|)" 3; - "~*(?:\b)LinksManager(?:\b|)" 3; - "~*(?:\b)LinkWalker(?:\b|)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; - "~*(?:\b)Lipperhey(?:\b|)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; - "~*(?:\b)Litemage_walker(?:\b|)" 3; - "~*(?:\b)Lmspider(?:\b|)" 3; - "~*(?:\b)LNSpiderguy(?:\b|)" 3; - "~*(?:\b)Ltx71(?:\b|)" 3; - "~*(?:\b)lwp-request(?:\b|)" 3; - "~*(?:\b)LWP::Simple(?:\b|)" 3; - "~*(?:\b)lwp-trivial(?:\b|)" 3; - "~*(?:\b)Magnet(?:\b|)" 3; - "~*(?:\b)Mag-Net(?:\b|)" 3; - "~*(?:\b)magpie-crawler(?:\b|)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; - "~*(?:\b)Majestic12(?:\b|)" 3; - "~*(?:\b)Majestic-SEO(?:\b|)" 3; - "~*(?:\b)Majestic\ SEO(?:\b|)" 3; - "~*(?:\b)MarkMonitor(?:\b|)" 3; - "~*(?:\b)MarkWatch(?:\b|)" 3; - "~*(?:\b)Masscan(?:\b|)" 3; - "~*(?:\b)Mass\ Downloader(?:\b|)" 3; - "~*(?:\b)Mata\ Hari(?:\b|)" 3; - "~*(?:\b)MauiBot(?:\b|)" 3; - "~*(?:\b)meanpathbot(?:\b|)" 3; - "~*(?:\b)Meanpathbot(?:\b|)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; - "~*(?:\b)mediawords(?:\b|)" 3; - "~*(?:\b)MegaIndex.ru(?:\b|)" 3; - "~*(?:\b)Metauri(?:\b|)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; - "~*(?:\b)MIDown\ tool(?:\b|)" 3; - "~*(?:\b)MIIxpc(?:\b|)" 3; - "~*(?:\b)Mister\ PiX(?:\b|)" 3; - "~*(?:\b)MJ12bot(?:\b|)" 3; - "~*(?:\b)Mojeek(?:\b|)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; - "~*(?:\b)Mr.4x3(?:\b|)" 3; - "~*(?:\b)MSFrontPage(?:\b|)" 3; - "~*(?:\b)MSIECrawler(?:\b|)" 3; - "~*(?:\b)Msrabot(?:\b|)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; - "~*(?:\b)muhstik-scan(?:\b|)" 3; - "~*(?:\b)Musobot(?:\b|)" 3; - "~*(?:\b)Name\ Intelligence(?:\b|)" 3; - "~*(?:\b)Nameprotect(?:\b|)" 3; - "~*(?:\b)Navroad(?:\b|)" 3; - "~*(?:\b)NearSite(?:\b|)" 3; - "~*(?:\b)Needle(?:\b|)" 3; - "~*(?:\b)Nessus(?:\b|)" 3; - "~*(?:\b)NetAnts(?:\b|)" 3; - "~*(?:\b)Netcraft(?:\b|)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; - "~*(?:\b)NetLyzer(?:\b|)" 3; - "~*(?:\b)NetMechanic(?:\b|)" 3; - "~*(?:\b)NetSpider(?:\b|)" 3; - "~*(?:\b)Nettrack(?:\b|)" 3; - "~*(?:\b)Net\ Vampire(?:\b|)" 3; - "~*(?:\b)Netvibes(?:\b|)" 3; - "~*(?:\b)NetZIP(?:\b|)" 3; - "~*(?:\b)NextGenSearchBot(?:\b|)" 3; - "~*(?:\b)Nibbler(?:\b|)" 3; - "~*(?:\b)NICErsPRO(?:\b|)" 3; - "~*(?:\b)Niki-bot(?:\b|)" 3; - "~*(?:\b)Nikto(?:\b|)" 3; - "~*(?:\b)NimbleCrawler(?:\b|)" 3; - "~*(?:\b)Nimbostratus(?:\b|)" 3; - "~*(?:\b)Ninja(?:\b|)" 3; - "~*(?:\b)Nmap(?:\b|)" 3; - "~*(?:\b)NPbot(?:\b|)" 3; - "~*(?:\b)Nutch(?:\b|)" 3; - "~*(?:\b)oBot(?:\b|)" 3; - "~*(?:\b)Octopus(?:\b|)" 3; - "~*(?:\b)Offline\ Explorer(?:\b|)" 3; - "~*(?:\b)Offline\ Navigator(?:\b|)" 3; - "~*(?:\b)OnCrawl(?:\b|)" 3; - "~*(?:\b)Openfind(?:\b|)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; - "~*(?:\b)Openvas(?:\b|)" 3; - "~*(?:\b)OpenVAS(?:\b|)" 3; - "~*(?:\b)OrangeBot(?:\b|)" 3; - "~*(?:\b)OrangeSpider(?:\b|)" 3; - "~*(?:\b)OutclicksBot(?:\b|)" 3; - "~*(?:\b)OutfoxBot(?:\b|)" 3; - "~*(?:\b)PageAnalyzer(?:\b|)" 3; - "~*(?:\b)Page\ Analyzer(?:\b|)" 3; - "~*(?:\b)PageGrabber(?:\b|)" 3; - "~*(?:\b)page\ scorer(?:\b|)" 3; - "~*(?:\b)PageScorer(?:\b|)" 3; - "~*(?:\b)Pandalytics(?:\b|)" 3; - "~*(?:\b)Panscient(?:\b|)" 3; - "~*(?:\b)Papa\ Foto(?:\b|)" 3; - "~*(?:\b)Pavuk(?:\b|)" 3; - "~*(?:\b)pcBrowser(?:\b|)" 3; - "~*(?:\b)PECL::HTTP(?:\b|)" 3; - "~*(?:\b)PeoplePal(?:\b|)" 3; - "~*(?:\b)PHPCrawl(?:\b|)" 3; - "~*(?:\b)Picscout(?:\b|)" 3; - "~*(?:\b)Picsearch(?:\b|)" 3; - "~*(?:\b)PictureFinder(?:\b|)" 3; - "~*(?:\b)Pimonster(?:\b|)" 3; - "~*(?:\b)Pi-Monster(?:\b|)" 3; - "~*(?:\b)Pixray(?:\b|)" 3; - "~*(?:\b)PleaseCrawl(?:\b|)" 3; - "~*(?:\b)plumanalytics(?:\b|)" 3; - "~*(?:\b)Pockey(?:\b|)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; - "~*(?:\b)Probethenet(?:\b|)" 3; - "~*(?:\b)ProPowerBot(?:\b|)" 3; - "~*(?:\b)ProWebWalker(?:\b|)" 3; - "~*(?:\b)Psbot(?:\b|)" 3; - "~*(?:\b)Pump(?:\b|)" 3; - "~*(?:\b)PxBroker(?:\b|)" 3; - "~*(?:\b)PyCurl(?:\b|)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; - "~*(?:\b)Quick-Crawler(?:\b|)" 3; - "~*(?:\b)RankActive(?:\b|)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; - "~*(?:\b)RankFlex(?:\b|)" 3; - "~*(?:\b)RankingBot(?:\b|)" 3; - "~*(?:\b)RankingBot2(?:\b|)" 3; - "~*(?:\b)Rankivabot(?:\b|)" 3; - "~*(?:\b)RankurBot(?:\b|)" 3; - "~*(?:\b)RealDownload(?:\b|)" 3; - "~*(?:\b)Reaper(?:\b|)" 3; - "~*(?:\b)RebelMouse(?:\b|)" 3; - "~*(?:\b)Recorder(?:\b|)" 3; - "~*(?:\b)RedesScrapy(?:\b|)" 3; - "~*(?:\b)ReGet(?:\b|)" 3; - "~*(?:\b)RepoMonkey(?:\b|)" 3; - "~*(?:\b)Ripper(?:\b|)" 3; - "~*(?:\b)RocketCrawler(?:\b|)" 3; - "~*(?:\b)Rogerbot(?:\b|)" 3; - "~*(?:\b)s1z.ru(?:\b|)" 3; - "~*(?:\b)SalesIntelligent(?:\b|)" 3; - "~*(?:\b)SBIder(?:\b|)" 3; - "~*(?:\b)ScanAlert(?:\b|)" 3; - "~*(?:\b)Scanbot(?:\b|)" 3; - "~*(?:\b)scan.lol(?:\b|)" 3; - "~*(?:\b)ScoutJet(?:\b|)" 3; - "~*(?:\b)Scrapy(?:\b|)" 3; - "~*(?:\b)Screaming(?:\b|)" 3; - "~*(?:\b)ScreenerBot(?:\b|)" 3; - "~*(?:\b)Searchestate(?:\b|)" 3; - "~*(?:\b)SearchmetricsBot(?:\b|)" 3; - "~*(?:\b)Semrush(?:\b|)" 3; - "~*(?:\b)SemrushBot(?:\b|)" 3; - "~*(?:\b)SEOkicks(?:\b|)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; - "~*(?:\b)Seomoz(?:\b|)" 3; - "~*(?:\b)SEOprofiler(?:\b|)" 3; - "~*(?:\b)seoscanners(?:\b|)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; - "~*(?:\b)SEOstats(?:\b|)" 3; - "~*(?:\b)serpstatbot(?:\b|)" 3; - "~*(?:\b)sexsearcher(?:\b|)" 3; - "~*(?:\b)Shodan(?:\b|)" 3; - "~*(?:\b)Siphon(?:\b|)" 3; - "~*(?:\b)SISTRIX(?:\b|)" 3; - "~*(?:\b)Sitebeam(?:\b|)" 3; - "~*(?:\b)SiteExplorer(?:\b|)" 3; - "~*(?:\b)Siteimprove(?:\b|)" 3; - "~*(?:\b)SiteLockSpider(?:\b|)" 3; - "~*(?:\b)SiteSnagger(?:\b|)" 3; - "~*(?:\b)SiteSucker(?:\b|)" 3; - "~*(?:\b)Site\ Sucker(?:\b|)" 3; - "~*(?:\b)Sitevigil(?:\b|)" 3; - "~*(?:\b)SlySearch(?:\b|)" 3; - "~*(?:\b)SmartDownload(?:\b|)" 3; - "~*(?:\b)SMTBot(?:\b|)" 3; - "~*(?:\b)Snake(?:\b|)" 3; - "~*(?:\b)Snapbot(?:\b|)" 3; - "~*(?:\b)Snoopy(?:\b|)" 3; - "~*(?:\b)SocialRankIOBot(?:\b|)" 3; - "~*(?:\b)Sociscraper(?:\b|)" 3; - "~*(?:\b)sogouspider(?:\b|)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; - "~*(?:\b)Sosospider(?:\b|)" 3; - "~*(?:\b)Sottopop(?:\b|)" 3; - "~*(?:\b)SpaceBison(?:\b|)" 3; - "~*(?:\b)Spammen(?:\b|)" 3; - "~*(?:\b)SpankBot(?:\b|)" 3; - "~*(?:\b)Spanner(?:\b|)" 3; - "~*(?:\b)sp_auditbot(?:\b|)" 3; - "~*(?:\b)Spbot(?:\b|)" 3; - "~*(?:\b)Spinn3r(?:\b|)" 3; - "~*(?:\b)SputnikBot(?:\b|)" 3; - "~*(?:\b)spyfu(?:\b|)" 3; - "~*(?:\b)Sqlmap(?:\b|)" 3; - "~*(?:\b)Sqlworm(?:\b|)" 3; - "~*(?:\b)Sqworm(?:\b|)" 3; - "~*(?:\b)Steeler(?:\b|)" 3; - "~*(?:\b)Stripper(?:\b|)" 3; - "~*(?:\b)Sucker(?:\b|)" 3; - "~*(?:\b)Sucuri(?:\b|)" 3; - "~*(?:\b)SuperBot(?:\b|)" 3; - "~*(?:\b)SuperHTTP(?:\b|)" 3; - "~*(?:\b)Surfbot(?:\b|)" 3; - "~*(?:\b)SurveyBot(?:\b|)" 3; - "~*(?:\b)Suzuran(?:\b|)" 3; - "~*(?:\b)Swiftbot(?:\b|)" 3; - "~*(?:\b)sysscan(?:\b|)" 3; - "~*(?:\b)Szukacz(?:\b|)" 3; - "~*(?:\b)T0PHackTeam(?:\b|)" 3; - "~*(?:\b)T8Abot(?:\b|)" 3; - "~*(?:\b)tAkeOut(?:\b|)" 3; - "~*(?:\b)Teleport(?:\b|)" 3; - "~*(?:\b)TeleportPro(?:\b|)" 3; - "~*(?:\b)Telesoft(?:\b|)" 3; - "~*(?:\b)Telesphoreo(?:\b|)" 3; - "~*(?:\b)Telesphorep(?:\b|)" 3; - "~*(?:\b)The\ Intraformant(?:\b|)" 3; - "~*(?:\b)TheNomad(?:\b|)" 3; - "~*(?:\b)Thumbor(?:\b|)" 3; - "~*(?:\b)TightTwatBot(?:\b|)" 3; - "~*(?:\b)Titan(?:\b|)" 3; - "~*(?:\b)Toata(?:\b|)" 3; - "~*(?:\b)Toweyabot(?:\b|)" 3; - "~*(?:\b)Tracemyfile(?:\b|)" 3; - "~*(?:\b)Trendiction(?:\b|)" 3; - "~*(?:\b)Trendictionbot(?:\b|)" 3; - "~*(?:\b)trendiction.com(?:\b|)" 3; - "~*(?:\b)trendiction.de(?:\b|)" 3; - "~*(?:\b)True_Robot(?:\b|)" 3; - "~*(?:\b)Turingos(?:\b|)" 3; - "~*(?:\b)Turnitin(?:\b|)" 3; - "~*(?:\b)TurnitinBot(?:\b|)" 3; - "~*(?:\b)TwengaBot(?:\b|)" 3; - "~*(?:\b)Twice(?:\b|)" 3; - "~*(?:\b)Typhoeus(?:\b|)" 3; - "~*(?:\b)UnisterBot(?:\b|)" 3; - "~*(?:\b)Upflow(?:\b|)" 3; - "~*(?:\b)URLy.Warning(?:\b|)" 3; - "~*(?:\b)URLy\ Warning(?:\b|)" 3; - "~*(?:\b)Vacuum(?:\b|)" 3; - "~*(?:\b)Vagabondo(?:\b|)" 3; - "~*(?:\b)VB\ Project(?:\b|)" 3; - "~*(?:\b)VCI(?:\b|)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; - "~*(?:\b)VidibleScraper(?:\b|)" 3; - "~*(?:\b)Virusdie(?:\b|)" 3; - "~*(?:\b)VoidEYE(?:\b|)" 3; - "~*(?:\b)Voil(?:\b|)" 3; - "~*(?:\b)Voltron(?:\b|)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; - "~*(?:\b)WallpapersHD(?:\b|)" 3; - "~*(?:\b)WASALive-Bot(?:\b|)" 3; - "~*(?:\b)WBSearchBot(?:\b|)" 3; - "~*(?:\b)Webalta(?:\b|)" 3; - "~*(?:\b)WebAuto(?:\b|)" 3; - "~*(?:\b)Web\ Auto(?:\b|)" 3; - "~*(?:\b)WebBandit(?:\b|)" 3; - "~*(?:\b)WebCollage(?:\b|)" 3; - "~*(?:\b)Web\ Collage(?:\b|)" 3; - "~*(?:\b)WebCopier(?:\b|)" 3; - "~*(?:\b)WEBDAV(?:\b|)" 3; - "~*(?:\b)WebEnhancer(?:\b|)" 3; - "~*(?:\b)Web\ Enhancer(?:\b|)" 3; - "~*(?:\b)WebFetch(?:\b|)" 3; - "~*(?:\b)Web\ Fetch(?:\b|)" 3; - "~*(?:\b)WebFuck(?:\b|)" 3; - "~*(?:\b)Web\ Fuck(?:\b|)" 3; - "~*(?:\b)WebGo\ IS(?:\b|)" 3; - "~*(?:\b)WebImageCollector(?:\b|)" 3; - "~*(?:\b)WebLeacher(?:\b|)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; - "~*(?:\b)webmeup-crawler(?:\b|)" 3; - "~*(?:\b)WebPix(?:\b|)" 3; - "~*(?:\b)Web\ Pix(?:\b|)" 3; - "~*(?:\b)WebReaper(?:\b|)" 3; - "~*(?:\b)WebSauger(?:\b|)" 3; - "~*(?:\b)Web\ Sauger(?:\b|)" 3; - "~*(?:\b)Webshag(?:\b|)" 3; - "~*(?:\b)WebsiteExtractor(?:\b|)" 3; - "~*(?:\b)WebsiteQuester(?:\b|)" 3; - "~*(?:\b)Website\ Quester(?:\b|)" 3; - "~*(?:\b)Webster(?:\b|)" 3; - "~*(?:\b)WebStripper(?:\b|)" 3; - "~*(?:\b)WebSucker(?:\b|)" 3; - "~*(?:\b)Web\ Sucker(?:\b|)" 3; - "~*(?:\b)WebWhacker(?:\b|)" 3; - "~*(?:\b)WebZIP(?:\b|)" 3; - "~*(?:\b)WeSEE(?:\b|)" 3; - "~*(?:\b)Whack(?:\b|)" 3; - "~*(?:\b)Whacker(?:\b|)" 3; - "~*(?:\b)Whatweb(?:\b|)" 3; - "~*(?:\b)Who.is\ Bot(?:\b|)" 3; - "~*(?:\b)Widow(?:\b|)" 3; - "~*(?:\b)WinHTTrack(?:\b|)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; - "~*(?:\b)WISENutbot(?:\b|)" 3; - "~*(?:\b)Wonderbot(?:\b|)" 3; - "~*(?:\b)Woobot(?:\b|)" 3; - "~*(?:\b)Wotbox(?:\b|)" 3; - "~*(?:\b)Wprecon(?:\b|)" 3; - "~*(?:\b)WPScan(?:\b|)" 3; - "~*(?:\b)WWW-Collector-E(?:\b|)" 3; - "~*(?:\b)WWW-Mechanize(?:\b|)" 3; - "~*(?:\b)WWW::Mechanize(?:\b|)" 3; - "~*(?:\b)WWWOFFLE(?:\b|)" 3; - "~*(?:\b)x09Mozilla(?:\b|)" 3; - "~*(?:\b)x22Mozilla(?:\b|)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; - "~*(?:\b)Xenu(?:\b|)" 3; - "~*(?:\b)xpymep1.exe(?:\b|)" 3; - "~*(?:\b)YoudaoBot(?:\b|)" 3; - "~*(?:\b)Zade(?:\b|)" 3; - "~*(?:\b)Zauba(?:\b|)" 3; - "~*(?:\b)zauba.io(?:\b|)" 3; - "~*(?:\b)Zermelo(?:\b|)" 3; - "~*(?:\b)Zeus(?:\b|)" 3; - "~*(?:\b)zgrab(?:\b|)" 3; - "~*(?:\b)Zitebot(?:\b|)" 3; - "~*(?:\b)ZmEu(?:\b|)" 3; - "~*(?:\b)ZumBot(?:\b|)" 3; - "~*(?:\b)ZyBorg(?:\b|)" 3; + "~*(?:\b)360Spider(?:\b)" 3; + "~*(?:\b)404checker(?:\b)" 3; + "~*(?:\b)404enemy(?:\b)" 3; + "~*(?:\b)80legs(?:\b)" 3; + "~*(?:\b)Abonti(?:\b)" 3; + "~*(?:\b)Aboundex(?:\b)" 3; + "~*(?:\b)Aboundexbot(?:\b)" 3; + "~*(?:\b)Acunetix(?:\b)" 3; + "~*(?:\b)ADmantX(?:\b)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; + "~*(?:\b)AhrefsBot(?:\b)" 3; + "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)AiHitBot(?:\b)" 3; + "~*(?:\b)Aipbot(?:\b)" 3; + "~*(?:\b)Alexibot(?:\b)" 3; + "~*(?:\b)Alligator(?:\b)" 3; + "~*(?:\b)AllSubmitter(?:\b)" 3; + "~*(?:\b)AlphaBot(?:\b)" 3; + "~*(?:\b)Anarchie(?:\b)" 3; + "~*(?:\b)Apexoo(?:\b)" 3; + "~*(?:\b)archive.org_bot(?:\b)" 3; + "~*(?:\b)ASPSeek(?:\b)" 3; + "~*(?:\b)Asterias(?:\b)" 3; + "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)BackDoorBot(?:\b)" 3; + "~*(?:\b)Backlink-Ceck(?:\b)" 3; + "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)BacklinkCrawler(?:\b)" 3; + "~*(?:\b)BackStreet(?:\b)" 3; + "~*(?:\b)BackWeb(?:\b)" 3; + "~*(?:\b)Badass(?:\b)" 3; + "~*(?:\b)Bandit(?:\b)" 3; + "~*(?:\b)Barkrowler(?:\b)" 3; + "~*(?:\b)BatchFTP(?:\b)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; + "~*(?:\b)BBBike(?:\b)" 3; + "~*(?:\b)BDCbot(?:\b)" 3; + "~*(?:\b)BDFetch(?:\b)" 3; + "~*(?:\b)BetaBot(?:\b)" 3; + "~*(?:\b)Bigfoot(?:\b)" 3; + "~*(?:\b)Bitacle(?:\b)" 3; + "~*(?:\b)Blackboard(?:\b)" 3; + "~*(?:\b)Black\ Hole(?:\b)" 3; + "~*(?:\b)BlackWidow(?:\b)" 3; + "~*(?:\b)BLEXBot(?:\b)" 3; + "~*(?:\b)Blow(?:\b)" 3; + "~*(?:\b)BlowFish(?:\b)" 3; + "~*(?:\b)Boardreader(?:\b)" 3; + "~*(?:\b)Bolt(?:\b)" 3; + "~*(?:\b)BotALot(?:\b)" 3; + "~*(?:\b)Brandprotect(?:\b)" 3; + "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buddy(?:\b)" 3; + "~*(?:\b)BuiltBotTough(?:\b)" 3; + "~*(?:\b)BuiltWith(?:\b)" 3; + "~*(?:\b)Bullseye(?:\b)" 3; + "~*(?:\b)BunnySlippers(?:\b)" 3; + "~*(?:\b)BuzzSumo(?:\b)" 3; + "~*(?:\b)Calculon(?:\b)" 3; + "~*(?:\b)CATExplorador(?:\b)" 3; + "~*(?:\b)CazoodleBot(?:\b)" 3; + "~*(?:\b)CCBot(?:\b)" 3; + "~*(?:\b)Cegbfeieh(?:\b)" 3; + "~*(?:\b)CheeseBot(?:\b)" 3; + "~*(?:\b)CherryPicker(?:\b)" 3; + "~*(?:\b)CheTeam(?:\b)" 3; + "~*(?:\b)ChinaClaw(?:\b)" 3; + "~*(?:\b)Chlooe(?:\b)" 3; + "~*(?:\b)Claritybot(?:\b)" 3; + "~*(?:\b)Cliqzbot(?:\b)" 3; + "~*(?:\b)Cloud\ mapping(?:\b)" 3; + "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cogentbot(?:\b)" 3; + "~*(?:\b)cognitiveseo(?:\b)" 3; + "~*(?:\b)Collector(?:\b)" 3; + "~*(?:\b)com.plumanalytics(?:\b)" 3; + "~*(?:\b)Copier(?:\b)" 3; + "~*(?:\b)CopyRightCheck(?:\b)" 3; + "~*(?:\b)Copyscape(?:\b)" 3; + "~*(?:\b)Cosmos(?:\b)" 3; + "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)crawler4j(?:\b)" 3; + "~*(?:\b)crawler.feedback(?:\b)" 3; + "~*(?:\b)crawl.sogou.com(?:\b)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b)" 3; + "~*(?:\b)Crescent(?:\b)" 3; + "~*(?:\b)CrunchBot(?:\b)" 3; + "~*(?:\b)CSHttp(?:\b)" 3; + "~*(?:\b)Curious(?:\b)" 3; + "~*(?:\b)Custo(?:\b)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; + "~*(?:\b)DataCha0s(?:\b)" 3; + "~*(?:\b)DBLBot(?:\b)" 3; + "~*(?:\b)demandbase-bot(?:\b)" 3; + "~*(?:\b)Demon(?:\b)" 3; + "~*(?:\b)Deusu(?:\b)" 3; + "~*(?:\b)Devil(?:\b)" 3; + "~*(?:\b)Digincore(?:\b)" 3; + "~*(?:\b)DigitalPebble(?:\b)" 3; + "~*(?:\b)DIIbot(?:\b)" 3; + "~*(?:\b)Dirbuster(?:\b)" 3; + "~*(?:\b)Disco(?:\b)" 3; + "~*(?:\b)Discobot(?:\b)" 3; + "~*(?:\b)Discoverybot(?:\b)" 3; + "~*(?:\b)Dispatch(?:\b)" 3; + "~*(?:\b)DittoSpyder(?:\b)" 3; + "~*(?:\b)DnyzBot(?:\b)" 3; + "~*(?:\b)DomainAppender(?:\b)" 3; + "~*(?:\b)DomainCrawler(?:\b)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)DomainStatsBot(?:\b)" 3; + "~*(?:\b)Dotbot(?:\b)" 3; + "~*(?:\b)Download\ Wonder(?:\b)" 3; + "~*(?:\b)Dragonfly(?:\b)" 3; + "~*(?:\b)Drip(?:\b)" 3; + "~*(?:\b)DSearch(?:\b)" 3; + "~*(?:\b)DTS\ Agent(?:\b)" 3; + "~*(?:\b)EasyDL(?:\b)" 3; + "~*(?:\b)Ebingbong(?:\b)" 3; + "~*(?:\b)eCatch(?:\b)" 3; + "~*(?:\b)ECCP/1.0(?:\b)" 3; + "~*(?:\b)Ecxi(?:\b)" 3; + "~*(?:\b)EirGrabber(?:\b)" 3; + "~*(?:\b)EMail\ Siphon(?:\b)" 3; + "~*(?:\b)EMail\ Wolf(?:\b)" 3; + "~*(?:\b)EroCrawler(?:\b)" 3; + "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)Evil(?:\b)" 3; + "~*(?:\b)Exabot(?:\b)" 3; + "~*(?:\b)Express\ WebPictures(?:\b)" 3; + "~*(?:\b)ExtLinksBot(?:\b)" 3; + "~*(?:\b)Extractor(?:\b)" 3; + "~*(?:\b)ExtractorPro(?:\b)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; + "~*(?:\b)EyeNetIE(?:\b)" 3; + "~*(?:\b)Ezooms(?:\b)" 3; + "~*(?:\b)facebookscraper(?:\b)" 3; + "~*(?:\b)FDM(?:\b)" 3; + "~*(?:\b)FemtosearchBot(?:\b)" 3; + "~*(?:\b)FHscan(?:\b)" 3; + "~*(?:\b)Fimap(?:\b)" 3; + "~*(?:\b)Firefox/7.0(?:\b)" 3; + "~*(?:\b)FlashGet(?:\b)" 3; + "~*(?:\b)Flunky(?:\b)" 3; + "~*(?:\b)Foobot(?:\b)" 3; + "~*(?:\b)Freeuploader(?:\b)" 3; + "~*(?:\b)FrontPage(?:\b)" 3; + "~*(?:\b)FyberSpider(?:\b)" 3; + "~*(?:\b)Fyrebot(?:\b)" 3; + "~*(?:\b)GalaxyBot(?:\b)" 3; + "~*(?:\b)Genieo(?:\b)" 3; + "~*(?:\b)GermCrawler(?:\b)" 3; + "~*(?:\b)Getintent(?:\b)" 3; + "~*(?:\b)GetRight(?:\b)" 3; + "~*(?:\b)GetWeb(?:\b)" 3; + "~*(?:\b)Gigablast(?:\b)" 3; + "~*(?:\b)Gigabot(?:\b)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; + "~*(?:\b)Gotit(?:\b)" 3; + "~*(?:\b)GoZilla(?:\b)" 3; + "~*(?:\b)Go!Zilla(?:\b)" 3; + "~*(?:\b)Grabber(?:\b)" 3; + "~*(?:\b)GrabNet(?:\b)" 3; + "~*(?:\b)Grafula(?:\b)" 3; + "~*(?:\b)GrapeFX(?:\b)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b)" 3; + "~*(?:\b)GridBot(?:\b)" 3; + "~*(?:\b)GT::WWW(?:\b)" 3; + "~*(?:\b)Haansoft(?:\b)" 3; + "~*(?:\b)HaosouSpider(?:\b)" 3; + "~*(?:\b)Harvest(?:\b)" 3; + "~*(?:\b)Havij(?:\b)" 3; + "~*(?:\b)HEADMasterSEO(?:\b)" 3; + "~*(?:\b)Heritrix(?:\b)" 3; + "~*(?:\b)Hloader(?:\b)" 3; + "~*(?:\b)HMView(?:\b)" 3; + "~*(?:\b)HTMLparser(?:\b)" 3; + "~*(?:\b)HTTP::Lite(?:\b)" 3; + "~*(?:\b)HTTrack(?:\b)" 3; + "~*(?:\b)Humanlinks(?:\b)" 3; + "~*(?:\b)HybridBot(?:\b)" 3; + "~*(?:\b)Iblog(?:\b)" 3; + "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)Id-search(?:\b)" 3; + "~*(?:\b)IlseBot(?:\b)" 3; + "~*(?:\b)Image\ Fetch(?:\b)" 3; + "~*(?:\b)Image\ Sucker(?:\b)" 3; + "~*(?:\b)IndeedBot(?:\b)" 3; + "~*(?:\b)Indy\ Library(?:\b)" 3; + "~*(?:\b)InfoNaviRobot(?:\b)" 3; + "~*(?:\b)InfoTekies(?:\b)" 3; + "~*(?:\b)instabid(?:\b)" 3; + "~*(?:\b)Intelliseek(?:\b)" 3; + "~*(?:\b)InterGET(?:\b)" 3; + "~*(?:\b)Internet\ Ninja(?:\b)" 3; + "~*(?:\b)InternetSeer(?:\b)" 3; + "~*(?:\b)internetVista\ monitor(?:\b)" 3; + "~*(?:\b)ips-agent(?:\b)" 3; + "~*(?:\b)Iria(?:\b)" 3; + "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)Iskanie(?:\b)" 3; + "~*(?:\b)IstellaBot(?:\b)" 3; + "~*(?:\b)JamesBOT(?:\b)" 3; + "~*(?:\b)Jbrofuzz(?:\b)" 3; + "~*(?:\b)JennyBot(?:\b)" 3; + "~*(?:\b)JetCar(?:\b)" 3; + "~*(?:\b)Jetty(?:\b)" 3; + "~*(?:\b)JikeSpider(?:\b)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; + "~*(?:\b)Joomla(?:\b)" 3; + "~*(?:\b)Jorgee(?:\b)" 3; + "~*(?:\b)JustView(?:\b)" 3; + "~*(?:\b)Jyxobot(?:\b)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b)" 3; + "~*(?:\b)Keyword\ Density(?:\b)" 3; + "~*(?:\b)Kozmosbot(?:\b)" 3; + "~*(?:\b)Lanshanbot(?:\b)" 3; + "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)LeechFTP(?:\b)" 3; + "~*(?:\b)LeechGet(?:\b)" 3; + "~*(?:\b)LexiBot(?:\b)" 3; + "~*(?:\b)Lftp(?:\b)" 3; + "~*(?:\b)LibWeb(?:\b)" 3; + "~*(?:\b)Libwhisker(?:\b)" 3; + "~*(?:\b)Lightspeedsystems(?:\b)" 3; + "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkdexbot(?:\b)" 3; + "~*(?:\b)LinkextractorPro(?:\b)" 3; + "~*(?:\b)LinkpadBot(?:\b)" 3; + "~*(?:\b)LinkScan(?:\b)" 3; + "~*(?:\b)LinksManager(?:\b)" 3; + "~*(?:\b)LinkWalker(?:\b)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; + "~*(?:\b)Lipperhey(?:\b)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; + "~*(?:\b)Litemage_walker(?:\b)" 3; + "~*(?:\b)Lmspider(?:\b)" 3; + "~*(?:\b)LNSpiderguy(?:\b)" 3; + "~*(?:\b)Ltx71(?:\b)" 3; + "~*(?:\b)lwp-request(?:\b)" 3; + "~*(?:\b)LWP::Simple(?:\b)" 3; + "~*(?:\b)lwp-trivial(?:\b)" 3; + "~*(?:\b)Magnet(?:\b)" 3; + "~*(?:\b)Mag-Net(?:\b)" 3; + "~*(?:\b)magpie-crawler(?:\b)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b)" 3; + "~*(?:\b)Majestic12(?:\b)" 3; + "~*(?:\b)Majestic-SEO(?:\b)" 3; + "~*(?:\b)Majestic\ SEO(?:\b)" 3; + "~*(?:\b)MarkMonitor(?:\b)" 3; + "~*(?:\b)MarkWatch(?:\b)" 3; + "~*(?:\b)Masscan(?:\b)" 3; + "~*(?:\b)Mass\ Downloader(?:\b)" 3; + "~*(?:\b)Mata\ Hari(?:\b)" 3; + "~*(?:\b)MauiBot(?:\b)" 3; + "~*(?:\b)meanpathbot(?:\b)" 3; + "~*(?:\b)Meanpathbot(?:\b)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b)" 3; + "~*(?:\b)mediawords(?:\b)" 3; + "~*(?:\b)MegaIndex.ru(?:\b)" 3; + "~*(?:\b)Metauri(?:\b)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; + "~*(?:\b)MIDown\ tool(?:\b)" 3; + "~*(?:\b)MIIxpc(?:\b)" 3; + "~*(?:\b)Mister\ PiX(?:\b)" 3; + "~*(?:\b)MJ12bot(?:\b)" 3; + "~*(?:\b)Mojeek(?:\b)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; + "~*(?:\b)Mr.4x3(?:\b)" 3; + "~*(?:\b)MSFrontPage(?:\b)" 3; + "~*(?:\b)MSIECrawler(?:\b)" 3; + "~*(?:\b)Msrabot(?:\b)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; + "~*(?:\b)muhstik-scan(?:\b)" 3; + "~*(?:\b)Musobot(?:\b)" 3; + "~*(?:\b)Name\ Intelligence(?:\b)" 3; + "~*(?:\b)Nameprotect(?:\b)" 3; + "~*(?:\b)Navroad(?:\b)" 3; + "~*(?:\b)NearSite(?:\b)" 3; + "~*(?:\b)Needle(?:\b)" 3; + "~*(?:\b)Nessus(?:\b)" 3; + "~*(?:\b)NetAnts(?:\b)" 3; + "~*(?:\b)Netcraft(?:\b)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; + "~*(?:\b)NetLyzer(?:\b)" 3; + "~*(?:\b)NetMechanic(?:\b)" 3; + "~*(?:\b)NetSpider(?:\b)" 3; + "~*(?:\b)Nettrack(?:\b)" 3; + "~*(?:\b)Net\ Vampire(?:\b)" 3; + "~*(?:\b)Netvibes(?:\b)" 3; + "~*(?:\b)NetZIP(?:\b)" 3; + "~*(?:\b)NextGenSearchBot(?:\b)" 3; + "~*(?:\b)Nibbler(?:\b)" 3; + "~*(?:\b)NICErsPRO(?:\b)" 3; + "~*(?:\b)Niki-bot(?:\b)" 3; + "~*(?:\b)Nikto(?:\b)" 3; + "~*(?:\b)NimbleCrawler(?:\b)" 3; + "~*(?:\b)Nimbostratus(?:\b)" 3; + "~*(?:\b)Ninja(?:\b)" 3; + "~*(?:\b)Nmap(?:\b)" 3; + "~*(?:\b)NPbot(?:\b)" 3; + "~*(?:\b)Nutch(?:\b)" 3; + "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)Octopus(?:\b)" 3; + "~*(?:\b)Offline\ Explorer(?:\b)" 3; + "~*(?:\b)Offline\ Navigator(?:\b)" 3; + "~*(?:\b)OnCrawl(?:\b)" 3; + "~*(?:\b)Openfind(?:\b)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b)" 3; + "~*(?:\b)Openvas(?:\b)" 3; + "~*(?:\b)OpenVAS(?:\b)" 3; + "~*(?:\b)OrangeBot(?:\b)" 3; + "~*(?:\b)OrangeSpider(?:\b)" 3; + "~*(?:\b)OutclicksBot(?:\b)" 3; + "~*(?:\b)OutfoxBot(?:\b)" 3; + "~*(?:\b)PageAnalyzer(?:\b)" 3; + "~*(?:\b)Page\ Analyzer(?:\b)" 3; + "~*(?:\b)PageGrabber(?:\b)" 3; + "~*(?:\b)page\ scorer(?:\b)" 3; + "~*(?:\b)PageScorer(?:\b)" 3; + "~*(?:\b)Pandalytics(?:\b)" 3; + "~*(?:\b)Panscient(?:\b)" 3; + "~*(?:\b)Papa\ Foto(?:\b)" 3; + "~*(?:\b)Pavuk(?:\b)" 3; + "~*(?:\b)pcBrowser(?:\b)" 3; + "~*(?:\b)PECL::HTTP(?:\b)" 3; + "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)PHPCrawl(?:\b)" 3; + "~*(?:\b)Picscout(?:\b)" 3; + "~*(?:\b)Picsearch(?:\b)" 3; + "~*(?:\b)PictureFinder(?:\b)" 3; + "~*(?:\b)Pimonster(?:\b)" 3; + "~*(?:\b)Pi-Monster(?:\b)" 3; + "~*(?:\b)Pixray(?:\b)" 3; + "~*(?:\b)PleaseCrawl(?:\b)" 3; + "~*(?:\b)plumanalytics(?:\b)" 3; + "~*(?:\b)Pockey(?:\b)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; + "~*(?:\b)Probethenet(?:\b)" 3; + "~*(?:\b)ProPowerBot(?:\b)" 3; + "~*(?:\b)ProWebWalker(?:\b)" 3; + "~*(?:\b)Psbot(?:\b)" 3; + "~*(?:\b)Pump(?:\b)" 3; + "~*(?:\b)PxBroker(?:\b)" 3; + "~*(?:\b)PyCurl(?:\b)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; + "~*(?:\b)Quick-Crawler(?:\b)" 3; + "~*(?:\b)RankActive(?:\b)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b)" 3; + "~*(?:\b)RankFlex(?:\b)" 3; + "~*(?:\b)RankingBot(?:\b)" 3; + "~*(?:\b)RankingBot2(?:\b)" 3; + "~*(?:\b)Rankivabot(?:\b)" 3; + "~*(?:\b)RankurBot(?:\b)" 3; + "~*(?:\b)RealDownload(?:\b)" 3; + "~*(?:\b)Reaper(?:\b)" 3; + "~*(?:\b)RebelMouse(?:\b)" 3; + "~*(?:\b)Recorder(?:\b)" 3; + "~*(?:\b)RedesScrapy(?:\b)" 3; + "~*(?:\b)ReGet(?:\b)" 3; + "~*(?:\b)RepoMonkey(?:\b)" 3; + "~*(?:\b)Ripper(?:\b)" 3; + "~*(?:\b)RocketCrawler(?:\b)" 3; + "~*(?:\b)Rogerbot(?:\b)" 3; + "~*(?:\b)s1z.ru(?:\b)" 3; + "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)SBIder(?:\b)" 3; + "~*(?:\b)ScanAlert(?:\b)" 3; + "~*(?:\b)Scanbot(?:\b)" 3; + "~*(?:\b)scan.lol(?:\b)" 3; + "~*(?:\b)ScoutJet(?:\b)" 3; + "~*(?:\b)Scrapy(?:\b)" 3; + "~*(?:\b)Screaming(?:\b)" 3; + "~*(?:\b)ScreenerBot(?:\b)" 3; + "~*(?:\b)Searchestate(?:\b)" 3; + "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Semrush(?:\b)" 3; + "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SEOkicks(?:\b)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; + "~*(?:\b)Seomoz(?:\b)" 3; + "~*(?:\b)SEOprofiler(?:\b)" 3; + "~*(?:\b)seoscanners(?:\b)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b)" 3; + "~*(?:\b)SEOstats(?:\b)" 3; + "~*(?:\b)serpstatbot(?:\b)" 3; + "~*(?:\b)sexsearcher(?:\b)" 3; + "~*(?:\b)Shodan(?:\b)" 3; + "~*(?:\b)Siphon(?:\b)" 3; + "~*(?:\b)SISTRIX(?:\b)" 3; + "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteExplorer(?:\b)" 3; + "~*(?:\b)Siteimprove(?:\b)" 3; + "~*(?:\b)SiteLockSpider(?:\b)" 3; + "~*(?:\b)SiteSnagger(?:\b)" 3; + "~*(?:\b)SiteSucker(?:\b)" 3; + "~*(?:\b)Site\ Sucker(?:\b)" 3; + "~*(?:\b)Sitevigil(?:\b)" 3; + "~*(?:\b)SlySearch(?:\b)" 3; + "~*(?:\b)SmartDownload(?:\b)" 3; + "~*(?:\b)SMTBot(?:\b)" 3; + "~*(?:\b)Snake(?:\b)" 3; + "~*(?:\b)Snapbot(?:\b)" 3; + "~*(?:\b)Snoopy(?:\b)" 3; + "~*(?:\b)SocialRankIOBot(?:\b)" 3; + "~*(?:\b)Sociscraper(?:\b)" 3; + "~*(?:\b)sogouspider(?:\b)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; + "~*(?:\b)Sosospider(?:\b)" 3; + "~*(?:\b)Sottopop(?:\b)" 3; + "~*(?:\b)SpaceBison(?:\b)" 3; + "~*(?:\b)Spammen(?:\b)" 3; + "~*(?:\b)SpankBot(?:\b)" 3; + "~*(?:\b)Spanner(?:\b)" 3; + "~*(?:\b)sp_auditbot(?:\b)" 3; + "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spinn3r(?:\b)" 3; + "~*(?:\b)SputnikBot(?:\b)" 3; + "~*(?:\b)spyfu(?:\b)" 3; + "~*(?:\b)Sqlmap(?:\b)" 3; + "~*(?:\b)Sqlworm(?:\b)" 3; + "~*(?:\b)Sqworm(?:\b)" 3; + "~*(?:\b)Steeler(?:\b)" 3; + "~*(?:\b)Stripper(?:\b)" 3; + "~*(?:\b)Sucker(?:\b)" 3; + "~*(?:\b)Sucuri(?:\b)" 3; + "~*(?:\b)SuperBot(?:\b)" 3; + "~*(?:\b)SuperHTTP(?:\b)" 3; + "~*(?:\b)Surfbot(?:\b)" 3; + "~*(?:\b)SurveyBot(?:\b)" 3; + "~*(?:\b)Suzuran(?:\b)" 3; + "~*(?:\b)Swiftbot(?:\b)" 3; + "~*(?:\b)sysscan(?:\b)" 3; + "~*(?:\b)Szukacz(?:\b)" 3; + "~*(?:\b)T0PHackTeam(?:\b)" 3; + "~*(?:\b)T8Abot(?:\b)" 3; + "~*(?:\b)tAkeOut(?:\b)" 3; + "~*(?:\b)Teleport(?:\b)" 3; + "~*(?:\b)TeleportPro(?:\b)" 3; + "~*(?:\b)Telesoft(?:\b)" 3; + "~*(?:\b)Telesphoreo(?:\b)" 3; + "~*(?:\b)Telesphorep(?:\b)" 3; + "~*(?:\b)The\ Intraformant(?:\b)" 3; + "~*(?:\b)TheNomad(?:\b)" 3; + "~*(?:\b)Thumbor(?:\b)" 3; + "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)Titan(?:\b)" 3; + "~*(?:\b)Toata(?:\b)" 3; + "~*(?:\b)Toweyabot(?:\b)" 3; + "~*(?:\b)Tracemyfile(?:\b)" 3; + "~*(?:\b)Trendiction(?:\b)" 3; + "~*(?:\b)Trendictionbot(?:\b)" 3; + "~*(?:\b)trendiction.com(?:\b)" 3; + "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)True_Robot(?:\b)" 3; + "~*(?:\b)Turingos(?:\b)" 3; + "~*(?:\b)Turnitin(?:\b)" 3; + "~*(?:\b)TurnitinBot(?:\b)" 3; + "~*(?:\b)TwengaBot(?:\b)" 3; + "~*(?:\b)Twice(?:\b)" 3; + "~*(?:\b)Typhoeus(?:\b)" 3; + "~*(?:\b)UnisterBot(?:\b)" 3; + "~*(?:\b)Upflow(?:\b)" 3; + "~*(?:\b)URLy.Warning(?:\b)" 3; + "~*(?:\b)URLy\ Warning(?:\b)" 3; + "~*(?:\b)Vacuum(?:\b)" 3; + "~*(?:\b)Vagabondo(?:\b)" 3; + "~*(?:\b)VB\ Project(?:\b)" 3; + "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b)" 3; + "~*(?:\b)VidibleScraper(?:\b)" 3; + "~*(?:\b)Virusdie(?:\b)" 3; + "~*(?:\b)VoidEYE(?:\b)" 3; + "~*(?:\b)Voil(?:\b)" 3; + "~*(?:\b)Voltron(?:\b)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b)" 3; + "~*(?:\b)WallpapersHD(?:\b)" 3; + "~*(?:\b)WASALive-Bot(?:\b)" 3; + "~*(?:\b)WBSearchBot(?:\b)" 3; + "~*(?:\b)Webalta(?:\b)" 3; + "~*(?:\b)WebAuto(?:\b)" 3; + "~*(?:\b)Web\ Auto(?:\b)" 3; + "~*(?:\b)WebBandit(?:\b)" 3; + "~*(?:\b)WebCollage(?:\b)" 3; + "~*(?:\b)Web\ Collage(?:\b)" 3; + "~*(?:\b)WebCopier(?:\b)" 3; + "~*(?:\b)WEBDAV(?:\b)" 3; + "~*(?:\b)WebEnhancer(?:\b)" 3; + "~*(?:\b)Web\ Enhancer(?:\b)" 3; + "~*(?:\b)WebFetch(?:\b)" 3; + "~*(?:\b)Web\ Fetch(?:\b)" 3; + "~*(?:\b)WebFuck(?:\b)" 3; + "~*(?:\b)Web\ Fuck(?:\b)" 3; + "~*(?:\b)WebGo\ IS(?:\b)" 3; + "~*(?:\b)WebImageCollector(?:\b)" 3; + "~*(?:\b)WebLeacher(?:\b)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; + "~*(?:\b)webmeup-crawler(?:\b)" 3; + "~*(?:\b)WebPix(?:\b)" 3; + "~*(?:\b)Web\ Pix(?:\b)" 3; + "~*(?:\b)WebReaper(?:\b)" 3; + "~*(?:\b)WebSauger(?:\b)" 3; + "~*(?:\b)Web\ Sauger(?:\b)" 3; + "~*(?:\b)Webshag(?:\b)" 3; + "~*(?:\b)WebsiteExtractor(?:\b)" 3; + "~*(?:\b)WebsiteQuester(?:\b)" 3; + "~*(?:\b)Website\ Quester(?:\b)" 3; + "~*(?:\b)Webster(?:\b)" 3; + "~*(?:\b)WebStripper(?:\b)" 3; + "~*(?:\b)WebSucker(?:\b)" 3; + "~*(?:\b)Web\ Sucker(?:\b)" 3; + "~*(?:\b)WebWhacker(?:\b)" 3; + "~*(?:\b)WebZIP(?:\b)" 3; + "~*(?:\b)WeSEE(?:\b)" 3; + "~*(?:\b)Whack(?:\b)" 3; + "~*(?:\b)Whacker(?:\b)" 3; + "~*(?:\b)Whatweb(?:\b)" 3; + "~*(?:\b)Who.is\ Bot(?:\b)" 3; + "~*(?:\b)Widow(?:\b)" 3; + "~*(?:\b)WinHTTrack(?:\b)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; + "~*(?:\b)WISENutbot(?:\b)" 3; + "~*(?:\b)Wonderbot(?:\b)" 3; + "~*(?:\b)Woobot(?:\b)" 3; + "~*(?:\b)Wotbox(?:\b)" 3; + "~*(?:\b)Wprecon(?:\b)" 3; + "~*(?:\b)WPScan(?:\b)" 3; + "~*(?:\b)WWW-Collector-E(?:\b)" 3; + "~*(?:\b)WWW-Mechanize(?:\b)" 3; + "~*(?:\b)WWW::Mechanize(?:\b)" 3; + "~*(?:\b)WWWOFFLE(?:\b)" 3; + "~*(?:\b)x09Mozilla(?:\b)" 3; + "~*(?:\b)x22Mozilla(?:\b)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; + "~*(?:\b)Xenu(?:\b)" 3; + "~*(?:\b)xpymep1.exe(?:\b)" 3; + "~*(?:\b)YoudaoBot(?:\b)" 3; + "~*(?:\b)Zade(?:\b)" 3; + "~*(?:\b)Zauba(?:\b)" 3; + "~*(?:\b)zauba.io(?:\b)" 3; + "~*(?:\b)Zermelo(?:\b)" 3; + "~*(?:\b)Zeus(?:\b)" 3; + "~*(?:\b)zgrab(?:\b)" 3; + "~*(?:\b)Zitebot(?:\b)" 3; + "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZumBot(?:\b)" 3; + "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------- @@ -717,38 +717,38 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)adidxbot(?:\b|)" 0; - "~*(?:\b)AdsBot-Google(?:\b|)" 0; - "~*(?:\b)aolbuild(?:\b|)" 0; - "~*(?:\b)bingbot(?:\b|)" 0; - "~*(?:\b)bingpreview(?:\b|)" 0; - "~*(?:\b)DoCoMo(?:\b|)" 0; - "~*(?:\b)duckduckgo(?:\b|)" 0; - "~*(?:\b)facebookexternalhit(?:\b|)" 0; - "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; - "~*(?:\b)Googlebot(?:\b|)" 0; - "~*(?:\b)Googlebot-Image(?:\b|)" 0; - "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; - "~*(?:\b)Googlebot-News(?:\b|)" 0; - "~*(?:\b)Googlebot/Test(?:\b|)" 0; - "~*(?:\b)Googlebot-Video(?:\b|)" 0; - "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; - "~*(?:\b)Gravityscan(?:\b|)" 0; - "~*(?:\b)gsa-crawler(?:\b|)" 0; - "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; - "~*(?:\b)Kraken/0.1(?:\b|)" 0; - "~*(?:\b)LinkedInBot(?:\b|)" 0; - "~*(?:\b)Mediapartners-Google(?:\b|)" 0; - "~*(?:\b)msnbot(?:\b|)" 0; - "~*(?:\b)msnbot-media(?:\b|)" 0; - "~*(?:\b)SAMSUNG(?:\b|)" 0; - "~*(?:\b)Slackbot(?:\b|)" 0; - "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; - "~*(?:\b)slurp(?:\b|)" 0; - "~*(?:\b)teoma(?:\b|)" 0; - "~*(?:\b)TwitterBot(?:\b|)" 0; - "~*(?:\b)Wordpress(?:\b|)" 0; - "~*(?:\b)yahoo(?:\b|)" 0; + "~*(?:\b)adidxbot(?:\b)" 0; + "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bingbot(?:\b)" 0; + "~*(?:\b)bingpreview(?:\b)" 0; + "~*(?:\b)DoCoMo(?:\b)" 0; + "~*(?:\b)duckduckgo(?:\b)" 0; + "~*(?:\b)facebookexternalhit(?:\b)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b)" 0; + "~*(?:\b)Googlebot(?:\b)" 0; + "~*(?:\b)Googlebot-Image(?:\b)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b)" 0; + "~*(?:\b)Googlebot-News(?:\b)" 0; + "~*(?:\b)Googlebot/Test(?:\b)" 0; + "~*(?:\b)Googlebot-Video(?:\b)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; + "~*(?:\b)Gravityscan(?:\b)" 0; + "~*(?:\b)gsa-crawler(?:\b)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b)" 0; + "~*(?:\b)Kraken/0.1(?:\b)" 0; + "~*(?:\b)LinkedInBot(?:\b)" 0; + "~*(?:\b)Mediapartners-Google(?:\b)" 0; + "~*(?:\b)msnbot(?:\b)" 0; + "~*(?:\b)msnbot-media(?:\b)" 0; + "~*(?:\b)SAMSUNG(?:\b)" 0; + "~*(?:\b)Slackbot(?:\b)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 0; + "~*(?:\b)slurp(?:\b)" 0; + "~*(?:\b)teoma(?:\b)" 0; + "~*(?:\b)TwitterBot(?:\b)" 0; + "~*(?:\b)Wordpress(?:\b)" 0; + "~*(?:\b)yahoo(?:\b)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------------------- @@ -760,12 +760,12 @@ map $http_user_agent $bad_bot { #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)jetmon(?:\b|)" 1; - "~*(?:\b)libwww-perl(?:\b|)" 1; - "~*(?:\b)Lynx(?:\b|)" 1; - "~*(?:\b)munin(?:\b|)" 1; - "~*(?:\b)Presto(?:\b|)" 1; - "~*(?:\b)Wget/1.15(?:\b|)" 1; + "~*(?:\b)jetmon(?:\b)" 1; + "~*(?:\b)libwww-perl(?:\b)" 1; + "~*(?:\b)Lynx(?:\b)" 1; + "~*(?:\b)munin(?:\b)" 1; + "~*(?:\b)Presto(?:\b)" 1; + "~*(?:\b)Wget/1.15(?:\b)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ------------------------------------------------------- @@ -778,23 +778,23 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)Alexa(?:\b|)" 2; - "~*(?:\b)archive.org(?:\b|)" 2; - "~*(?:\b)Baidu(?:\b|)" 2; - "~*(?:\b)BUbiNG(?:\b|)" 2; - "~*(?:\b)FlipboardProxy(?:\b|)" 2; - "~*(?:\b)ia_archiver(?:\b|)" 2; - "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; - "~*(?:\b)Proximic(?:\b|)" 2; - "~*(?:\b)R6_CommentReader(?:\b|)" 2; - "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; - "~*(?:\b)RED/1(?:\b|)" 2; - "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; - "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; - "~*(?:\b)Spaidu(?:\b|)" 2; - "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; - "~*(?:\b)YandexBot(?:\b|)" 2; - "~*(?:\b)YandexImages(?:\b|)" 2; + "~*(?:\b)Alexa(?:\b)" 2; + "~*(?:\b)archive.org(?:\b)" 2; + "~*(?:\b)Baidu(?:\b)" 2; + "~*(?:\b)BUbiNG(?:\b)" 2; + "~*(?:\b)FlipboardProxy(?:\b)" 2; + "~*(?:\b)ia_archiver(?:\b)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b)" 2; + "~*(?:\b)Proximic(?:\b)" 2; + "~*(?:\b)R6_CommentReader(?:\b)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b)" 2; + "~*(?:\b)RED/1(?:\b)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b)" 2; + "~*(?:\b)Spaidu(?:\b)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b)" 2; + "~*(?:\b)YandexBot(?:\b)" 2; + "~*(?:\b)YandexImages(?:\b)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 403c2e793..81d44ffba 100644 --- a/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,5 +45,5 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST +35.188.73.34 0; 35.193.14.140 0; -104.198.131.58 0; diff --git a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 71b996828..3116b23ce 100644 --- a/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1709 -### Updated: Mon Jul 1 10:37:04 SAST 2019 +### Version: V4.2019.07.1712 +### Updated: Mon Jul 1 11:24:52 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### @@ -147,562 +147,562 @@ map $http_user_agent $bad_bot { # -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b|)" 3; - "~*(?:\b)404checker(?:\b|)" 3; - "~*(?:\b)404enemy(?:\b|)" 3; - "~*(?:\b)80legs(?:\b|)" 3; - "~*(?:\b)Abonti(?:\b|)" 3; - "~*(?:\b)Aboundex(?:\b|)" 3; - "~*(?:\b)Aboundexbot(?:\b|)" 3; - "~*(?:\b)Acunetix(?:\b|)" 3; - "~*(?:\b)ADmantX(?:\b|)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; - "~*(?:\b)AhrefsBot(?:\b|)" 3; - "~*(?:\b)AIBOT(?:\b|)" 3; - "~*(?:\b)AiHitBot(?:\b|)" 3; - "~*(?:\b)Aipbot(?:\b|)" 3; - "~*(?:\b)Alexibot(?:\b|)" 3; - "~*(?:\b)Alligator(?:\b|)" 3; - "~*(?:\b)AllSubmitter(?:\b|)" 3; - "~*(?:\b)AlphaBot(?:\b|)" 3; - "~*(?:\b)Anarchie(?:\b|)" 3; - "~*(?:\b)Apexoo(?:\b|)" 3; - "~*(?:\b)archive.org_bot(?:\b|)" 3; - "~*(?:\b)ASPSeek(?:\b|)" 3; - "~*(?:\b)Asterias(?:\b|)" 3; - "~*(?:\b)Attach(?:\b|)" 3; - "~*(?:\b)autoemailspider(?:\b|)" 3; - "~*(?:\b)BackDoorBot(?:\b|)" 3; - "~*(?:\b)Backlink-Ceck(?:\b|)" 3; - "~*(?:\b)backlink-check(?:\b|)" 3; - "~*(?:\b)BacklinkCrawler(?:\b|)" 3; - "~*(?:\b)BackStreet(?:\b|)" 3; - "~*(?:\b)BackWeb(?:\b|)" 3; - "~*(?:\b)Badass(?:\b|)" 3; - "~*(?:\b)Bandit(?:\b|)" 3; - "~*(?:\b)Barkrowler(?:\b|)" 3; - "~*(?:\b)BatchFTP(?:\b|)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; - "~*(?:\b)BBBike(?:\b|)" 3; - "~*(?:\b)BDCbot(?:\b|)" 3; - "~*(?:\b)BDFetch(?:\b|)" 3; - "~*(?:\b)BetaBot(?:\b|)" 3; - "~*(?:\b)Bigfoot(?:\b|)" 3; - "~*(?:\b)Bitacle(?:\b|)" 3; - "~*(?:\b)Blackboard(?:\b|)" 3; - "~*(?:\b)Black\ Hole(?:\b|)" 3; - "~*(?:\b)BlackWidow(?:\b|)" 3; - "~*(?:\b)BLEXBot(?:\b|)" 3; - "~*(?:\b)Blow(?:\b|)" 3; - "~*(?:\b)BlowFish(?:\b|)" 3; - "~*(?:\b)Boardreader(?:\b|)" 3; - "~*(?:\b)Bolt(?:\b|)" 3; - "~*(?:\b)BotALot(?:\b|)" 3; - "~*(?:\b)Brandprotect(?:\b|)" 3; - "~*(?:\b)Brandwatch(?:\b|)" 3; - "~*(?:\b)Buddy(?:\b|)" 3; - "~*(?:\b)BuiltBotTough(?:\b|)" 3; - "~*(?:\b)BuiltWith(?:\b|)" 3; - "~*(?:\b)Bullseye(?:\b|)" 3; - "~*(?:\b)BunnySlippers(?:\b|)" 3; - "~*(?:\b)BuzzSumo(?:\b|)" 3; - "~*(?:\b)Calculon(?:\b|)" 3; - "~*(?:\b)CATExplorador(?:\b|)" 3; - "~*(?:\b)CazoodleBot(?:\b|)" 3; - "~*(?:\b)CCBot(?:\b|)" 3; - "~*(?:\b)Cegbfeieh(?:\b|)" 3; - "~*(?:\b)CheeseBot(?:\b|)" 3; - "~*(?:\b)CherryPicker(?:\b|)" 3; - "~*(?:\b)CheTeam(?:\b|)" 3; - "~*(?:\b)ChinaClaw(?:\b|)" 3; - "~*(?:\b)Chlooe(?:\b|)" 3; - "~*(?:\b)Claritybot(?:\b|)" 3; - "~*(?:\b)Cliqzbot(?:\b|)" 3; - "~*(?:\b)Cloud\ mapping(?:\b|)" 3; - "~*(?:\b)coccocbot-web(?:\b|)" 3; - "~*(?:\b)Cogentbot(?:\b|)" 3; - "~*(?:\b)cognitiveseo(?:\b|)" 3; - "~*(?:\b)Collector(?:\b|)" 3; - "~*(?:\b)com.plumanalytics(?:\b|)" 3; - "~*(?:\b)Copier(?:\b|)" 3; - "~*(?:\b)CopyRightCheck(?:\b|)" 3; - "~*(?:\b)Copyscape(?:\b|)" 3; - "~*(?:\b)Cosmos(?:\b|)" 3; - "~*(?:\b)Craftbot(?:\b|)" 3; - "~*(?:\b)crawler4j(?:\b|)" 3; - "~*(?:\b)crawler.feedback(?:\b|)" 3; - "~*(?:\b)crawl.sogou.com(?:\b|)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; - "~*(?:\b)Crescent(?:\b|)" 3; - "~*(?:\b)CrunchBot(?:\b|)" 3; - "~*(?:\b)CSHttp(?:\b|)" 3; - "~*(?:\b)Curious(?:\b|)" 3; - "~*(?:\b)Custo(?:\b|)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; - "~*(?:\b)DataCha0s(?:\b|)" 3; - "~*(?:\b)DBLBot(?:\b|)" 3; - "~*(?:\b)demandbase-bot(?:\b|)" 3; - "~*(?:\b)Demon(?:\b|)" 3; - "~*(?:\b)Deusu(?:\b|)" 3; - "~*(?:\b)Devil(?:\b|)" 3; - "~*(?:\b)Digincore(?:\b|)" 3; - "~*(?:\b)DigitalPebble(?:\b|)" 3; - "~*(?:\b)DIIbot(?:\b|)" 3; - "~*(?:\b)Dirbuster(?:\b|)" 3; - "~*(?:\b)Disco(?:\b|)" 3; - "~*(?:\b)Discobot(?:\b|)" 3; - "~*(?:\b)Discoverybot(?:\b|)" 3; - "~*(?:\b)Dispatch(?:\b|)" 3; - "~*(?:\b)DittoSpyder(?:\b|)" 3; - "~*(?:\b)DnyzBot(?:\b|)" 3; - "~*(?:\b)DomainAppender(?:\b|)" 3; - "~*(?:\b)DomainCrawler(?:\b|)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; - "~*(?:\b)DomainStatsBot(?:\b|)" 3; - "~*(?:\b)Dotbot(?:\b|)" 3; - "~*(?:\b)Download\ Wonder(?:\b|)" 3; - "~*(?:\b)Dragonfly(?:\b|)" 3; - "~*(?:\b)Drip(?:\b|)" 3; - "~*(?:\b)DSearch(?:\b|)" 3; - "~*(?:\b)DTS\ Agent(?:\b|)" 3; - "~*(?:\b)EasyDL(?:\b|)" 3; - "~*(?:\b)Ebingbong(?:\b|)" 3; - "~*(?:\b)eCatch(?:\b|)" 3; - "~*(?:\b)ECCP/1.0(?:\b|)" 3; - "~*(?:\b)Ecxi(?:\b|)" 3; - "~*(?:\b)EirGrabber(?:\b|)" 3; - "~*(?:\b)EMail\ Siphon(?:\b|)" 3; - "~*(?:\b)EMail\ Wolf(?:\b|)" 3; - "~*(?:\b)EroCrawler(?:\b|)" 3; - "~*(?:\b)evc-batch(?:\b|)" 3; - "~*(?:\b)Evil(?:\b|)" 3; - "~*(?:\b)Exabot(?:\b|)" 3; - "~*(?:\b)Express\ WebPictures(?:\b|)" 3; - "~*(?:\b)ExtLinksBot(?:\b|)" 3; - "~*(?:\b)Extractor(?:\b|)" 3; - "~*(?:\b)ExtractorPro(?:\b|)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; - "~*(?:\b)EyeNetIE(?:\b|)" 3; - "~*(?:\b)Ezooms(?:\b|)" 3; - "~*(?:\b)facebookscraper(?:\b|)" 3; - "~*(?:\b)FDM(?:\b|)" 3; - "~*(?:\b)FemtosearchBot(?:\b|)" 3; - "~*(?:\b)FHscan(?:\b|)" 3; - "~*(?:\b)Fimap(?:\b|)" 3; - "~*(?:\b)Firefox/7.0(?:\b|)" 3; - "~*(?:\b)FlashGet(?:\b|)" 3; - "~*(?:\b)Flunky(?:\b|)" 3; - "~*(?:\b)Foobot(?:\b|)" 3; - "~*(?:\b)Freeuploader(?:\b|)" 3; - "~*(?:\b)FrontPage(?:\b|)" 3; - "~*(?:\b)FyberSpider(?:\b|)" 3; - "~*(?:\b)Fyrebot(?:\b|)" 3; - "~*(?:\b)GalaxyBot(?:\b|)" 3; - "~*(?:\b)Genieo(?:\b|)" 3; - "~*(?:\b)GermCrawler(?:\b|)" 3; - "~*(?:\b)Getintent(?:\b|)" 3; - "~*(?:\b)GetRight(?:\b|)" 3; - "~*(?:\b)GetWeb(?:\b|)" 3; - "~*(?:\b)Gigablast(?:\b|)" 3; - "~*(?:\b)Gigabot(?:\b|)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; - "~*(?:\b)Gotit(?:\b|)" 3; - "~*(?:\b)GoZilla(?:\b|)" 3; - "~*(?:\b)Go!Zilla(?:\b|)" 3; - "~*(?:\b)Grabber(?:\b|)" 3; - "~*(?:\b)GrabNet(?:\b|)" 3; - "~*(?:\b)Grafula(?:\b|)" 3; - "~*(?:\b)GrapeFX(?:\b|)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; - "~*(?:\b)GridBot(?:\b|)" 3; - "~*(?:\b)GT::WWW(?:\b|)" 3; - "~*(?:\b)Haansoft(?:\b|)" 3; - "~*(?:\b)HaosouSpider(?:\b|)" 3; - "~*(?:\b)Harvest(?:\b|)" 3; - "~*(?:\b)Havij(?:\b|)" 3; - "~*(?:\b)HEADMasterSEO(?:\b|)" 3; - "~*(?:\b)Heritrix(?:\b|)" 3; - "~*(?:\b)Hloader(?:\b|)" 3; - "~*(?:\b)HMView(?:\b|)" 3; - "~*(?:\b)HTMLparser(?:\b|)" 3; - "~*(?:\b)HTTP::Lite(?:\b|)" 3; - "~*(?:\b)HTTrack(?:\b|)" 3; - "~*(?:\b)Humanlinks(?:\b|)" 3; - "~*(?:\b)HybridBot(?:\b|)" 3; - "~*(?:\b)Iblog(?:\b|)" 3; - "~*(?:\b)IDBot(?:\b|)" 3; - "~*(?:\b)Id-search(?:\b|)" 3; - "~*(?:\b)IlseBot(?:\b|)" 3; - "~*(?:\b)Image\ Fetch(?:\b|)" 3; - "~*(?:\b)Image\ Sucker(?:\b|)" 3; - "~*(?:\b)IndeedBot(?:\b|)" 3; - "~*(?:\b)Indy\ Library(?:\b|)" 3; - "~*(?:\b)InfoNaviRobot(?:\b|)" 3; - "~*(?:\b)InfoTekies(?:\b|)" 3; - "~*(?:\b)instabid(?:\b|)" 3; - "~*(?:\b)Intelliseek(?:\b|)" 3; - "~*(?:\b)InterGET(?:\b|)" 3; - "~*(?:\b)Internet\ Ninja(?:\b|)" 3; - "~*(?:\b)InternetSeer(?:\b|)" 3; - "~*(?:\b)internetVista\ monitor(?:\b|)" 3; - "~*(?:\b)ips-agent(?:\b|)" 3; - "~*(?:\b)Iria(?:\b|)" 3; - "~*(?:\b)IRLbot(?:\b|)" 3; - "~*(?:\b)Iskanie(?:\b|)" 3; - "~*(?:\b)IstellaBot(?:\b|)" 3; - "~*(?:\b)JamesBOT(?:\b|)" 3; - "~*(?:\b)Jbrofuzz(?:\b|)" 3; - "~*(?:\b)JennyBot(?:\b|)" 3; - "~*(?:\b)JetCar(?:\b|)" 3; - "~*(?:\b)Jetty(?:\b|)" 3; - "~*(?:\b)JikeSpider(?:\b|)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; - "~*(?:\b)Joomla(?:\b|)" 3; - "~*(?:\b)Jorgee(?:\b|)" 3; - "~*(?:\b)JustView(?:\b|)" 3; - "~*(?:\b)Jyxobot(?:\b|)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; - "~*(?:\b)Keyword\ Density(?:\b|)" 3; - "~*(?:\b)Kozmosbot(?:\b|)" 3; - "~*(?:\b)Lanshanbot(?:\b|)" 3; - "~*(?:\b)Larbin(?:\b|)" 3; - "~*(?:\b)LeechFTP(?:\b|)" 3; - "~*(?:\b)LeechGet(?:\b|)" 3; - "~*(?:\b)LexiBot(?:\b|)" 3; - "~*(?:\b)Lftp(?:\b|)" 3; - "~*(?:\b)LibWeb(?:\b|)" 3; - "~*(?:\b)Libwhisker(?:\b|)" 3; - "~*(?:\b)Lightspeedsystems(?:\b|)" 3; - "~*(?:\b)Likse(?:\b|)" 3; - "~*(?:\b)Linkdexbot(?:\b|)" 3; - "~*(?:\b)LinkextractorPro(?:\b|)" 3; - "~*(?:\b)LinkpadBot(?:\b|)" 3; - "~*(?:\b)LinkScan(?:\b|)" 3; - "~*(?:\b)LinksManager(?:\b|)" 3; - "~*(?:\b)LinkWalker(?:\b|)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; - "~*(?:\b)Lipperhey(?:\b|)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; - "~*(?:\b)Litemage_walker(?:\b|)" 3; - "~*(?:\b)Lmspider(?:\b|)" 3; - "~*(?:\b)LNSpiderguy(?:\b|)" 3; - "~*(?:\b)Ltx71(?:\b|)" 3; - "~*(?:\b)lwp-request(?:\b|)" 3; - "~*(?:\b)LWP::Simple(?:\b|)" 3; - "~*(?:\b)lwp-trivial(?:\b|)" 3; - "~*(?:\b)Magnet(?:\b|)" 3; - "~*(?:\b)Mag-Net(?:\b|)" 3; - "~*(?:\b)magpie-crawler(?:\b|)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; - "~*(?:\b)Majestic12(?:\b|)" 3; - "~*(?:\b)Majestic-SEO(?:\b|)" 3; - "~*(?:\b)Majestic\ SEO(?:\b|)" 3; - "~*(?:\b)MarkMonitor(?:\b|)" 3; - "~*(?:\b)MarkWatch(?:\b|)" 3; - "~*(?:\b)Masscan(?:\b|)" 3; - "~*(?:\b)Mass\ Downloader(?:\b|)" 3; - "~*(?:\b)Mata\ Hari(?:\b|)" 3; - "~*(?:\b)MauiBot(?:\b|)" 3; - "~*(?:\b)meanpathbot(?:\b|)" 3; - "~*(?:\b)Meanpathbot(?:\b|)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; - "~*(?:\b)mediawords(?:\b|)" 3; - "~*(?:\b)MegaIndex.ru(?:\b|)" 3; - "~*(?:\b)Metauri(?:\b|)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; - "~*(?:\b)MIDown\ tool(?:\b|)" 3; - "~*(?:\b)MIIxpc(?:\b|)" 3; - "~*(?:\b)Mister\ PiX(?:\b|)" 3; - "~*(?:\b)MJ12bot(?:\b|)" 3; - "~*(?:\b)Mojeek(?:\b|)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; - "~*(?:\b)Mr.4x3(?:\b|)" 3; - "~*(?:\b)MSFrontPage(?:\b|)" 3; - "~*(?:\b)MSIECrawler(?:\b|)" 3; - "~*(?:\b)Msrabot(?:\b|)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; - "~*(?:\b)muhstik-scan(?:\b|)" 3; - "~*(?:\b)Musobot(?:\b|)" 3; - "~*(?:\b)Name\ Intelligence(?:\b|)" 3; - "~*(?:\b)Nameprotect(?:\b|)" 3; - "~*(?:\b)Navroad(?:\b|)" 3; - "~*(?:\b)NearSite(?:\b|)" 3; - "~*(?:\b)Needle(?:\b|)" 3; - "~*(?:\b)Nessus(?:\b|)" 3; - "~*(?:\b)NetAnts(?:\b|)" 3; - "~*(?:\b)Netcraft(?:\b|)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; - "~*(?:\b)NetLyzer(?:\b|)" 3; - "~*(?:\b)NetMechanic(?:\b|)" 3; - "~*(?:\b)NetSpider(?:\b|)" 3; - "~*(?:\b)Nettrack(?:\b|)" 3; - "~*(?:\b)Net\ Vampire(?:\b|)" 3; - "~*(?:\b)Netvibes(?:\b|)" 3; - "~*(?:\b)NetZIP(?:\b|)" 3; - "~*(?:\b)NextGenSearchBot(?:\b|)" 3; - "~*(?:\b)Nibbler(?:\b|)" 3; - "~*(?:\b)NICErsPRO(?:\b|)" 3; - "~*(?:\b)Niki-bot(?:\b|)" 3; - "~*(?:\b)Nikto(?:\b|)" 3; - "~*(?:\b)NimbleCrawler(?:\b|)" 3; - "~*(?:\b)Nimbostratus(?:\b|)" 3; - "~*(?:\b)Ninja(?:\b|)" 3; - "~*(?:\b)Nmap(?:\b|)" 3; - "~*(?:\b)NPbot(?:\b|)" 3; - "~*(?:\b)Nutch(?:\b|)" 3; - "~*(?:\b)oBot(?:\b|)" 3; - "~*(?:\b)Octopus(?:\b|)" 3; - "~*(?:\b)Offline\ Explorer(?:\b|)" 3; - "~*(?:\b)Offline\ Navigator(?:\b|)" 3; - "~*(?:\b)OnCrawl(?:\b|)" 3; - "~*(?:\b)Openfind(?:\b|)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; - "~*(?:\b)Openvas(?:\b|)" 3; - "~*(?:\b)OpenVAS(?:\b|)" 3; - "~*(?:\b)OrangeBot(?:\b|)" 3; - "~*(?:\b)OrangeSpider(?:\b|)" 3; - "~*(?:\b)OutclicksBot(?:\b|)" 3; - "~*(?:\b)OutfoxBot(?:\b|)" 3; - "~*(?:\b)PageAnalyzer(?:\b|)" 3; - "~*(?:\b)Page\ Analyzer(?:\b|)" 3; - "~*(?:\b)PageGrabber(?:\b|)" 3; - "~*(?:\b)page\ scorer(?:\b|)" 3; - "~*(?:\b)PageScorer(?:\b|)" 3; - "~*(?:\b)Pandalytics(?:\b|)" 3; - "~*(?:\b)Panscient(?:\b|)" 3; - "~*(?:\b)Papa\ Foto(?:\b|)" 3; - "~*(?:\b)Pavuk(?:\b|)" 3; - "~*(?:\b)pcBrowser(?:\b|)" 3; - "~*(?:\b)PECL::HTTP(?:\b|)" 3; - "~*(?:\b)PeoplePal(?:\b|)" 3; - "~*(?:\b)PHPCrawl(?:\b|)" 3; - "~*(?:\b)Picscout(?:\b|)" 3; - "~*(?:\b)Picsearch(?:\b|)" 3; - "~*(?:\b)PictureFinder(?:\b|)" 3; - "~*(?:\b)Pimonster(?:\b|)" 3; - "~*(?:\b)Pi-Monster(?:\b|)" 3; - "~*(?:\b)Pixray(?:\b|)" 3; - "~*(?:\b)PleaseCrawl(?:\b|)" 3; - "~*(?:\b)plumanalytics(?:\b|)" 3; - "~*(?:\b)Pockey(?:\b|)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; - "~*(?:\b)Probethenet(?:\b|)" 3; - "~*(?:\b)ProPowerBot(?:\b|)" 3; - "~*(?:\b)ProWebWalker(?:\b|)" 3; - "~*(?:\b)Psbot(?:\b|)" 3; - "~*(?:\b)Pump(?:\b|)" 3; - "~*(?:\b)PxBroker(?:\b|)" 3; - "~*(?:\b)PyCurl(?:\b|)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; - "~*(?:\b)Quick-Crawler(?:\b|)" 3; - "~*(?:\b)RankActive(?:\b|)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; - "~*(?:\b)RankFlex(?:\b|)" 3; - "~*(?:\b)RankingBot(?:\b|)" 3; - "~*(?:\b)RankingBot2(?:\b|)" 3; - "~*(?:\b)Rankivabot(?:\b|)" 3; - "~*(?:\b)RankurBot(?:\b|)" 3; - "~*(?:\b)RealDownload(?:\b|)" 3; - "~*(?:\b)Reaper(?:\b|)" 3; - "~*(?:\b)RebelMouse(?:\b|)" 3; - "~*(?:\b)Recorder(?:\b|)" 3; - "~*(?:\b)RedesScrapy(?:\b|)" 3; - "~*(?:\b)ReGet(?:\b|)" 3; - "~*(?:\b)RepoMonkey(?:\b|)" 3; - "~*(?:\b)Ripper(?:\b|)" 3; - "~*(?:\b)RocketCrawler(?:\b|)" 3; - "~*(?:\b)Rogerbot(?:\b|)" 3; - "~*(?:\b)s1z.ru(?:\b|)" 3; - "~*(?:\b)SalesIntelligent(?:\b|)" 3; - "~*(?:\b)SBIder(?:\b|)" 3; - "~*(?:\b)ScanAlert(?:\b|)" 3; - "~*(?:\b)Scanbot(?:\b|)" 3; - "~*(?:\b)scan.lol(?:\b|)" 3; - "~*(?:\b)ScoutJet(?:\b|)" 3; - "~*(?:\b)Scrapy(?:\b|)" 3; - "~*(?:\b)Screaming(?:\b|)" 3; - "~*(?:\b)ScreenerBot(?:\b|)" 3; - "~*(?:\b)Searchestate(?:\b|)" 3; - "~*(?:\b)SearchmetricsBot(?:\b|)" 3; - "~*(?:\b)Semrush(?:\b|)" 3; - "~*(?:\b)SemrushBot(?:\b|)" 3; - "~*(?:\b)SEOkicks(?:\b|)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; - "~*(?:\b)Seomoz(?:\b|)" 3; - "~*(?:\b)SEOprofiler(?:\b|)" 3; - "~*(?:\b)seoscanners(?:\b|)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; - "~*(?:\b)SEOstats(?:\b|)" 3; - "~*(?:\b)serpstatbot(?:\b|)" 3; - "~*(?:\b)sexsearcher(?:\b|)" 3; - "~*(?:\b)Shodan(?:\b|)" 3; - "~*(?:\b)Siphon(?:\b|)" 3; - "~*(?:\b)SISTRIX(?:\b|)" 3; - "~*(?:\b)Sitebeam(?:\b|)" 3; - "~*(?:\b)SiteExplorer(?:\b|)" 3; - "~*(?:\b)Siteimprove(?:\b|)" 3; - "~*(?:\b)SiteLockSpider(?:\b|)" 3; - "~*(?:\b)SiteSnagger(?:\b|)" 3; - "~*(?:\b)SiteSucker(?:\b|)" 3; - "~*(?:\b)Site\ Sucker(?:\b|)" 3; - "~*(?:\b)Sitevigil(?:\b|)" 3; - "~*(?:\b)SlySearch(?:\b|)" 3; - "~*(?:\b)SmartDownload(?:\b|)" 3; - "~*(?:\b)SMTBot(?:\b|)" 3; - "~*(?:\b)Snake(?:\b|)" 3; - "~*(?:\b)Snapbot(?:\b|)" 3; - "~*(?:\b)Snoopy(?:\b|)" 3; - "~*(?:\b)SocialRankIOBot(?:\b|)" 3; - "~*(?:\b)Sociscraper(?:\b|)" 3; - "~*(?:\b)sogouspider(?:\b|)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; - "~*(?:\b)Sosospider(?:\b|)" 3; - "~*(?:\b)Sottopop(?:\b|)" 3; - "~*(?:\b)SpaceBison(?:\b|)" 3; - "~*(?:\b)Spammen(?:\b|)" 3; - "~*(?:\b)SpankBot(?:\b|)" 3; - "~*(?:\b)Spanner(?:\b|)" 3; - "~*(?:\b)sp_auditbot(?:\b|)" 3; - "~*(?:\b)Spbot(?:\b|)" 3; - "~*(?:\b)Spinn3r(?:\b|)" 3; - "~*(?:\b)SputnikBot(?:\b|)" 3; - "~*(?:\b)spyfu(?:\b|)" 3; - "~*(?:\b)Sqlmap(?:\b|)" 3; - "~*(?:\b)Sqlworm(?:\b|)" 3; - "~*(?:\b)Sqworm(?:\b|)" 3; - "~*(?:\b)Steeler(?:\b|)" 3; - "~*(?:\b)Stripper(?:\b|)" 3; - "~*(?:\b)Sucker(?:\b|)" 3; - "~*(?:\b)Sucuri(?:\b|)" 3; - "~*(?:\b)SuperBot(?:\b|)" 3; - "~*(?:\b)SuperHTTP(?:\b|)" 3; - "~*(?:\b)Surfbot(?:\b|)" 3; - "~*(?:\b)SurveyBot(?:\b|)" 3; - "~*(?:\b)Suzuran(?:\b|)" 3; - "~*(?:\b)Swiftbot(?:\b|)" 3; - "~*(?:\b)sysscan(?:\b|)" 3; - "~*(?:\b)Szukacz(?:\b|)" 3; - "~*(?:\b)T0PHackTeam(?:\b|)" 3; - "~*(?:\b)T8Abot(?:\b|)" 3; - "~*(?:\b)tAkeOut(?:\b|)" 3; - "~*(?:\b)Teleport(?:\b|)" 3; - "~*(?:\b)TeleportPro(?:\b|)" 3; - "~*(?:\b)Telesoft(?:\b|)" 3; - "~*(?:\b)Telesphoreo(?:\b|)" 3; - "~*(?:\b)Telesphorep(?:\b|)" 3; - "~*(?:\b)The\ Intraformant(?:\b|)" 3; - "~*(?:\b)TheNomad(?:\b|)" 3; - "~*(?:\b)Thumbor(?:\b|)" 3; - "~*(?:\b)TightTwatBot(?:\b|)" 3; - "~*(?:\b)Titan(?:\b|)" 3; - "~*(?:\b)Toata(?:\b|)" 3; - "~*(?:\b)Toweyabot(?:\b|)" 3; - "~*(?:\b)Tracemyfile(?:\b|)" 3; - "~*(?:\b)Trendiction(?:\b|)" 3; - "~*(?:\b)Trendictionbot(?:\b|)" 3; - "~*(?:\b)trendiction.com(?:\b|)" 3; - "~*(?:\b)trendiction.de(?:\b|)" 3; - "~*(?:\b)True_Robot(?:\b|)" 3; - "~*(?:\b)Turingos(?:\b|)" 3; - "~*(?:\b)Turnitin(?:\b|)" 3; - "~*(?:\b)TurnitinBot(?:\b|)" 3; - "~*(?:\b)TwengaBot(?:\b|)" 3; - "~*(?:\b)Twice(?:\b|)" 3; - "~*(?:\b)Typhoeus(?:\b|)" 3; - "~*(?:\b)UnisterBot(?:\b|)" 3; - "~*(?:\b)Upflow(?:\b|)" 3; - "~*(?:\b)URLy.Warning(?:\b|)" 3; - "~*(?:\b)URLy\ Warning(?:\b|)" 3; - "~*(?:\b)Vacuum(?:\b|)" 3; - "~*(?:\b)Vagabondo(?:\b|)" 3; - "~*(?:\b)VB\ Project(?:\b|)" 3; - "~*(?:\b)VCI(?:\b|)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; - "~*(?:\b)VidibleScraper(?:\b|)" 3; - "~*(?:\b)Virusdie(?:\b|)" 3; - "~*(?:\b)VoidEYE(?:\b|)" 3; - "~*(?:\b)Voil(?:\b|)" 3; - "~*(?:\b)Voltron(?:\b|)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; - "~*(?:\b)WallpapersHD(?:\b|)" 3; - "~*(?:\b)WASALive-Bot(?:\b|)" 3; - "~*(?:\b)WBSearchBot(?:\b|)" 3; - "~*(?:\b)Webalta(?:\b|)" 3; - "~*(?:\b)WebAuto(?:\b|)" 3; - "~*(?:\b)Web\ Auto(?:\b|)" 3; - "~*(?:\b)WebBandit(?:\b|)" 3; - "~*(?:\b)WebCollage(?:\b|)" 3; - "~*(?:\b)Web\ Collage(?:\b|)" 3; - "~*(?:\b)WebCopier(?:\b|)" 3; - "~*(?:\b)WEBDAV(?:\b|)" 3; - "~*(?:\b)WebEnhancer(?:\b|)" 3; - "~*(?:\b)Web\ Enhancer(?:\b|)" 3; - "~*(?:\b)WebFetch(?:\b|)" 3; - "~*(?:\b)Web\ Fetch(?:\b|)" 3; - "~*(?:\b)WebFuck(?:\b|)" 3; - "~*(?:\b)Web\ Fuck(?:\b|)" 3; - "~*(?:\b)WebGo\ IS(?:\b|)" 3; - "~*(?:\b)WebImageCollector(?:\b|)" 3; - "~*(?:\b)WebLeacher(?:\b|)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; - "~*(?:\b)webmeup-crawler(?:\b|)" 3; - "~*(?:\b)WebPix(?:\b|)" 3; - "~*(?:\b)Web\ Pix(?:\b|)" 3; - "~*(?:\b)WebReaper(?:\b|)" 3; - "~*(?:\b)WebSauger(?:\b|)" 3; - "~*(?:\b)Web\ Sauger(?:\b|)" 3; - "~*(?:\b)Webshag(?:\b|)" 3; - "~*(?:\b)WebsiteExtractor(?:\b|)" 3; - "~*(?:\b)WebsiteQuester(?:\b|)" 3; - "~*(?:\b)Website\ Quester(?:\b|)" 3; - "~*(?:\b)Webster(?:\b|)" 3; - "~*(?:\b)WebStripper(?:\b|)" 3; - "~*(?:\b)WebSucker(?:\b|)" 3; - "~*(?:\b)Web\ Sucker(?:\b|)" 3; - "~*(?:\b)WebWhacker(?:\b|)" 3; - "~*(?:\b)WebZIP(?:\b|)" 3; - "~*(?:\b)WeSEE(?:\b|)" 3; - "~*(?:\b)Whack(?:\b|)" 3; - "~*(?:\b)Whacker(?:\b|)" 3; - "~*(?:\b)Whatweb(?:\b|)" 3; - "~*(?:\b)Who.is\ Bot(?:\b|)" 3; - "~*(?:\b)Widow(?:\b|)" 3; - "~*(?:\b)WinHTTrack(?:\b|)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; - "~*(?:\b)WISENutbot(?:\b|)" 3; - "~*(?:\b)Wonderbot(?:\b|)" 3; - "~*(?:\b)Woobot(?:\b|)" 3; - "~*(?:\b)Wotbox(?:\b|)" 3; - "~*(?:\b)Wprecon(?:\b|)" 3; - "~*(?:\b)WPScan(?:\b|)" 3; - "~*(?:\b)WWW-Collector-E(?:\b|)" 3; - "~*(?:\b)WWW-Mechanize(?:\b|)" 3; - "~*(?:\b)WWW::Mechanize(?:\b|)" 3; - "~*(?:\b)WWWOFFLE(?:\b|)" 3; - "~*(?:\b)x09Mozilla(?:\b|)" 3; - "~*(?:\b)x22Mozilla(?:\b|)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; - "~*(?:\b)Xenu(?:\b|)" 3; - "~*(?:\b)xpymep1.exe(?:\b|)" 3; - "~*(?:\b)YoudaoBot(?:\b|)" 3; - "~*(?:\b)Zade(?:\b|)" 3; - "~*(?:\b)Zauba(?:\b|)" 3; - "~*(?:\b)zauba.io(?:\b|)" 3; - "~*(?:\b)Zermelo(?:\b|)" 3; - "~*(?:\b)Zeus(?:\b|)" 3; - "~*(?:\b)zgrab(?:\b|)" 3; - "~*(?:\b)Zitebot(?:\b|)" 3; - "~*(?:\b)ZmEu(?:\b|)" 3; - "~*(?:\b)ZumBot(?:\b|)" 3; - "~*(?:\b)ZyBorg(?:\b|)" 3; + "~*(?:\b)360Spider(?:\b)" 3; + "~*(?:\b)404checker(?:\b)" 3; + "~*(?:\b)404enemy(?:\b)" 3; + "~*(?:\b)80legs(?:\b)" 3; + "~*(?:\b)Abonti(?:\b)" 3; + "~*(?:\b)Aboundex(?:\b)" 3; + "~*(?:\b)Aboundexbot(?:\b)" 3; + "~*(?:\b)Acunetix(?:\b)" 3; + "~*(?:\b)ADmantX(?:\b)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; + "~*(?:\b)AhrefsBot(?:\b)" 3; + "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)AiHitBot(?:\b)" 3; + "~*(?:\b)Aipbot(?:\b)" 3; + "~*(?:\b)Alexibot(?:\b)" 3; + "~*(?:\b)Alligator(?:\b)" 3; + "~*(?:\b)AllSubmitter(?:\b)" 3; + "~*(?:\b)AlphaBot(?:\b)" 3; + "~*(?:\b)Anarchie(?:\b)" 3; + "~*(?:\b)Apexoo(?:\b)" 3; + "~*(?:\b)archive.org_bot(?:\b)" 3; + "~*(?:\b)ASPSeek(?:\b)" 3; + "~*(?:\b)Asterias(?:\b)" 3; + "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)BackDoorBot(?:\b)" 3; + "~*(?:\b)Backlink-Ceck(?:\b)" 3; + "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)BacklinkCrawler(?:\b)" 3; + "~*(?:\b)BackStreet(?:\b)" 3; + "~*(?:\b)BackWeb(?:\b)" 3; + "~*(?:\b)Badass(?:\b)" 3; + "~*(?:\b)Bandit(?:\b)" 3; + "~*(?:\b)Barkrowler(?:\b)" 3; + "~*(?:\b)BatchFTP(?:\b)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; + "~*(?:\b)BBBike(?:\b)" 3; + "~*(?:\b)BDCbot(?:\b)" 3; + "~*(?:\b)BDFetch(?:\b)" 3; + "~*(?:\b)BetaBot(?:\b)" 3; + "~*(?:\b)Bigfoot(?:\b)" 3; + "~*(?:\b)Bitacle(?:\b)" 3; + "~*(?:\b)Blackboard(?:\b)" 3; + "~*(?:\b)Black\ Hole(?:\b)" 3; + "~*(?:\b)BlackWidow(?:\b)" 3; + "~*(?:\b)BLEXBot(?:\b)" 3; + "~*(?:\b)Blow(?:\b)" 3; + "~*(?:\b)BlowFish(?:\b)" 3; + "~*(?:\b)Boardreader(?:\b)" 3; + "~*(?:\b)Bolt(?:\b)" 3; + "~*(?:\b)BotALot(?:\b)" 3; + "~*(?:\b)Brandprotect(?:\b)" 3; + "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buddy(?:\b)" 3; + "~*(?:\b)BuiltBotTough(?:\b)" 3; + "~*(?:\b)BuiltWith(?:\b)" 3; + "~*(?:\b)Bullseye(?:\b)" 3; + "~*(?:\b)BunnySlippers(?:\b)" 3; + "~*(?:\b)BuzzSumo(?:\b)" 3; + "~*(?:\b)Calculon(?:\b)" 3; + "~*(?:\b)CATExplorador(?:\b)" 3; + "~*(?:\b)CazoodleBot(?:\b)" 3; + "~*(?:\b)CCBot(?:\b)" 3; + "~*(?:\b)Cegbfeieh(?:\b)" 3; + "~*(?:\b)CheeseBot(?:\b)" 3; + "~*(?:\b)CherryPicker(?:\b)" 3; + "~*(?:\b)CheTeam(?:\b)" 3; + "~*(?:\b)ChinaClaw(?:\b)" 3; + "~*(?:\b)Chlooe(?:\b)" 3; + "~*(?:\b)Claritybot(?:\b)" 3; + "~*(?:\b)Cliqzbot(?:\b)" 3; + "~*(?:\b)Cloud\ mapping(?:\b)" 3; + "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cogentbot(?:\b)" 3; + "~*(?:\b)cognitiveseo(?:\b)" 3; + "~*(?:\b)Collector(?:\b)" 3; + "~*(?:\b)com.plumanalytics(?:\b)" 3; + "~*(?:\b)Copier(?:\b)" 3; + "~*(?:\b)CopyRightCheck(?:\b)" 3; + "~*(?:\b)Copyscape(?:\b)" 3; + "~*(?:\b)Cosmos(?:\b)" 3; + "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)crawler4j(?:\b)" 3; + "~*(?:\b)crawler.feedback(?:\b)" 3; + "~*(?:\b)crawl.sogou.com(?:\b)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b)" 3; + "~*(?:\b)Crescent(?:\b)" 3; + "~*(?:\b)CrunchBot(?:\b)" 3; + "~*(?:\b)CSHttp(?:\b)" 3; + "~*(?:\b)Curious(?:\b)" 3; + "~*(?:\b)Custo(?:\b)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; + "~*(?:\b)DataCha0s(?:\b)" 3; + "~*(?:\b)DBLBot(?:\b)" 3; + "~*(?:\b)demandbase-bot(?:\b)" 3; + "~*(?:\b)Demon(?:\b)" 3; + "~*(?:\b)Deusu(?:\b)" 3; + "~*(?:\b)Devil(?:\b)" 3; + "~*(?:\b)Digincore(?:\b)" 3; + "~*(?:\b)DigitalPebble(?:\b)" 3; + "~*(?:\b)DIIbot(?:\b)" 3; + "~*(?:\b)Dirbuster(?:\b)" 3; + "~*(?:\b)Disco(?:\b)" 3; + "~*(?:\b)Discobot(?:\b)" 3; + "~*(?:\b)Discoverybot(?:\b)" 3; + "~*(?:\b)Dispatch(?:\b)" 3; + "~*(?:\b)DittoSpyder(?:\b)" 3; + "~*(?:\b)DnyzBot(?:\b)" 3; + "~*(?:\b)DomainAppender(?:\b)" 3; + "~*(?:\b)DomainCrawler(?:\b)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)DomainStatsBot(?:\b)" 3; + "~*(?:\b)Dotbot(?:\b)" 3; + "~*(?:\b)Download\ Wonder(?:\b)" 3; + "~*(?:\b)Dragonfly(?:\b)" 3; + "~*(?:\b)Drip(?:\b)" 3; + "~*(?:\b)DSearch(?:\b)" 3; + "~*(?:\b)DTS\ Agent(?:\b)" 3; + "~*(?:\b)EasyDL(?:\b)" 3; + "~*(?:\b)Ebingbong(?:\b)" 3; + "~*(?:\b)eCatch(?:\b)" 3; + "~*(?:\b)ECCP/1.0(?:\b)" 3; + "~*(?:\b)Ecxi(?:\b)" 3; + "~*(?:\b)EirGrabber(?:\b)" 3; + "~*(?:\b)EMail\ Siphon(?:\b)" 3; + "~*(?:\b)EMail\ Wolf(?:\b)" 3; + "~*(?:\b)EroCrawler(?:\b)" 3; + "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)Evil(?:\b)" 3; + "~*(?:\b)Exabot(?:\b)" 3; + "~*(?:\b)Express\ WebPictures(?:\b)" 3; + "~*(?:\b)ExtLinksBot(?:\b)" 3; + "~*(?:\b)Extractor(?:\b)" 3; + "~*(?:\b)ExtractorPro(?:\b)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; + "~*(?:\b)EyeNetIE(?:\b)" 3; + "~*(?:\b)Ezooms(?:\b)" 3; + "~*(?:\b)facebookscraper(?:\b)" 3; + "~*(?:\b)FDM(?:\b)" 3; + "~*(?:\b)FemtosearchBot(?:\b)" 3; + "~*(?:\b)FHscan(?:\b)" 3; + "~*(?:\b)Fimap(?:\b)" 3; + "~*(?:\b)Firefox/7.0(?:\b)" 3; + "~*(?:\b)FlashGet(?:\b)" 3; + "~*(?:\b)Flunky(?:\b)" 3; + "~*(?:\b)Foobot(?:\b)" 3; + "~*(?:\b)Freeuploader(?:\b)" 3; + "~*(?:\b)FrontPage(?:\b)" 3; + "~*(?:\b)FyberSpider(?:\b)" 3; + "~*(?:\b)Fyrebot(?:\b)" 3; + "~*(?:\b)GalaxyBot(?:\b)" 3; + "~*(?:\b)Genieo(?:\b)" 3; + "~*(?:\b)GermCrawler(?:\b)" 3; + "~*(?:\b)Getintent(?:\b)" 3; + "~*(?:\b)GetRight(?:\b)" 3; + "~*(?:\b)GetWeb(?:\b)" 3; + "~*(?:\b)Gigablast(?:\b)" 3; + "~*(?:\b)Gigabot(?:\b)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; + "~*(?:\b)Gotit(?:\b)" 3; + "~*(?:\b)GoZilla(?:\b)" 3; + "~*(?:\b)Go!Zilla(?:\b)" 3; + "~*(?:\b)Grabber(?:\b)" 3; + "~*(?:\b)GrabNet(?:\b)" 3; + "~*(?:\b)Grafula(?:\b)" 3; + "~*(?:\b)GrapeFX(?:\b)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b)" 3; + "~*(?:\b)GridBot(?:\b)" 3; + "~*(?:\b)GT::WWW(?:\b)" 3; + "~*(?:\b)Haansoft(?:\b)" 3; + "~*(?:\b)HaosouSpider(?:\b)" 3; + "~*(?:\b)Harvest(?:\b)" 3; + "~*(?:\b)Havij(?:\b)" 3; + "~*(?:\b)HEADMasterSEO(?:\b)" 3; + "~*(?:\b)Heritrix(?:\b)" 3; + "~*(?:\b)Hloader(?:\b)" 3; + "~*(?:\b)HMView(?:\b)" 3; + "~*(?:\b)HTMLparser(?:\b)" 3; + "~*(?:\b)HTTP::Lite(?:\b)" 3; + "~*(?:\b)HTTrack(?:\b)" 3; + "~*(?:\b)Humanlinks(?:\b)" 3; + "~*(?:\b)HybridBot(?:\b)" 3; + "~*(?:\b)Iblog(?:\b)" 3; + "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)Id-search(?:\b)" 3; + "~*(?:\b)IlseBot(?:\b)" 3; + "~*(?:\b)Image\ Fetch(?:\b)" 3; + "~*(?:\b)Image\ Sucker(?:\b)" 3; + "~*(?:\b)IndeedBot(?:\b)" 3; + "~*(?:\b)Indy\ Library(?:\b)" 3; + "~*(?:\b)InfoNaviRobot(?:\b)" 3; + "~*(?:\b)InfoTekies(?:\b)" 3; + "~*(?:\b)instabid(?:\b)" 3; + "~*(?:\b)Intelliseek(?:\b)" 3; + "~*(?:\b)InterGET(?:\b)" 3; + "~*(?:\b)Internet\ Ninja(?:\b)" 3; + "~*(?:\b)InternetSeer(?:\b)" 3; + "~*(?:\b)internetVista\ monitor(?:\b)" 3; + "~*(?:\b)ips-agent(?:\b)" 3; + "~*(?:\b)Iria(?:\b)" 3; + "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)Iskanie(?:\b)" 3; + "~*(?:\b)IstellaBot(?:\b)" 3; + "~*(?:\b)JamesBOT(?:\b)" 3; + "~*(?:\b)Jbrofuzz(?:\b)" 3; + "~*(?:\b)JennyBot(?:\b)" 3; + "~*(?:\b)JetCar(?:\b)" 3; + "~*(?:\b)Jetty(?:\b)" 3; + "~*(?:\b)JikeSpider(?:\b)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; + "~*(?:\b)Joomla(?:\b)" 3; + "~*(?:\b)Jorgee(?:\b)" 3; + "~*(?:\b)JustView(?:\b)" 3; + "~*(?:\b)Jyxobot(?:\b)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b)" 3; + "~*(?:\b)Keyword\ Density(?:\b)" 3; + "~*(?:\b)Kozmosbot(?:\b)" 3; + "~*(?:\b)Lanshanbot(?:\b)" 3; + "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)LeechFTP(?:\b)" 3; + "~*(?:\b)LeechGet(?:\b)" 3; + "~*(?:\b)LexiBot(?:\b)" 3; + "~*(?:\b)Lftp(?:\b)" 3; + "~*(?:\b)LibWeb(?:\b)" 3; + "~*(?:\b)Libwhisker(?:\b)" 3; + "~*(?:\b)Lightspeedsystems(?:\b)" 3; + "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkdexbot(?:\b)" 3; + "~*(?:\b)LinkextractorPro(?:\b)" 3; + "~*(?:\b)LinkpadBot(?:\b)" 3; + "~*(?:\b)LinkScan(?:\b)" 3; + "~*(?:\b)LinksManager(?:\b)" 3; + "~*(?:\b)LinkWalker(?:\b)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; + "~*(?:\b)Lipperhey(?:\b)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; + "~*(?:\b)Litemage_walker(?:\b)" 3; + "~*(?:\b)Lmspider(?:\b)" 3; + "~*(?:\b)LNSpiderguy(?:\b)" 3; + "~*(?:\b)Ltx71(?:\b)" 3; + "~*(?:\b)lwp-request(?:\b)" 3; + "~*(?:\b)LWP::Simple(?:\b)" 3; + "~*(?:\b)lwp-trivial(?:\b)" 3; + "~*(?:\b)Magnet(?:\b)" 3; + "~*(?:\b)Mag-Net(?:\b)" 3; + "~*(?:\b)magpie-crawler(?:\b)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b)" 3; + "~*(?:\b)Majestic12(?:\b)" 3; + "~*(?:\b)Majestic-SEO(?:\b)" 3; + "~*(?:\b)Majestic\ SEO(?:\b)" 3; + "~*(?:\b)MarkMonitor(?:\b)" 3; + "~*(?:\b)MarkWatch(?:\b)" 3; + "~*(?:\b)Masscan(?:\b)" 3; + "~*(?:\b)Mass\ Downloader(?:\b)" 3; + "~*(?:\b)Mata\ Hari(?:\b)" 3; + "~*(?:\b)MauiBot(?:\b)" 3; + "~*(?:\b)meanpathbot(?:\b)" 3; + "~*(?:\b)Meanpathbot(?:\b)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b)" 3; + "~*(?:\b)mediawords(?:\b)" 3; + "~*(?:\b)MegaIndex.ru(?:\b)" 3; + "~*(?:\b)Metauri(?:\b)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; + "~*(?:\b)MIDown\ tool(?:\b)" 3; + "~*(?:\b)MIIxpc(?:\b)" 3; + "~*(?:\b)Mister\ PiX(?:\b)" 3; + "~*(?:\b)MJ12bot(?:\b)" 3; + "~*(?:\b)Mojeek(?:\b)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; + "~*(?:\b)Mr.4x3(?:\b)" 3; + "~*(?:\b)MSFrontPage(?:\b)" 3; + "~*(?:\b)MSIECrawler(?:\b)" 3; + "~*(?:\b)Msrabot(?:\b)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; + "~*(?:\b)muhstik-scan(?:\b)" 3; + "~*(?:\b)Musobot(?:\b)" 3; + "~*(?:\b)Name\ Intelligence(?:\b)" 3; + "~*(?:\b)Nameprotect(?:\b)" 3; + "~*(?:\b)Navroad(?:\b)" 3; + "~*(?:\b)NearSite(?:\b)" 3; + "~*(?:\b)Needle(?:\b)" 3; + "~*(?:\b)Nessus(?:\b)" 3; + "~*(?:\b)NetAnts(?:\b)" 3; + "~*(?:\b)Netcraft(?:\b)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; + "~*(?:\b)NetLyzer(?:\b)" 3; + "~*(?:\b)NetMechanic(?:\b)" 3; + "~*(?:\b)NetSpider(?:\b)" 3; + "~*(?:\b)Nettrack(?:\b)" 3; + "~*(?:\b)Net\ Vampire(?:\b)" 3; + "~*(?:\b)Netvibes(?:\b)" 3; + "~*(?:\b)NetZIP(?:\b)" 3; + "~*(?:\b)NextGenSearchBot(?:\b)" 3; + "~*(?:\b)Nibbler(?:\b)" 3; + "~*(?:\b)NICErsPRO(?:\b)" 3; + "~*(?:\b)Niki-bot(?:\b)" 3; + "~*(?:\b)Nikto(?:\b)" 3; + "~*(?:\b)NimbleCrawler(?:\b)" 3; + "~*(?:\b)Nimbostratus(?:\b)" 3; + "~*(?:\b)Ninja(?:\b)" 3; + "~*(?:\b)Nmap(?:\b)" 3; + "~*(?:\b)NPbot(?:\b)" 3; + "~*(?:\b)Nutch(?:\b)" 3; + "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)Octopus(?:\b)" 3; + "~*(?:\b)Offline\ Explorer(?:\b)" 3; + "~*(?:\b)Offline\ Navigator(?:\b)" 3; + "~*(?:\b)OnCrawl(?:\b)" 3; + "~*(?:\b)Openfind(?:\b)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b)" 3; + "~*(?:\b)Openvas(?:\b)" 3; + "~*(?:\b)OpenVAS(?:\b)" 3; + "~*(?:\b)OrangeBot(?:\b)" 3; + "~*(?:\b)OrangeSpider(?:\b)" 3; + "~*(?:\b)OutclicksBot(?:\b)" 3; + "~*(?:\b)OutfoxBot(?:\b)" 3; + "~*(?:\b)PageAnalyzer(?:\b)" 3; + "~*(?:\b)Page\ Analyzer(?:\b)" 3; + "~*(?:\b)PageGrabber(?:\b)" 3; + "~*(?:\b)page\ scorer(?:\b)" 3; + "~*(?:\b)PageScorer(?:\b)" 3; + "~*(?:\b)Pandalytics(?:\b)" 3; + "~*(?:\b)Panscient(?:\b)" 3; + "~*(?:\b)Papa\ Foto(?:\b)" 3; + "~*(?:\b)Pavuk(?:\b)" 3; + "~*(?:\b)pcBrowser(?:\b)" 3; + "~*(?:\b)PECL::HTTP(?:\b)" 3; + "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)PHPCrawl(?:\b)" 3; + "~*(?:\b)Picscout(?:\b)" 3; + "~*(?:\b)Picsearch(?:\b)" 3; + "~*(?:\b)PictureFinder(?:\b)" 3; + "~*(?:\b)Pimonster(?:\b)" 3; + "~*(?:\b)Pi-Monster(?:\b)" 3; + "~*(?:\b)Pixray(?:\b)" 3; + "~*(?:\b)PleaseCrawl(?:\b)" 3; + "~*(?:\b)plumanalytics(?:\b)" 3; + "~*(?:\b)Pockey(?:\b)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; + "~*(?:\b)Probethenet(?:\b)" 3; + "~*(?:\b)ProPowerBot(?:\b)" 3; + "~*(?:\b)ProWebWalker(?:\b)" 3; + "~*(?:\b)Psbot(?:\b)" 3; + "~*(?:\b)Pump(?:\b)" 3; + "~*(?:\b)PxBroker(?:\b)" 3; + "~*(?:\b)PyCurl(?:\b)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; + "~*(?:\b)Quick-Crawler(?:\b)" 3; + "~*(?:\b)RankActive(?:\b)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b)" 3; + "~*(?:\b)RankFlex(?:\b)" 3; + "~*(?:\b)RankingBot(?:\b)" 3; + "~*(?:\b)RankingBot2(?:\b)" 3; + "~*(?:\b)Rankivabot(?:\b)" 3; + "~*(?:\b)RankurBot(?:\b)" 3; + "~*(?:\b)RealDownload(?:\b)" 3; + "~*(?:\b)Reaper(?:\b)" 3; + "~*(?:\b)RebelMouse(?:\b)" 3; + "~*(?:\b)Recorder(?:\b)" 3; + "~*(?:\b)RedesScrapy(?:\b)" 3; + "~*(?:\b)ReGet(?:\b)" 3; + "~*(?:\b)RepoMonkey(?:\b)" 3; + "~*(?:\b)Ripper(?:\b)" 3; + "~*(?:\b)RocketCrawler(?:\b)" 3; + "~*(?:\b)Rogerbot(?:\b)" 3; + "~*(?:\b)s1z.ru(?:\b)" 3; + "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)SBIder(?:\b)" 3; + "~*(?:\b)ScanAlert(?:\b)" 3; + "~*(?:\b)Scanbot(?:\b)" 3; + "~*(?:\b)scan.lol(?:\b)" 3; + "~*(?:\b)ScoutJet(?:\b)" 3; + "~*(?:\b)Scrapy(?:\b)" 3; + "~*(?:\b)Screaming(?:\b)" 3; + "~*(?:\b)ScreenerBot(?:\b)" 3; + "~*(?:\b)Searchestate(?:\b)" 3; + "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Semrush(?:\b)" 3; + "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SEOkicks(?:\b)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; + "~*(?:\b)Seomoz(?:\b)" 3; + "~*(?:\b)SEOprofiler(?:\b)" 3; + "~*(?:\b)seoscanners(?:\b)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b)" 3; + "~*(?:\b)SEOstats(?:\b)" 3; + "~*(?:\b)serpstatbot(?:\b)" 3; + "~*(?:\b)sexsearcher(?:\b)" 3; + "~*(?:\b)Shodan(?:\b)" 3; + "~*(?:\b)Siphon(?:\b)" 3; + "~*(?:\b)SISTRIX(?:\b)" 3; + "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteExplorer(?:\b)" 3; + "~*(?:\b)Siteimprove(?:\b)" 3; + "~*(?:\b)SiteLockSpider(?:\b)" 3; + "~*(?:\b)SiteSnagger(?:\b)" 3; + "~*(?:\b)SiteSucker(?:\b)" 3; + "~*(?:\b)Site\ Sucker(?:\b)" 3; + "~*(?:\b)Sitevigil(?:\b)" 3; + "~*(?:\b)SlySearch(?:\b)" 3; + "~*(?:\b)SmartDownload(?:\b)" 3; + "~*(?:\b)SMTBot(?:\b)" 3; + "~*(?:\b)Snake(?:\b)" 3; + "~*(?:\b)Snapbot(?:\b)" 3; + "~*(?:\b)Snoopy(?:\b)" 3; + "~*(?:\b)SocialRankIOBot(?:\b)" 3; + "~*(?:\b)Sociscraper(?:\b)" 3; + "~*(?:\b)sogouspider(?:\b)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; + "~*(?:\b)Sosospider(?:\b)" 3; + "~*(?:\b)Sottopop(?:\b)" 3; + "~*(?:\b)SpaceBison(?:\b)" 3; + "~*(?:\b)Spammen(?:\b)" 3; + "~*(?:\b)SpankBot(?:\b)" 3; + "~*(?:\b)Spanner(?:\b)" 3; + "~*(?:\b)sp_auditbot(?:\b)" 3; + "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spinn3r(?:\b)" 3; + "~*(?:\b)SputnikBot(?:\b)" 3; + "~*(?:\b)spyfu(?:\b)" 3; + "~*(?:\b)Sqlmap(?:\b)" 3; + "~*(?:\b)Sqlworm(?:\b)" 3; + "~*(?:\b)Sqworm(?:\b)" 3; + "~*(?:\b)Steeler(?:\b)" 3; + "~*(?:\b)Stripper(?:\b)" 3; + "~*(?:\b)Sucker(?:\b)" 3; + "~*(?:\b)Sucuri(?:\b)" 3; + "~*(?:\b)SuperBot(?:\b)" 3; + "~*(?:\b)SuperHTTP(?:\b)" 3; + "~*(?:\b)Surfbot(?:\b)" 3; + "~*(?:\b)SurveyBot(?:\b)" 3; + "~*(?:\b)Suzuran(?:\b)" 3; + "~*(?:\b)Swiftbot(?:\b)" 3; + "~*(?:\b)sysscan(?:\b)" 3; + "~*(?:\b)Szukacz(?:\b)" 3; + "~*(?:\b)T0PHackTeam(?:\b)" 3; + "~*(?:\b)T8Abot(?:\b)" 3; + "~*(?:\b)tAkeOut(?:\b)" 3; + "~*(?:\b)Teleport(?:\b)" 3; + "~*(?:\b)TeleportPro(?:\b)" 3; + "~*(?:\b)Telesoft(?:\b)" 3; + "~*(?:\b)Telesphoreo(?:\b)" 3; + "~*(?:\b)Telesphorep(?:\b)" 3; + "~*(?:\b)The\ Intraformant(?:\b)" 3; + "~*(?:\b)TheNomad(?:\b)" 3; + "~*(?:\b)Thumbor(?:\b)" 3; + "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)Titan(?:\b)" 3; + "~*(?:\b)Toata(?:\b)" 3; + "~*(?:\b)Toweyabot(?:\b)" 3; + "~*(?:\b)Tracemyfile(?:\b)" 3; + "~*(?:\b)Trendiction(?:\b)" 3; + "~*(?:\b)Trendictionbot(?:\b)" 3; + "~*(?:\b)trendiction.com(?:\b)" 3; + "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)True_Robot(?:\b)" 3; + "~*(?:\b)Turingos(?:\b)" 3; + "~*(?:\b)Turnitin(?:\b)" 3; + "~*(?:\b)TurnitinBot(?:\b)" 3; + "~*(?:\b)TwengaBot(?:\b)" 3; + "~*(?:\b)Twice(?:\b)" 3; + "~*(?:\b)Typhoeus(?:\b)" 3; + "~*(?:\b)UnisterBot(?:\b)" 3; + "~*(?:\b)Upflow(?:\b)" 3; + "~*(?:\b)URLy.Warning(?:\b)" 3; + "~*(?:\b)URLy\ Warning(?:\b)" 3; + "~*(?:\b)Vacuum(?:\b)" 3; + "~*(?:\b)Vagabondo(?:\b)" 3; + "~*(?:\b)VB\ Project(?:\b)" 3; + "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b)" 3; + "~*(?:\b)VidibleScraper(?:\b)" 3; + "~*(?:\b)Virusdie(?:\b)" 3; + "~*(?:\b)VoidEYE(?:\b)" 3; + "~*(?:\b)Voil(?:\b)" 3; + "~*(?:\b)Voltron(?:\b)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b)" 3; + "~*(?:\b)WallpapersHD(?:\b)" 3; + "~*(?:\b)WASALive-Bot(?:\b)" 3; + "~*(?:\b)WBSearchBot(?:\b)" 3; + "~*(?:\b)Webalta(?:\b)" 3; + "~*(?:\b)WebAuto(?:\b)" 3; + "~*(?:\b)Web\ Auto(?:\b)" 3; + "~*(?:\b)WebBandit(?:\b)" 3; + "~*(?:\b)WebCollage(?:\b)" 3; + "~*(?:\b)Web\ Collage(?:\b)" 3; + "~*(?:\b)WebCopier(?:\b)" 3; + "~*(?:\b)WEBDAV(?:\b)" 3; + "~*(?:\b)WebEnhancer(?:\b)" 3; + "~*(?:\b)Web\ Enhancer(?:\b)" 3; + "~*(?:\b)WebFetch(?:\b)" 3; + "~*(?:\b)Web\ Fetch(?:\b)" 3; + "~*(?:\b)WebFuck(?:\b)" 3; + "~*(?:\b)Web\ Fuck(?:\b)" 3; + "~*(?:\b)WebGo\ IS(?:\b)" 3; + "~*(?:\b)WebImageCollector(?:\b)" 3; + "~*(?:\b)WebLeacher(?:\b)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; + "~*(?:\b)webmeup-crawler(?:\b)" 3; + "~*(?:\b)WebPix(?:\b)" 3; + "~*(?:\b)Web\ Pix(?:\b)" 3; + "~*(?:\b)WebReaper(?:\b)" 3; + "~*(?:\b)WebSauger(?:\b)" 3; + "~*(?:\b)Web\ Sauger(?:\b)" 3; + "~*(?:\b)Webshag(?:\b)" 3; + "~*(?:\b)WebsiteExtractor(?:\b)" 3; + "~*(?:\b)WebsiteQuester(?:\b)" 3; + "~*(?:\b)Website\ Quester(?:\b)" 3; + "~*(?:\b)Webster(?:\b)" 3; + "~*(?:\b)WebStripper(?:\b)" 3; + "~*(?:\b)WebSucker(?:\b)" 3; + "~*(?:\b)Web\ Sucker(?:\b)" 3; + "~*(?:\b)WebWhacker(?:\b)" 3; + "~*(?:\b)WebZIP(?:\b)" 3; + "~*(?:\b)WeSEE(?:\b)" 3; + "~*(?:\b)Whack(?:\b)" 3; + "~*(?:\b)Whacker(?:\b)" 3; + "~*(?:\b)Whatweb(?:\b)" 3; + "~*(?:\b)Who.is\ Bot(?:\b)" 3; + "~*(?:\b)Widow(?:\b)" 3; + "~*(?:\b)WinHTTrack(?:\b)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; + "~*(?:\b)WISENutbot(?:\b)" 3; + "~*(?:\b)Wonderbot(?:\b)" 3; + "~*(?:\b)Woobot(?:\b)" 3; + "~*(?:\b)Wotbox(?:\b)" 3; + "~*(?:\b)Wprecon(?:\b)" 3; + "~*(?:\b)WPScan(?:\b)" 3; + "~*(?:\b)WWW-Collector-E(?:\b)" 3; + "~*(?:\b)WWW-Mechanize(?:\b)" 3; + "~*(?:\b)WWW::Mechanize(?:\b)" 3; + "~*(?:\b)WWWOFFLE(?:\b)" 3; + "~*(?:\b)x09Mozilla(?:\b)" 3; + "~*(?:\b)x22Mozilla(?:\b)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; + "~*(?:\b)Xenu(?:\b)" 3; + "~*(?:\b)xpymep1.exe(?:\b)" 3; + "~*(?:\b)YoudaoBot(?:\b)" 3; + "~*(?:\b)Zade(?:\b)" 3; + "~*(?:\b)Zauba(?:\b)" 3; + "~*(?:\b)zauba.io(?:\b)" 3; + "~*(?:\b)Zermelo(?:\b)" 3; + "~*(?:\b)Zeus(?:\b)" 3; + "~*(?:\b)zgrab(?:\b)" 3; + "~*(?:\b)Zitebot(?:\b)" 3; + "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZumBot(?:\b)" 3; + "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------- @@ -717,38 +717,38 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)adidxbot(?:\b|)" 0; - "~*(?:\b)AdsBot-Google(?:\b|)" 0; - "~*(?:\b)aolbuild(?:\b|)" 0; - "~*(?:\b)bingbot(?:\b|)" 0; - "~*(?:\b)bingpreview(?:\b|)" 0; - "~*(?:\b)DoCoMo(?:\b|)" 0; - "~*(?:\b)duckduckgo(?:\b|)" 0; - "~*(?:\b)facebookexternalhit(?:\b|)" 0; - "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; - "~*(?:\b)Googlebot(?:\b|)" 0; - "~*(?:\b)Googlebot-Image(?:\b|)" 0; - "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; - "~*(?:\b)Googlebot-News(?:\b|)" 0; - "~*(?:\b)Googlebot/Test(?:\b|)" 0; - "~*(?:\b)Googlebot-Video(?:\b|)" 0; - "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; - "~*(?:\b)Gravityscan(?:\b|)" 0; - "~*(?:\b)gsa-crawler(?:\b|)" 0; - "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; - "~*(?:\b)Kraken/0.1(?:\b|)" 0; - "~*(?:\b)LinkedInBot(?:\b|)" 0; - "~*(?:\b)Mediapartners-Google(?:\b|)" 0; - "~*(?:\b)msnbot(?:\b|)" 0; - "~*(?:\b)msnbot-media(?:\b|)" 0; - "~*(?:\b)SAMSUNG(?:\b|)" 0; - "~*(?:\b)Slackbot(?:\b|)" 0; - "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; - "~*(?:\b)slurp(?:\b|)" 0; - "~*(?:\b)teoma(?:\b|)" 0; - "~*(?:\b)TwitterBot(?:\b|)" 0; - "~*(?:\b)Wordpress(?:\b|)" 0; - "~*(?:\b)yahoo(?:\b|)" 0; + "~*(?:\b)adidxbot(?:\b)" 0; + "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bingbot(?:\b)" 0; + "~*(?:\b)bingpreview(?:\b)" 0; + "~*(?:\b)DoCoMo(?:\b)" 0; + "~*(?:\b)duckduckgo(?:\b)" 0; + "~*(?:\b)facebookexternalhit(?:\b)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b)" 0; + "~*(?:\b)Googlebot(?:\b)" 0; + "~*(?:\b)Googlebot-Image(?:\b)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b)" 0; + "~*(?:\b)Googlebot-News(?:\b)" 0; + "~*(?:\b)Googlebot/Test(?:\b)" 0; + "~*(?:\b)Googlebot-Video(?:\b)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; + "~*(?:\b)Gravityscan(?:\b)" 0; + "~*(?:\b)gsa-crawler(?:\b)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b)" 0; + "~*(?:\b)Kraken/0.1(?:\b)" 0; + "~*(?:\b)LinkedInBot(?:\b)" 0; + "~*(?:\b)Mediapartners-Google(?:\b)" 0; + "~*(?:\b)msnbot(?:\b)" 0; + "~*(?:\b)msnbot-media(?:\b)" 0; + "~*(?:\b)SAMSUNG(?:\b)" 0; + "~*(?:\b)Slackbot(?:\b)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 0; + "~*(?:\b)slurp(?:\b)" 0; + "~*(?:\b)teoma(?:\b)" 0; + "~*(?:\b)TwitterBot(?:\b)" 0; + "~*(?:\b)Wordpress(?:\b)" 0; + "~*(?:\b)yahoo(?:\b)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------------------- @@ -760,12 +760,12 @@ map $http_user_agent $bad_bot { #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)jetmon(?:\b|)" 1; - "~*(?:\b)libwww-perl(?:\b|)" 1; - "~*(?:\b)Lynx(?:\b|)" 1; - "~*(?:\b)munin(?:\b|)" 1; - "~*(?:\b)Presto(?:\b|)" 1; - "~*(?:\b)Wget/1.15(?:\b|)" 1; + "~*(?:\b)jetmon(?:\b)" 1; + "~*(?:\b)libwww-perl(?:\b)" 1; + "~*(?:\b)Lynx(?:\b)" 1; + "~*(?:\b)munin(?:\b)" 1; + "~*(?:\b)Presto(?:\b)" 1; + "~*(?:\b)Wget/1.15(?:\b)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ------------------------------------------------------- @@ -778,23 +778,23 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)Alexa(?:\b|)" 2; - "~*(?:\b)archive.org(?:\b|)" 2; - "~*(?:\b)Baidu(?:\b|)" 2; - "~*(?:\b)BUbiNG(?:\b|)" 2; - "~*(?:\b)FlipboardProxy(?:\b|)" 2; - "~*(?:\b)ia_archiver(?:\b|)" 2; - "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; - "~*(?:\b)Proximic(?:\b|)" 2; - "~*(?:\b)R6_CommentReader(?:\b|)" 2; - "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; - "~*(?:\b)RED/1(?:\b|)" 2; - "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; - "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; - "~*(?:\b)Spaidu(?:\b|)" 2; - "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; - "~*(?:\b)YandexBot(?:\b|)" 2; - "~*(?:\b)YandexImages(?:\b|)" 2; + "~*(?:\b)Alexa(?:\b)" 2; + "~*(?:\b)archive.org(?:\b)" 2; + "~*(?:\b)Baidu(?:\b)" 2; + "~*(?:\b)BUbiNG(?:\b)" 2; + "~*(?:\b)FlipboardProxy(?:\b)" 2; + "~*(?:\b)ia_archiver(?:\b)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b)" 2; + "~*(?:\b)Proximic(?:\b)" 2; + "~*(?:\b)R6_CommentReader(?:\b)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b)" 2; + "~*(?:\b)RED/1(?:\b)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b)" 2; + "~*(?:\b)Spaidu(?:\b)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b)" 2; + "~*(?:\b)YandexBot(?:\b)" 2; + "~*(?:\b)YandexImages(?:\b)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf index 92e5004d6..ebe757f93 100644 --- a/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test4_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.154.113.151 0; +35.192.136.167 0; 34.66.178.120 0; -35.192.187.174 0; +34.68.144.114 0; diff --git a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf index f26e52add..a1999739a 100644 --- a/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test4_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1709 -### Updated: Mon Jul 1 10:37:04 SAST 2019 +### Version: V4.2019.07.1712 +### Updated: Mon Jul 1 11:24:52 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### @@ -147,562 +147,562 @@ map $http_user_agent $bad_bot { # -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b|)" 3; - "~*(?:\b)404checker(?:\b|)" 3; - "~*(?:\b)404enemy(?:\b|)" 3; - "~*(?:\b)80legs(?:\b|)" 3; - "~*(?:\b)Abonti(?:\b|)" 3; - "~*(?:\b)Aboundex(?:\b|)" 3; - "~*(?:\b)Aboundexbot(?:\b|)" 3; - "~*(?:\b)Acunetix(?:\b|)" 3; - "~*(?:\b)ADmantX(?:\b|)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; - "~*(?:\b)AhrefsBot(?:\b|)" 3; - "~*(?:\b)AIBOT(?:\b|)" 3; - "~*(?:\b)AiHitBot(?:\b|)" 3; - "~*(?:\b)Aipbot(?:\b|)" 3; - "~*(?:\b)Alexibot(?:\b|)" 3; - "~*(?:\b)Alligator(?:\b|)" 3; - "~*(?:\b)AllSubmitter(?:\b|)" 3; - "~*(?:\b)AlphaBot(?:\b|)" 3; - "~*(?:\b)Anarchie(?:\b|)" 3; - "~*(?:\b)Apexoo(?:\b|)" 3; - "~*(?:\b)archive.org_bot(?:\b|)" 3; - "~*(?:\b)ASPSeek(?:\b|)" 3; - "~*(?:\b)Asterias(?:\b|)" 3; - "~*(?:\b)Attach(?:\b|)" 3; - "~*(?:\b)autoemailspider(?:\b|)" 3; - "~*(?:\b)BackDoorBot(?:\b|)" 3; - "~*(?:\b)Backlink-Ceck(?:\b|)" 3; - "~*(?:\b)backlink-check(?:\b|)" 3; - "~*(?:\b)BacklinkCrawler(?:\b|)" 3; - "~*(?:\b)BackStreet(?:\b|)" 3; - "~*(?:\b)BackWeb(?:\b|)" 3; - "~*(?:\b)Badass(?:\b|)" 3; - "~*(?:\b)Bandit(?:\b|)" 3; - "~*(?:\b)Barkrowler(?:\b|)" 3; - "~*(?:\b)BatchFTP(?:\b|)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; - "~*(?:\b)BBBike(?:\b|)" 3; - "~*(?:\b)BDCbot(?:\b|)" 3; - "~*(?:\b)BDFetch(?:\b|)" 3; - "~*(?:\b)BetaBot(?:\b|)" 3; - "~*(?:\b)Bigfoot(?:\b|)" 3; - "~*(?:\b)Bitacle(?:\b|)" 3; - "~*(?:\b)Blackboard(?:\b|)" 3; - "~*(?:\b)Black\ Hole(?:\b|)" 3; - "~*(?:\b)BlackWidow(?:\b|)" 3; - "~*(?:\b)BLEXBot(?:\b|)" 3; - "~*(?:\b)Blow(?:\b|)" 3; - "~*(?:\b)BlowFish(?:\b|)" 3; - "~*(?:\b)Boardreader(?:\b|)" 3; - "~*(?:\b)Bolt(?:\b|)" 3; - "~*(?:\b)BotALot(?:\b|)" 3; - "~*(?:\b)Brandprotect(?:\b|)" 3; - "~*(?:\b)Brandwatch(?:\b|)" 3; - "~*(?:\b)Buddy(?:\b|)" 3; - "~*(?:\b)BuiltBotTough(?:\b|)" 3; - "~*(?:\b)BuiltWith(?:\b|)" 3; - "~*(?:\b)Bullseye(?:\b|)" 3; - "~*(?:\b)BunnySlippers(?:\b|)" 3; - "~*(?:\b)BuzzSumo(?:\b|)" 3; - "~*(?:\b)Calculon(?:\b|)" 3; - "~*(?:\b)CATExplorador(?:\b|)" 3; - "~*(?:\b)CazoodleBot(?:\b|)" 3; - "~*(?:\b)CCBot(?:\b|)" 3; - "~*(?:\b)Cegbfeieh(?:\b|)" 3; - "~*(?:\b)CheeseBot(?:\b|)" 3; - "~*(?:\b)CherryPicker(?:\b|)" 3; - "~*(?:\b)CheTeam(?:\b|)" 3; - "~*(?:\b)ChinaClaw(?:\b|)" 3; - "~*(?:\b)Chlooe(?:\b|)" 3; - "~*(?:\b)Claritybot(?:\b|)" 3; - "~*(?:\b)Cliqzbot(?:\b|)" 3; - "~*(?:\b)Cloud\ mapping(?:\b|)" 3; - "~*(?:\b)coccocbot-web(?:\b|)" 3; - "~*(?:\b)Cogentbot(?:\b|)" 3; - "~*(?:\b)cognitiveseo(?:\b|)" 3; - "~*(?:\b)Collector(?:\b|)" 3; - "~*(?:\b)com.plumanalytics(?:\b|)" 3; - "~*(?:\b)Copier(?:\b|)" 3; - "~*(?:\b)CopyRightCheck(?:\b|)" 3; - "~*(?:\b)Copyscape(?:\b|)" 3; - "~*(?:\b)Cosmos(?:\b|)" 3; - "~*(?:\b)Craftbot(?:\b|)" 3; - "~*(?:\b)crawler4j(?:\b|)" 3; - "~*(?:\b)crawler.feedback(?:\b|)" 3; - "~*(?:\b)crawl.sogou.com(?:\b|)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; - "~*(?:\b)Crescent(?:\b|)" 3; - "~*(?:\b)CrunchBot(?:\b|)" 3; - "~*(?:\b)CSHttp(?:\b|)" 3; - "~*(?:\b)Curious(?:\b|)" 3; - "~*(?:\b)Custo(?:\b|)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; - "~*(?:\b)DataCha0s(?:\b|)" 3; - "~*(?:\b)DBLBot(?:\b|)" 3; - "~*(?:\b)demandbase-bot(?:\b|)" 3; - "~*(?:\b)Demon(?:\b|)" 3; - "~*(?:\b)Deusu(?:\b|)" 3; - "~*(?:\b)Devil(?:\b|)" 3; - "~*(?:\b)Digincore(?:\b|)" 3; - "~*(?:\b)DigitalPebble(?:\b|)" 3; - "~*(?:\b)DIIbot(?:\b|)" 3; - "~*(?:\b)Dirbuster(?:\b|)" 3; - "~*(?:\b)Disco(?:\b|)" 3; - "~*(?:\b)Discobot(?:\b|)" 3; - "~*(?:\b)Discoverybot(?:\b|)" 3; - "~*(?:\b)Dispatch(?:\b|)" 3; - "~*(?:\b)DittoSpyder(?:\b|)" 3; - "~*(?:\b)DnyzBot(?:\b|)" 3; - "~*(?:\b)DomainAppender(?:\b|)" 3; - "~*(?:\b)DomainCrawler(?:\b|)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; - "~*(?:\b)DomainStatsBot(?:\b|)" 3; - "~*(?:\b)Dotbot(?:\b|)" 3; - "~*(?:\b)Download\ Wonder(?:\b|)" 3; - "~*(?:\b)Dragonfly(?:\b|)" 3; - "~*(?:\b)Drip(?:\b|)" 3; - "~*(?:\b)DSearch(?:\b|)" 3; - "~*(?:\b)DTS\ Agent(?:\b|)" 3; - "~*(?:\b)EasyDL(?:\b|)" 3; - "~*(?:\b)Ebingbong(?:\b|)" 3; - "~*(?:\b)eCatch(?:\b|)" 3; - "~*(?:\b)ECCP/1.0(?:\b|)" 3; - "~*(?:\b)Ecxi(?:\b|)" 3; - "~*(?:\b)EirGrabber(?:\b|)" 3; - "~*(?:\b)EMail\ Siphon(?:\b|)" 3; - "~*(?:\b)EMail\ Wolf(?:\b|)" 3; - "~*(?:\b)EroCrawler(?:\b|)" 3; - "~*(?:\b)evc-batch(?:\b|)" 3; - "~*(?:\b)Evil(?:\b|)" 3; - "~*(?:\b)Exabot(?:\b|)" 3; - "~*(?:\b)Express\ WebPictures(?:\b|)" 3; - "~*(?:\b)ExtLinksBot(?:\b|)" 3; - "~*(?:\b)Extractor(?:\b|)" 3; - "~*(?:\b)ExtractorPro(?:\b|)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; - "~*(?:\b)EyeNetIE(?:\b|)" 3; - "~*(?:\b)Ezooms(?:\b|)" 3; - "~*(?:\b)facebookscraper(?:\b|)" 3; - "~*(?:\b)FDM(?:\b|)" 3; - "~*(?:\b)FemtosearchBot(?:\b|)" 3; - "~*(?:\b)FHscan(?:\b|)" 3; - "~*(?:\b)Fimap(?:\b|)" 3; - "~*(?:\b)Firefox/7.0(?:\b|)" 3; - "~*(?:\b)FlashGet(?:\b|)" 3; - "~*(?:\b)Flunky(?:\b|)" 3; - "~*(?:\b)Foobot(?:\b|)" 3; - "~*(?:\b)Freeuploader(?:\b|)" 3; - "~*(?:\b)FrontPage(?:\b|)" 3; - "~*(?:\b)FyberSpider(?:\b|)" 3; - "~*(?:\b)Fyrebot(?:\b|)" 3; - "~*(?:\b)GalaxyBot(?:\b|)" 3; - "~*(?:\b)Genieo(?:\b|)" 3; - "~*(?:\b)GermCrawler(?:\b|)" 3; - "~*(?:\b)Getintent(?:\b|)" 3; - "~*(?:\b)GetRight(?:\b|)" 3; - "~*(?:\b)GetWeb(?:\b|)" 3; - "~*(?:\b)Gigablast(?:\b|)" 3; - "~*(?:\b)Gigabot(?:\b|)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; - "~*(?:\b)Gotit(?:\b|)" 3; - "~*(?:\b)GoZilla(?:\b|)" 3; - "~*(?:\b)Go!Zilla(?:\b|)" 3; - "~*(?:\b)Grabber(?:\b|)" 3; - "~*(?:\b)GrabNet(?:\b|)" 3; - "~*(?:\b)Grafula(?:\b|)" 3; - "~*(?:\b)GrapeFX(?:\b|)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; - "~*(?:\b)GridBot(?:\b|)" 3; - "~*(?:\b)GT::WWW(?:\b|)" 3; - "~*(?:\b)Haansoft(?:\b|)" 3; - "~*(?:\b)HaosouSpider(?:\b|)" 3; - "~*(?:\b)Harvest(?:\b|)" 3; - "~*(?:\b)Havij(?:\b|)" 3; - "~*(?:\b)HEADMasterSEO(?:\b|)" 3; - "~*(?:\b)Heritrix(?:\b|)" 3; - "~*(?:\b)Hloader(?:\b|)" 3; - "~*(?:\b)HMView(?:\b|)" 3; - "~*(?:\b)HTMLparser(?:\b|)" 3; - "~*(?:\b)HTTP::Lite(?:\b|)" 3; - "~*(?:\b)HTTrack(?:\b|)" 3; - "~*(?:\b)Humanlinks(?:\b|)" 3; - "~*(?:\b)HybridBot(?:\b|)" 3; - "~*(?:\b)Iblog(?:\b|)" 3; - "~*(?:\b)IDBot(?:\b|)" 3; - "~*(?:\b)Id-search(?:\b|)" 3; - "~*(?:\b)IlseBot(?:\b|)" 3; - "~*(?:\b)Image\ Fetch(?:\b|)" 3; - "~*(?:\b)Image\ Sucker(?:\b|)" 3; - "~*(?:\b)IndeedBot(?:\b|)" 3; - "~*(?:\b)Indy\ Library(?:\b|)" 3; - "~*(?:\b)InfoNaviRobot(?:\b|)" 3; - "~*(?:\b)InfoTekies(?:\b|)" 3; - "~*(?:\b)instabid(?:\b|)" 3; - "~*(?:\b)Intelliseek(?:\b|)" 3; - "~*(?:\b)InterGET(?:\b|)" 3; - "~*(?:\b)Internet\ Ninja(?:\b|)" 3; - "~*(?:\b)InternetSeer(?:\b|)" 3; - "~*(?:\b)internetVista\ monitor(?:\b|)" 3; - "~*(?:\b)ips-agent(?:\b|)" 3; - "~*(?:\b)Iria(?:\b|)" 3; - "~*(?:\b)IRLbot(?:\b|)" 3; - "~*(?:\b)Iskanie(?:\b|)" 3; - "~*(?:\b)IstellaBot(?:\b|)" 3; - "~*(?:\b)JamesBOT(?:\b|)" 3; - "~*(?:\b)Jbrofuzz(?:\b|)" 3; - "~*(?:\b)JennyBot(?:\b|)" 3; - "~*(?:\b)JetCar(?:\b|)" 3; - "~*(?:\b)Jetty(?:\b|)" 3; - "~*(?:\b)JikeSpider(?:\b|)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; - "~*(?:\b)Joomla(?:\b|)" 3; - "~*(?:\b)Jorgee(?:\b|)" 3; - "~*(?:\b)JustView(?:\b|)" 3; - "~*(?:\b)Jyxobot(?:\b|)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; - "~*(?:\b)Keyword\ Density(?:\b|)" 3; - "~*(?:\b)Kozmosbot(?:\b|)" 3; - "~*(?:\b)Lanshanbot(?:\b|)" 3; - "~*(?:\b)Larbin(?:\b|)" 3; - "~*(?:\b)LeechFTP(?:\b|)" 3; - "~*(?:\b)LeechGet(?:\b|)" 3; - "~*(?:\b)LexiBot(?:\b|)" 3; - "~*(?:\b)Lftp(?:\b|)" 3; - "~*(?:\b)LibWeb(?:\b|)" 3; - "~*(?:\b)Libwhisker(?:\b|)" 3; - "~*(?:\b)Lightspeedsystems(?:\b|)" 3; - "~*(?:\b)Likse(?:\b|)" 3; - "~*(?:\b)Linkdexbot(?:\b|)" 3; - "~*(?:\b)LinkextractorPro(?:\b|)" 3; - "~*(?:\b)LinkpadBot(?:\b|)" 3; - "~*(?:\b)LinkScan(?:\b|)" 3; - "~*(?:\b)LinksManager(?:\b|)" 3; - "~*(?:\b)LinkWalker(?:\b|)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; - "~*(?:\b)Lipperhey(?:\b|)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; - "~*(?:\b)Litemage_walker(?:\b|)" 3; - "~*(?:\b)Lmspider(?:\b|)" 3; - "~*(?:\b)LNSpiderguy(?:\b|)" 3; - "~*(?:\b)Ltx71(?:\b|)" 3; - "~*(?:\b)lwp-request(?:\b|)" 3; - "~*(?:\b)LWP::Simple(?:\b|)" 3; - "~*(?:\b)lwp-trivial(?:\b|)" 3; - "~*(?:\b)Magnet(?:\b|)" 3; - "~*(?:\b)Mag-Net(?:\b|)" 3; - "~*(?:\b)magpie-crawler(?:\b|)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; - "~*(?:\b)Majestic12(?:\b|)" 3; - "~*(?:\b)Majestic-SEO(?:\b|)" 3; - "~*(?:\b)Majestic\ SEO(?:\b|)" 3; - "~*(?:\b)MarkMonitor(?:\b|)" 3; - "~*(?:\b)MarkWatch(?:\b|)" 3; - "~*(?:\b)Masscan(?:\b|)" 3; - "~*(?:\b)Mass\ Downloader(?:\b|)" 3; - "~*(?:\b)Mata\ Hari(?:\b|)" 3; - "~*(?:\b)MauiBot(?:\b|)" 3; - "~*(?:\b)meanpathbot(?:\b|)" 3; - "~*(?:\b)Meanpathbot(?:\b|)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; - "~*(?:\b)mediawords(?:\b|)" 3; - "~*(?:\b)MegaIndex.ru(?:\b|)" 3; - "~*(?:\b)Metauri(?:\b|)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; - "~*(?:\b)MIDown\ tool(?:\b|)" 3; - "~*(?:\b)MIIxpc(?:\b|)" 3; - "~*(?:\b)Mister\ PiX(?:\b|)" 3; - "~*(?:\b)MJ12bot(?:\b|)" 3; - "~*(?:\b)Mojeek(?:\b|)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; - "~*(?:\b)Mr.4x3(?:\b|)" 3; - "~*(?:\b)MSFrontPage(?:\b|)" 3; - "~*(?:\b)MSIECrawler(?:\b|)" 3; - "~*(?:\b)Msrabot(?:\b|)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; - "~*(?:\b)muhstik-scan(?:\b|)" 3; - "~*(?:\b)Musobot(?:\b|)" 3; - "~*(?:\b)Name\ Intelligence(?:\b|)" 3; - "~*(?:\b)Nameprotect(?:\b|)" 3; - "~*(?:\b)Navroad(?:\b|)" 3; - "~*(?:\b)NearSite(?:\b|)" 3; - "~*(?:\b)Needle(?:\b|)" 3; - "~*(?:\b)Nessus(?:\b|)" 3; - "~*(?:\b)NetAnts(?:\b|)" 3; - "~*(?:\b)Netcraft(?:\b|)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; - "~*(?:\b)NetLyzer(?:\b|)" 3; - "~*(?:\b)NetMechanic(?:\b|)" 3; - "~*(?:\b)NetSpider(?:\b|)" 3; - "~*(?:\b)Nettrack(?:\b|)" 3; - "~*(?:\b)Net\ Vampire(?:\b|)" 3; - "~*(?:\b)Netvibes(?:\b|)" 3; - "~*(?:\b)NetZIP(?:\b|)" 3; - "~*(?:\b)NextGenSearchBot(?:\b|)" 3; - "~*(?:\b)Nibbler(?:\b|)" 3; - "~*(?:\b)NICErsPRO(?:\b|)" 3; - "~*(?:\b)Niki-bot(?:\b|)" 3; - "~*(?:\b)Nikto(?:\b|)" 3; - "~*(?:\b)NimbleCrawler(?:\b|)" 3; - "~*(?:\b)Nimbostratus(?:\b|)" 3; - "~*(?:\b)Ninja(?:\b|)" 3; - "~*(?:\b)Nmap(?:\b|)" 3; - "~*(?:\b)NPbot(?:\b|)" 3; - "~*(?:\b)Nutch(?:\b|)" 3; - "~*(?:\b)oBot(?:\b|)" 3; - "~*(?:\b)Octopus(?:\b|)" 3; - "~*(?:\b)Offline\ Explorer(?:\b|)" 3; - "~*(?:\b)Offline\ Navigator(?:\b|)" 3; - "~*(?:\b)OnCrawl(?:\b|)" 3; - "~*(?:\b)Openfind(?:\b|)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; - "~*(?:\b)Openvas(?:\b|)" 3; - "~*(?:\b)OpenVAS(?:\b|)" 3; - "~*(?:\b)OrangeBot(?:\b|)" 3; - "~*(?:\b)OrangeSpider(?:\b|)" 3; - "~*(?:\b)OutclicksBot(?:\b|)" 3; - "~*(?:\b)OutfoxBot(?:\b|)" 3; - "~*(?:\b)PageAnalyzer(?:\b|)" 3; - "~*(?:\b)Page\ Analyzer(?:\b|)" 3; - "~*(?:\b)PageGrabber(?:\b|)" 3; - "~*(?:\b)page\ scorer(?:\b|)" 3; - "~*(?:\b)PageScorer(?:\b|)" 3; - "~*(?:\b)Pandalytics(?:\b|)" 3; - "~*(?:\b)Panscient(?:\b|)" 3; - "~*(?:\b)Papa\ Foto(?:\b|)" 3; - "~*(?:\b)Pavuk(?:\b|)" 3; - "~*(?:\b)pcBrowser(?:\b|)" 3; - "~*(?:\b)PECL::HTTP(?:\b|)" 3; - "~*(?:\b)PeoplePal(?:\b|)" 3; - "~*(?:\b)PHPCrawl(?:\b|)" 3; - "~*(?:\b)Picscout(?:\b|)" 3; - "~*(?:\b)Picsearch(?:\b|)" 3; - "~*(?:\b)PictureFinder(?:\b|)" 3; - "~*(?:\b)Pimonster(?:\b|)" 3; - "~*(?:\b)Pi-Monster(?:\b|)" 3; - "~*(?:\b)Pixray(?:\b|)" 3; - "~*(?:\b)PleaseCrawl(?:\b|)" 3; - "~*(?:\b)plumanalytics(?:\b|)" 3; - "~*(?:\b)Pockey(?:\b|)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; - "~*(?:\b)Probethenet(?:\b|)" 3; - "~*(?:\b)ProPowerBot(?:\b|)" 3; - "~*(?:\b)ProWebWalker(?:\b|)" 3; - "~*(?:\b)Psbot(?:\b|)" 3; - "~*(?:\b)Pump(?:\b|)" 3; - "~*(?:\b)PxBroker(?:\b|)" 3; - "~*(?:\b)PyCurl(?:\b|)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; - "~*(?:\b)Quick-Crawler(?:\b|)" 3; - "~*(?:\b)RankActive(?:\b|)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; - "~*(?:\b)RankFlex(?:\b|)" 3; - "~*(?:\b)RankingBot(?:\b|)" 3; - "~*(?:\b)RankingBot2(?:\b|)" 3; - "~*(?:\b)Rankivabot(?:\b|)" 3; - "~*(?:\b)RankurBot(?:\b|)" 3; - "~*(?:\b)RealDownload(?:\b|)" 3; - "~*(?:\b)Reaper(?:\b|)" 3; - "~*(?:\b)RebelMouse(?:\b|)" 3; - "~*(?:\b)Recorder(?:\b|)" 3; - "~*(?:\b)RedesScrapy(?:\b|)" 3; - "~*(?:\b)ReGet(?:\b|)" 3; - "~*(?:\b)RepoMonkey(?:\b|)" 3; - "~*(?:\b)Ripper(?:\b|)" 3; - "~*(?:\b)RocketCrawler(?:\b|)" 3; - "~*(?:\b)Rogerbot(?:\b|)" 3; - "~*(?:\b)s1z.ru(?:\b|)" 3; - "~*(?:\b)SalesIntelligent(?:\b|)" 3; - "~*(?:\b)SBIder(?:\b|)" 3; - "~*(?:\b)ScanAlert(?:\b|)" 3; - "~*(?:\b)Scanbot(?:\b|)" 3; - "~*(?:\b)scan.lol(?:\b|)" 3; - "~*(?:\b)ScoutJet(?:\b|)" 3; - "~*(?:\b)Scrapy(?:\b|)" 3; - "~*(?:\b)Screaming(?:\b|)" 3; - "~*(?:\b)ScreenerBot(?:\b|)" 3; - "~*(?:\b)Searchestate(?:\b|)" 3; - "~*(?:\b)SearchmetricsBot(?:\b|)" 3; - "~*(?:\b)Semrush(?:\b|)" 3; - "~*(?:\b)SemrushBot(?:\b|)" 3; - "~*(?:\b)SEOkicks(?:\b|)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; - "~*(?:\b)Seomoz(?:\b|)" 3; - "~*(?:\b)SEOprofiler(?:\b|)" 3; - "~*(?:\b)seoscanners(?:\b|)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; - "~*(?:\b)SEOstats(?:\b|)" 3; - "~*(?:\b)serpstatbot(?:\b|)" 3; - "~*(?:\b)sexsearcher(?:\b|)" 3; - "~*(?:\b)Shodan(?:\b|)" 3; - "~*(?:\b)Siphon(?:\b|)" 3; - "~*(?:\b)SISTRIX(?:\b|)" 3; - "~*(?:\b)Sitebeam(?:\b|)" 3; - "~*(?:\b)SiteExplorer(?:\b|)" 3; - "~*(?:\b)Siteimprove(?:\b|)" 3; - "~*(?:\b)SiteLockSpider(?:\b|)" 3; - "~*(?:\b)SiteSnagger(?:\b|)" 3; - "~*(?:\b)SiteSucker(?:\b|)" 3; - "~*(?:\b)Site\ Sucker(?:\b|)" 3; - "~*(?:\b)Sitevigil(?:\b|)" 3; - "~*(?:\b)SlySearch(?:\b|)" 3; - "~*(?:\b)SmartDownload(?:\b|)" 3; - "~*(?:\b)SMTBot(?:\b|)" 3; - "~*(?:\b)Snake(?:\b|)" 3; - "~*(?:\b)Snapbot(?:\b|)" 3; - "~*(?:\b)Snoopy(?:\b|)" 3; - "~*(?:\b)SocialRankIOBot(?:\b|)" 3; - "~*(?:\b)Sociscraper(?:\b|)" 3; - "~*(?:\b)sogouspider(?:\b|)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; - "~*(?:\b)Sosospider(?:\b|)" 3; - "~*(?:\b)Sottopop(?:\b|)" 3; - "~*(?:\b)SpaceBison(?:\b|)" 3; - "~*(?:\b)Spammen(?:\b|)" 3; - "~*(?:\b)SpankBot(?:\b|)" 3; - "~*(?:\b)Spanner(?:\b|)" 3; - "~*(?:\b)sp_auditbot(?:\b|)" 3; - "~*(?:\b)Spbot(?:\b|)" 3; - "~*(?:\b)Spinn3r(?:\b|)" 3; - "~*(?:\b)SputnikBot(?:\b|)" 3; - "~*(?:\b)spyfu(?:\b|)" 3; - "~*(?:\b)Sqlmap(?:\b|)" 3; - "~*(?:\b)Sqlworm(?:\b|)" 3; - "~*(?:\b)Sqworm(?:\b|)" 3; - "~*(?:\b)Steeler(?:\b|)" 3; - "~*(?:\b)Stripper(?:\b|)" 3; - "~*(?:\b)Sucker(?:\b|)" 3; - "~*(?:\b)Sucuri(?:\b|)" 3; - "~*(?:\b)SuperBot(?:\b|)" 3; - "~*(?:\b)SuperHTTP(?:\b|)" 3; - "~*(?:\b)Surfbot(?:\b|)" 3; - "~*(?:\b)SurveyBot(?:\b|)" 3; - "~*(?:\b)Suzuran(?:\b|)" 3; - "~*(?:\b)Swiftbot(?:\b|)" 3; - "~*(?:\b)sysscan(?:\b|)" 3; - "~*(?:\b)Szukacz(?:\b|)" 3; - "~*(?:\b)T0PHackTeam(?:\b|)" 3; - "~*(?:\b)T8Abot(?:\b|)" 3; - "~*(?:\b)tAkeOut(?:\b|)" 3; - "~*(?:\b)Teleport(?:\b|)" 3; - "~*(?:\b)TeleportPro(?:\b|)" 3; - "~*(?:\b)Telesoft(?:\b|)" 3; - "~*(?:\b)Telesphoreo(?:\b|)" 3; - "~*(?:\b)Telesphorep(?:\b|)" 3; - "~*(?:\b)The\ Intraformant(?:\b|)" 3; - "~*(?:\b)TheNomad(?:\b|)" 3; - "~*(?:\b)Thumbor(?:\b|)" 3; - "~*(?:\b)TightTwatBot(?:\b|)" 3; - "~*(?:\b)Titan(?:\b|)" 3; - "~*(?:\b)Toata(?:\b|)" 3; - "~*(?:\b)Toweyabot(?:\b|)" 3; - "~*(?:\b)Tracemyfile(?:\b|)" 3; - "~*(?:\b)Trendiction(?:\b|)" 3; - "~*(?:\b)Trendictionbot(?:\b|)" 3; - "~*(?:\b)trendiction.com(?:\b|)" 3; - "~*(?:\b)trendiction.de(?:\b|)" 3; - "~*(?:\b)True_Robot(?:\b|)" 3; - "~*(?:\b)Turingos(?:\b|)" 3; - "~*(?:\b)Turnitin(?:\b|)" 3; - "~*(?:\b)TurnitinBot(?:\b|)" 3; - "~*(?:\b)TwengaBot(?:\b|)" 3; - "~*(?:\b)Twice(?:\b|)" 3; - "~*(?:\b)Typhoeus(?:\b|)" 3; - "~*(?:\b)UnisterBot(?:\b|)" 3; - "~*(?:\b)Upflow(?:\b|)" 3; - "~*(?:\b)URLy.Warning(?:\b|)" 3; - "~*(?:\b)URLy\ Warning(?:\b|)" 3; - "~*(?:\b)Vacuum(?:\b|)" 3; - "~*(?:\b)Vagabondo(?:\b|)" 3; - "~*(?:\b)VB\ Project(?:\b|)" 3; - "~*(?:\b)VCI(?:\b|)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; - "~*(?:\b)VidibleScraper(?:\b|)" 3; - "~*(?:\b)Virusdie(?:\b|)" 3; - "~*(?:\b)VoidEYE(?:\b|)" 3; - "~*(?:\b)Voil(?:\b|)" 3; - "~*(?:\b)Voltron(?:\b|)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; - "~*(?:\b)WallpapersHD(?:\b|)" 3; - "~*(?:\b)WASALive-Bot(?:\b|)" 3; - "~*(?:\b)WBSearchBot(?:\b|)" 3; - "~*(?:\b)Webalta(?:\b|)" 3; - "~*(?:\b)WebAuto(?:\b|)" 3; - "~*(?:\b)Web\ Auto(?:\b|)" 3; - "~*(?:\b)WebBandit(?:\b|)" 3; - "~*(?:\b)WebCollage(?:\b|)" 3; - "~*(?:\b)Web\ Collage(?:\b|)" 3; - "~*(?:\b)WebCopier(?:\b|)" 3; - "~*(?:\b)WEBDAV(?:\b|)" 3; - "~*(?:\b)WebEnhancer(?:\b|)" 3; - "~*(?:\b)Web\ Enhancer(?:\b|)" 3; - "~*(?:\b)WebFetch(?:\b|)" 3; - "~*(?:\b)Web\ Fetch(?:\b|)" 3; - "~*(?:\b)WebFuck(?:\b|)" 3; - "~*(?:\b)Web\ Fuck(?:\b|)" 3; - "~*(?:\b)WebGo\ IS(?:\b|)" 3; - "~*(?:\b)WebImageCollector(?:\b|)" 3; - "~*(?:\b)WebLeacher(?:\b|)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; - "~*(?:\b)webmeup-crawler(?:\b|)" 3; - "~*(?:\b)WebPix(?:\b|)" 3; - "~*(?:\b)Web\ Pix(?:\b|)" 3; - "~*(?:\b)WebReaper(?:\b|)" 3; - "~*(?:\b)WebSauger(?:\b|)" 3; - "~*(?:\b)Web\ Sauger(?:\b|)" 3; - "~*(?:\b)Webshag(?:\b|)" 3; - "~*(?:\b)WebsiteExtractor(?:\b|)" 3; - "~*(?:\b)WebsiteQuester(?:\b|)" 3; - "~*(?:\b)Website\ Quester(?:\b|)" 3; - "~*(?:\b)Webster(?:\b|)" 3; - "~*(?:\b)WebStripper(?:\b|)" 3; - "~*(?:\b)WebSucker(?:\b|)" 3; - "~*(?:\b)Web\ Sucker(?:\b|)" 3; - "~*(?:\b)WebWhacker(?:\b|)" 3; - "~*(?:\b)WebZIP(?:\b|)" 3; - "~*(?:\b)WeSEE(?:\b|)" 3; - "~*(?:\b)Whack(?:\b|)" 3; - "~*(?:\b)Whacker(?:\b|)" 3; - "~*(?:\b)Whatweb(?:\b|)" 3; - "~*(?:\b)Who.is\ Bot(?:\b|)" 3; - "~*(?:\b)Widow(?:\b|)" 3; - "~*(?:\b)WinHTTrack(?:\b|)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; - "~*(?:\b)WISENutbot(?:\b|)" 3; - "~*(?:\b)Wonderbot(?:\b|)" 3; - "~*(?:\b)Woobot(?:\b|)" 3; - "~*(?:\b)Wotbox(?:\b|)" 3; - "~*(?:\b)Wprecon(?:\b|)" 3; - "~*(?:\b)WPScan(?:\b|)" 3; - "~*(?:\b)WWW-Collector-E(?:\b|)" 3; - "~*(?:\b)WWW-Mechanize(?:\b|)" 3; - "~*(?:\b)WWW::Mechanize(?:\b|)" 3; - "~*(?:\b)WWWOFFLE(?:\b|)" 3; - "~*(?:\b)x09Mozilla(?:\b|)" 3; - "~*(?:\b)x22Mozilla(?:\b|)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; - "~*(?:\b)Xenu(?:\b|)" 3; - "~*(?:\b)xpymep1.exe(?:\b|)" 3; - "~*(?:\b)YoudaoBot(?:\b|)" 3; - "~*(?:\b)Zade(?:\b|)" 3; - "~*(?:\b)Zauba(?:\b|)" 3; - "~*(?:\b)zauba.io(?:\b|)" 3; - "~*(?:\b)Zermelo(?:\b|)" 3; - "~*(?:\b)Zeus(?:\b|)" 3; - "~*(?:\b)zgrab(?:\b|)" 3; - "~*(?:\b)Zitebot(?:\b|)" 3; - "~*(?:\b)ZmEu(?:\b|)" 3; - "~*(?:\b)ZumBot(?:\b|)" 3; - "~*(?:\b)ZyBorg(?:\b|)" 3; + "~*(?:\b)360Spider(?:\b)" 3; + "~*(?:\b)404checker(?:\b)" 3; + "~*(?:\b)404enemy(?:\b)" 3; + "~*(?:\b)80legs(?:\b)" 3; + "~*(?:\b)Abonti(?:\b)" 3; + "~*(?:\b)Aboundex(?:\b)" 3; + "~*(?:\b)Aboundexbot(?:\b)" 3; + "~*(?:\b)Acunetix(?:\b)" 3; + "~*(?:\b)ADmantX(?:\b)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; + "~*(?:\b)AhrefsBot(?:\b)" 3; + "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)AiHitBot(?:\b)" 3; + "~*(?:\b)Aipbot(?:\b)" 3; + "~*(?:\b)Alexibot(?:\b)" 3; + "~*(?:\b)Alligator(?:\b)" 3; + "~*(?:\b)AllSubmitter(?:\b)" 3; + "~*(?:\b)AlphaBot(?:\b)" 3; + "~*(?:\b)Anarchie(?:\b)" 3; + "~*(?:\b)Apexoo(?:\b)" 3; + "~*(?:\b)archive.org_bot(?:\b)" 3; + "~*(?:\b)ASPSeek(?:\b)" 3; + "~*(?:\b)Asterias(?:\b)" 3; + "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)BackDoorBot(?:\b)" 3; + "~*(?:\b)Backlink-Ceck(?:\b)" 3; + "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)BacklinkCrawler(?:\b)" 3; + "~*(?:\b)BackStreet(?:\b)" 3; + "~*(?:\b)BackWeb(?:\b)" 3; + "~*(?:\b)Badass(?:\b)" 3; + "~*(?:\b)Bandit(?:\b)" 3; + "~*(?:\b)Barkrowler(?:\b)" 3; + "~*(?:\b)BatchFTP(?:\b)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; + "~*(?:\b)BBBike(?:\b)" 3; + "~*(?:\b)BDCbot(?:\b)" 3; + "~*(?:\b)BDFetch(?:\b)" 3; + "~*(?:\b)BetaBot(?:\b)" 3; + "~*(?:\b)Bigfoot(?:\b)" 3; + "~*(?:\b)Bitacle(?:\b)" 3; + "~*(?:\b)Blackboard(?:\b)" 3; + "~*(?:\b)Black\ Hole(?:\b)" 3; + "~*(?:\b)BlackWidow(?:\b)" 3; + "~*(?:\b)BLEXBot(?:\b)" 3; + "~*(?:\b)Blow(?:\b)" 3; + "~*(?:\b)BlowFish(?:\b)" 3; + "~*(?:\b)Boardreader(?:\b)" 3; + "~*(?:\b)Bolt(?:\b)" 3; + "~*(?:\b)BotALot(?:\b)" 3; + "~*(?:\b)Brandprotect(?:\b)" 3; + "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buddy(?:\b)" 3; + "~*(?:\b)BuiltBotTough(?:\b)" 3; + "~*(?:\b)BuiltWith(?:\b)" 3; + "~*(?:\b)Bullseye(?:\b)" 3; + "~*(?:\b)BunnySlippers(?:\b)" 3; + "~*(?:\b)BuzzSumo(?:\b)" 3; + "~*(?:\b)Calculon(?:\b)" 3; + "~*(?:\b)CATExplorador(?:\b)" 3; + "~*(?:\b)CazoodleBot(?:\b)" 3; + "~*(?:\b)CCBot(?:\b)" 3; + "~*(?:\b)Cegbfeieh(?:\b)" 3; + "~*(?:\b)CheeseBot(?:\b)" 3; + "~*(?:\b)CherryPicker(?:\b)" 3; + "~*(?:\b)CheTeam(?:\b)" 3; + "~*(?:\b)ChinaClaw(?:\b)" 3; + "~*(?:\b)Chlooe(?:\b)" 3; + "~*(?:\b)Claritybot(?:\b)" 3; + "~*(?:\b)Cliqzbot(?:\b)" 3; + "~*(?:\b)Cloud\ mapping(?:\b)" 3; + "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cogentbot(?:\b)" 3; + "~*(?:\b)cognitiveseo(?:\b)" 3; + "~*(?:\b)Collector(?:\b)" 3; + "~*(?:\b)com.plumanalytics(?:\b)" 3; + "~*(?:\b)Copier(?:\b)" 3; + "~*(?:\b)CopyRightCheck(?:\b)" 3; + "~*(?:\b)Copyscape(?:\b)" 3; + "~*(?:\b)Cosmos(?:\b)" 3; + "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)crawler4j(?:\b)" 3; + "~*(?:\b)crawler.feedback(?:\b)" 3; + "~*(?:\b)crawl.sogou.com(?:\b)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b)" 3; + "~*(?:\b)Crescent(?:\b)" 3; + "~*(?:\b)CrunchBot(?:\b)" 3; + "~*(?:\b)CSHttp(?:\b)" 3; + "~*(?:\b)Curious(?:\b)" 3; + "~*(?:\b)Custo(?:\b)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; + "~*(?:\b)DataCha0s(?:\b)" 3; + "~*(?:\b)DBLBot(?:\b)" 3; + "~*(?:\b)demandbase-bot(?:\b)" 3; + "~*(?:\b)Demon(?:\b)" 3; + "~*(?:\b)Deusu(?:\b)" 3; + "~*(?:\b)Devil(?:\b)" 3; + "~*(?:\b)Digincore(?:\b)" 3; + "~*(?:\b)DigitalPebble(?:\b)" 3; + "~*(?:\b)DIIbot(?:\b)" 3; + "~*(?:\b)Dirbuster(?:\b)" 3; + "~*(?:\b)Disco(?:\b)" 3; + "~*(?:\b)Discobot(?:\b)" 3; + "~*(?:\b)Discoverybot(?:\b)" 3; + "~*(?:\b)Dispatch(?:\b)" 3; + "~*(?:\b)DittoSpyder(?:\b)" 3; + "~*(?:\b)DnyzBot(?:\b)" 3; + "~*(?:\b)DomainAppender(?:\b)" 3; + "~*(?:\b)DomainCrawler(?:\b)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)DomainStatsBot(?:\b)" 3; + "~*(?:\b)Dotbot(?:\b)" 3; + "~*(?:\b)Download\ Wonder(?:\b)" 3; + "~*(?:\b)Dragonfly(?:\b)" 3; + "~*(?:\b)Drip(?:\b)" 3; + "~*(?:\b)DSearch(?:\b)" 3; + "~*(?:\b)DTS\ Agent(?:\b)" 3; + "~*(?:\b)EasyDL(?:\b)" 3; + "~*(?:\b)Ebingbong(?:\b)" 3; + "~*(?:\b)eCatch(?:\b)" 3; + "~*(?:\b)ECCP/1.0(?:\b)" 3; + "~*(?:\b)Ecxi(?:\b)" 3; + "~*(?:\b)EirGrabber(?:\b)" 3; + "~*(?:\b)EMail\ Siphon(?:\b)" 3; + "~*(?:\b)EMail\ Wolf(?:\b)" 3; + "~*(?:\b)EroCrawler(?:\b)" 3; + "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)Evil(?:\b)" 3; + "~*(?:\b)Exabot(?:\b)" 3; + "~*(?:\b)Express\ WebPictures(?:\b)" 3; + "~*(?:\b)ExtLinksBot(?:\b)" 3; + "~*(?:\b)Extractor(?:\b)" 3; + "~*(?:\b)ExtractorPro(?:\b)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; + "~*(?:\b)EyeNetIE(?:\b)" 3; + "~*(?:\b)Ezooms(?:\b)" 3; + "~*(?:\b)facebookscraper(?:\b)" 3; + "~*(?:\b)FDM(?:\b)" 3; + "~*(?:\b)FemtosearchBot(?:\b)" 3; + "~*(?:\b)FHscan(?:\b)" 3; + "~*(?:\b)Fimap(?:\b)" 3; + "~*(?:\b)Firefox/7.0(?:\b)" 3; + "~*(?:\b)FlashGet(?:\b)" 3; + "~*(?:\b)Flunky(?:\b)" 3; + "~*(?:\b)Foobot(?:\b)" 3; + "~*(?:\b)Freeuploader(?:\b)" 3; + "~*(?:\b)FrontPage(?:\b)" 3; + "~*(?:\b)FyberSpider(?:\b)" 3; + "~*(?:\b)Fyrebot(?:\b)" 3; + "~*(?:\b)GalaxyBot(?:\b)" 3; + "~*(?:\b)Genieo(?:\b)" 3; + "~*(?:\b)GermCrawler(?:\b)" 3; + "~*(?:\b)Getintent(?:\b)" 3; + "~*(?:\b)GetRight(?:\b)" 3; + "~*(?:\b)GetWeb(?:\b)" 3; + "~*(?:\b)Gigablast(?:\b)" 3; + "~*(?:\b)Gigabot(?:\b)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; + "~*(?:\b)Gotit(?:\b)" 3; + "~*(?:\b)GoZilla(?:\b)" 3; + "~*(?:\b)Go!Zilla(?:\b)" 3; + "~*(?:\b)Grabber(?:\b)" 3; + "~*(?:\b)GrabNet(?:\b)" 3; + "~*(?:\b)Grafula(?:\b)" 3; + "~*(?:\b)GrapeFX(?:\b)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b)" 3; + "~*(?:\b)GridBot(?:\b)" 3; + "~*(?:\b)GT::WWW(?:\b)" 3; + "~*(?:\b)Haansoft(?:\b)" 3; + "~*(?:\b)HaosouSpider(?:\b)" 3; + "~*(?:\b)Harvest(?:\b)" 3; + "~*(?:\b)Havij(?:\b)" 3; + "~*(?:\b)HEADMasterSEO(?:\b)" 3; + "~*(?:\b)Heritrix(?:\b)" 3; + "~*(?:\b)Hloader(?:\b)" 3; + "~*(?:\b)HMView(?:\b)" 3; + "~*(?:\b)HTMLparser(?:\b)" 3; + "~*(?:\b)HTTP::Lite(?:\b)" 3; + "~*(?:\b)HTTrack(?:\b)" 3; + "~*(?:\b)Humanlinks(?:\b)" 3; + "~*(?:\b)HybridBot(?:\b)" 3; + "~*(?:\b)Iblog(?:\b)" 3; + "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)Id-search(?:\b)" 3; + "~*(?:\b)IlseBot(?:\b)" 3; + "~*(?:\b)Image\ Fetch(?:\b)" 3; + "~*(?:\b)Image\ Sucker(?:\b)" 3; + "~*(?:\b)IndeedBot(?:\b)" 3; + "~*(?:\b)Indy\ Library(?:\b)" 3; + "~*(?:\b)InfoNaviRobot(?:\b)" 3; + "~*(?:\b)InfoTekies(?:\b)" 3; + "~*(?:\b)instabid(?:\b)" 3; + "~*(?:\b)Intelliseek(?:\b)" 3; + "~*(?:\b)InterGET(?:\b)" 3; + "~*(?:\b)Internet\ Ninja(?:\b)" 3; + "~*(?:\b)InternetSeer(?:\b)" 3; + "~*(?:\b)internetVista\ monitor(?:\b)" 3; + "~*(?:\b)ips-agent(?:\b)" 3; + "~*(?:\b)Iria(?:\b)" 3; + "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)Iskanie(?:\b)" 3; + "~*(?:\b)IstellaBot(?:\b)" 3; + "~*(?:\b)JamesBOT(?:\b)" 3; + "~*(?:\b)Jbrofuzz(?:\b)" 3; + "~*(?:\b)JennyBot(?:\b)" 3; + "~*(?:\b)JetCar(?:\b)" 3; + "~*(?:\b)Jetty(?:\b)" 3; + "~*(?:\b)JikeSpider(?:\b)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; + "~*(?:\b)Joomla(?:\b)" 3; + "~*(?:\b)Jorgee(?:\b)" 3; + "~*(?:\b)JustView(?:\b)" 3; + "~*(?:\b)Jyxobot(?:\b)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b)" 3; + "~*(?:\b)Keyword\ Density(?:\b)" 3; + "~*(?:\b)Kozmosbot(?:\b)" 3; + "~*(?:\b)Lanshanbot(?:\b)" 3; + "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)LeechFTP(?:\b)" 3; + "~*(?:\b)LeechGet(?:\b)" 3; + "~*(?:\b)LexiBot(?:\b)" 3; + "~*(?:\b)Lftp(?:\b)" 3; + "~*(?:\b)LibWeb(?:\b)" 3; + "~*(?:\b)Libwhisker(?:\b)" 3; + "~*(?:\b)Lightspeedsystems(?:\b)" 3; + "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkdexbot(?:\b)" 3; + "~*(?:\b)LinkextractorPro(?:\b)" 3; + "~*(?:\b)LinkpadBot(?:\b)" 3; + "~*(?:\b)LinkScan(?:\b)" 3; + "~*(?:\b)LinksManager(?:\b)" 3; + "~*(?:\b)LinkWalker(?:\b)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; + "~*(?:\b)Lipperhey(?:\b)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; + "~*(?:\b)Litemage_walker(?:\b)" 3; + "~*(?:\b)Lmspider(?:\b)" 3; + "~*(?:\b)LNSpiderguy(?:\b)" 3; + "~*(?:\b)Ltx71(?:\b)" 3; + "~*(?:\b)lwp-request(?:\b)" 3; + "~*(?:\b)LWP::Simple(?:\b)" 3; + "~*(?:\b)lwp-trivial(?:\b)" 3; + "~*(?:\b)Magnet(?:\b)" 3; + "~*(?:\b)Mag-Net(?:\b)" 3; + "~*(?:\b)magpie-crawler(?:\b)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b)" 3; + "~*(?:\b)Majestic12(?:\b)" 3; + "~*(?:\b)Majestic-SEO(?:\b)" 3; + "~*(?:\b)Majestic\ SEO(?:\b)" 3; + "~*(?:\b)MarkMonitor(?:\b)" 3; + "~*(?:\b)MarkWatch(?:\b)" 3; + "~*(?:\b)Masscan(?:\b)" 3; + "~*(?:\b)Mass\ Downloader(?:\b)" 3; + "~*(?:\b)Mata\ Hari(?:\b)" 3; + "~*(?:\b)MauiBot(?:\b)" 3; + "~*(?:\b)meanpathbot(?:\b)" 3; + "~*(?:\b)Meanpathbot(?:\b)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b)" 3; + "~*(?:\b)mediawords(?:\b)" 3; + "~*(?:\b)MegaIndex.ru(?:\b)" 3; + "~*(?:\b)Metauri(?:\b)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; + "~*(?:\b)MIDown\ tool(?:\b)" 3; + "~*(?:\b)MIIxpc(?:\b)" 3; + "~*(?:\b)Mister\ PiX(?:\b)" 3; + "~*(?:\b)MJ12bot(?:\b)" 3; + "~*(?:\b)Mojeek(?:\b)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; + "~*(?:\b)Mr.4x3(?:\b)" 3; + "~*(?:\b)MSFrontPage(?:\b)" 3; + "~*(?:\b)MSIECrawler(?:\b)" 3; + "~*(?:\b)Msrabot(?:\b)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; + "~*(?:\b)muhstik-scan(?:\b)" 3; + "~*(?:\b)Musobot(?:\b)" 3; + "~*(?:\b)Name\ Intelligence(?:\b)" 3; + "~*(?:\b)Nameprotect(?:\b)" 3; + "~*(?:\b)Navroad(?:\b)" 3; + "~*(?:\b)NearSite(?:\b)" 3; + "~*(?:\b)Needle(?:\b)" 3; + "~*(?:\b)Nessus(?:\b)" 3; + "~*(?:\b)NetAnts(?:\b)" 3; + "~*(?:\b)Netcraft(?:\b)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; + "~*(?:\b)NetLyzer(?:\b)" 3; + "~*(?:\b)NetMechanic(?:\b)" 3; + "~*(?:\b)NetSpider(?:\b)" 3; + "~*(?:\b)Nettrack(?:\b)" 3; + "~*(?:\b)Net\ Vampire(?:\b)" 3; + "~*(?:\b)Netvibes(?:\b)" 3; + "~*(?:\b)NetZIP(?:\b)" 3; + "~*(?:\b)NextGenSearchBot(?:\b)" 3; + "~*(?:\b)Nibbler(?:\b)" 3; + "~*(?:\b)NICErsPRO(?:\b)" 3; + "~*(?:\b)Niki-bot(?:\b)" 3; + "~*(?:\b)Nikto(?:\b)" 3; + "~*(?:\b)NimbleCrawler(?:\b)" 3; + "~*(?:\b)Nimbostratus(?:\b)" 3; + "~*(?:\b)Ninja(?:\b)" 3; + "~*(?:\b)Nmap(?:\b)" 3; + "~*(?:\b)NPbot(?:\b)" 3; + "~*(?:\b)Nutch(?:\b)" 3; + "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)Octopus(?:\b)" 3; + "~*(?:\b)Offline\ Explorer(?:\b)" 3; + "~*(?:\b)Offline\ Navigator(?:\b)" 3; + "~*(?:\b)OnCrawl(?:\b)" 3; + "~*(?:\b)Openfind(?:\b)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b)" 3; + "~*(?:\b)Openvas(?:\b)" 3; + "~*(?:\b)OpenVAS(?:\b)" 3; + "~*(?:\b)OrangeBot(?:\b)" 3; + "~*(?:\b)OrangeSpider(?:\b)" 3; + "~*(?:\b)OutclicksBot(?:\b)" 3; + "~*(?:\b)OutfoxBot(?:\b)" 3; + "~*(?:\b)PageAnalyzer(?:\b)" 3; + "~*(?:\b)Page\ Analyzer(?:\b)" 3; + "~*(?:\b)PageGrabber(?:\b)" 3; + "~*(?:\b)page\ scorer(?:\b)" 3; + "~*(?:\b)PageScorer(?:\b)" 3; + "~*(?:\b)Pandalytics(?:\b)" 3; + "~*(?:\b)Panscient(?:\b)" 3; + "~*(?:\b)Papa\ Foto(?:\b)" 3; + "~*(?:\b)Pavuk(?:\b)" 3; + "~*(?:\b)pcBrowser(?:\b)" 3; + "~*(?:\b)PECL::HTTP(?:\b)" 3; + "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)PHPCrawl(?:\b)" 3; + "~*(?:\b)Picscout(?:\b)" 3; + "~*(?:\b)Picsearch(?:\b)" 3; + "~*(?:\b)PictureFinder(?:\b)" 3; + "~*(?:\b)Pimonster(?:\b)" 3; + "~*(?:\b)Pi-Monster(?:\b)" 3; + "~*(?:\b)Pixray(?:\b)" 3; + "~*(?:\b)PleaseCrawl(?:\b)" 3; + "~*(?:\b)plumanalytics(?:\b)" 3; + "~*(?:\b)Pockey(?:\b)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; + "~*(?:\b)Probethenet(?:\b)" 3; + "~*(?:\b)ProPowerBot(?:\b)" 3; + "~*(?:\b)ProWebWalker(?:\b)" 3; + "~*(?:\b)Psbot(?:\b)" 3; + "~*(?:\b)Pump(?:\b)" 3; + "~*(?:\b)PxBroker(?:\b)" 3; + "~*(?:\b)PyCurl(?:\b)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; + "~*(?:\b)Quick-Crawler(?:\b)" 3; + "~*(?:\b)RankActive(?:\b)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b)" 3; + "~*(?:\b)RankFlex(?:\b)" 3; + "~*(?:\b)RankingBot(?:\b)" 3; + "~*(?:\b)RankingBot2(?:\b)" 3; + "~*(?:\b)Rankivabot(?:\b)" 3; + "~*(?:\b)RankurBot(?:\b)" 3; + "~*(?:\b)RealDownload(?:\b)" 3; + "~*(?:\b)Reaper(?:\b)" 3; + "~*(?:\b)RebelMouse(?:\b)" 3; + "~*(?:\b)Recorder(?:\b)" 3; + "~*(?:\b)RedesScrapy(?:\b)" 3; + "~*(?:\b)ReGet(?:\b)" 3; + "~*(?:\b)RepoMonkey(?:\b)" 3; + "~*(?:\b)Ripper(?:\b)" 3; + "~*(?:\b)RocketCrawler(?:\b)" 3; + "~*(?:\b)Rogerbot(?:\b)" 3; + "~*(?:\b)s1z.ru(?:\b)" 3; + "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)SBIder(?:\b)" 3; + "~*(?:\b)ScanAlert(?:\b)" 3; + "~*(?:\b)Scanbot(?:\b)" 3; + "~*(?:\b)scan.lol(?:\b)" 3; + "~*(?:\b)ScoutJet(?:\b)" 3; + "~*(?:\b)Scrapy(?:\b)" 3; + "~*(?:\b)Screaming(?:\b)" 3; + "~*(?:\b)ScreenerBot(?:\b)" 3; + "~*(?:\b)Searchestate(?:\b)" 3; + "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Semrush(?:\b)" 3; + "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SEOkicks(?:\b)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; + "~*(?:\b)Seomoz(?:\b)" 3; + "~*(?:\b)SEOprofiler(?:\b)" 3; + "~*(?:\b)seoscanners(?:\b)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b)" 3; + "~*(?:\b)SEOstats(?:\b)" 3; + "~*(?:\b)serpstatbot(?:\b)" 3; + "~*(?:\b)sexsearcher(?:\b)" 3; + "~*(?:\b)Shodan(?:\b)" 3; + "~*(?:\b)Siphon(?:\b)" 3; + "~*(?:\b)SISTRIX(?:\b)" 3; + "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteExplorer(?:\b)" 3; + "~*(?:\b)Siteimprove(?:\b)" 3; + "~*(?:\b)SiteLockSpider(?:\b)" 3; + "~*(?:\b)SiteSnagger(?:\b)" 3; + "~*(?:\b)SiteSucker(?:\b)" 3; + "~*(?:\b)Site\ Sucker(?:\b)" 3; + "~*(?:\b)Sitevigil(?:\b)" 3; + "~*(?:\b)SlySearch(?:\b)" 3; + "~*(?:\b)SmartDownload(?:\b)" 3; + "~*(?:\b)SMTBot(?:\b)" 3; + "~*(?:\b)Snake(?:\b)" 3; + "~*(?:\b)Snapbot(?:\b)" 3; + "~*(?:\b)Snoopy(?:\b)" 3; + "~*(?:\b)SocialRankIOBot(?:\b)" 3; + "~*(?:\b)Sociscraper(?:\b)" 3; + "~*(?:\b)sogouspider(?:\b)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; + "~*(?:\b)Sosospider(?:\b)" 3; + "~*(?:\b)Sottopop(?:\b)" 3; + "~*(?:\b)SpaceBison(?:\b)" 3; + "~*(?:\b)Spammen(?:\b)" 3; + "~*(?:\b)SpankBot(?:\b)" 3; + "~*(?:\b)Spanner(?:\b)" 3; + "~*(?:\b)sp_auditbot(?:\b)" 3; + "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spinn3r(?:\b)" 3; + "~*(?:\b)SputnikBot(?:\b)" 3; + "~*(?:\b)spyfu(?:\b)" 3; + "~*(?:\b)Sqlmap(?:\b)" 3; + "~*(?:\b)Sqlworm(?:\b)" 3; + "~*(?:\b)Sqworm(?:\b)" 3; + "~*(?:\b)Steeler(?:\b)" 3; + "~*(?:\b)Stripper(?:\b)" 3; + "~*(?:\b)Sucker(?:\b)" 3; + "~*(?:\b)Sucuri(?:\b)" 3; + "~*(?:\b)SuperBot(?:\b)" 3; + "~*(?:\b)SuperHTTP(?:\b)" 3; + "~*(?:\b)Surfbot(?:\b)" 3; + "~*(?:\b)SurveyBot(?:\b)" 3; + "~*(?:\b)Suzuran(?:\b)" 3; + "~*(?:\b)Swiftbot(?:\b)" 3; + "~*(?:\b)sysscan(?:\b)" 3; + "~*(?:\b)Szukacz(?:\b)" 3; + "~*(?:\b)T0PHackTeam(?:\b)" 3; + "~*(?:\b)T8Abot(?:\b)" 3; + "~*(?:\b)tAkeOut(?:\b)" 3; + "~*(?:\b)Teleport(?:\b)" 3; + "~*(?:\b)TeleportPro(?:\b)" 3; + "~*(?:\b)Telesoft(?:\b)" 3; + "~*(?:\b)Telesphoreo(?:\b)" 3; + "~*(?:\b)Telesphorep(?:\b)" 3; + "~*(?:\b)The\ Intraformant(?:\b)" 3; + "~*(?:\b)TheNomad(?:\b)" 3; + "~*(?:\b)Thumbor(?:\b)" 3; + "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)Titan(?:\b)" 3; + "~*(?:\b)Toata(?:\b)" 3; + "~*(?:\b)Toweyabot(?:\b)" 3; + "~*(?:\b)Tracemyfile(?:\b)" 3; + "~*(?:\b)Trendiction(?:\b)" 3; + "~*(?:\b)Trendictionbot(?:\b)" 3; + "~*(?:\b)trendiction.com(?:\b)" 3; + "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)True_Robot(?:\b)" 3; + "~*(?:\b)Turingos(?:\b)" 3; + "~*(?:\b)Turnitin(?:\b)" 3; + "~*(?:\b)TurnitinBot(?:\b)" 3; + "~*(?:\b)TwengaBot(?:\b)" 3; + "~*(?:\b)Twice(?:\b)" 3; + "~*(?:\b)Typhoeus(?:\b)" 3; + "~*(?:\b)UnisterBot(?:\b)" 3; + "~*(?:\b)Upflow(?:\b)" 3; + "~*(?:\b)URLy.Warning(?:\b)" 3; + "~*(?:\b)URLy\ Warning(?:\b)" 3; + "~*(?:\b)Vacuum(?:\b)" 3; + "~*(?:\b)Vagabondo(?:\b)" 3; + "~*(?:\b)VB\ Project(?:\b)" 3; + "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b)" 3; + "~*(?:\b)VidibleScraper(?:\b)" 3; + "~*(?:\b)Virusdie(?:\b)" 3; + "~*(?:\b)VoidEYE(?:\b)" 3; + "~*(?:\b)Voil(?:\b)" 3; + "~*(?:\b)Voltron(?:\b)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b)" 3; + "~*(?:\b)WallpapersHD(?:\b)" 3; + "~*(?:\b)WASALive-Bot(?:\b)" 3; + "~*(?:\b)WBSearchBot(?:\b)" 3; + "~*(?:\b)Webalta(?:\b)" 3; + "~*(?:\b)WebAuto(?:\b)" 3; + "~*(?:\b)Web\ Auto(?:\b)" 3; + "~*(?:\b)WebBandit(?:\b)" 3; + "~*(?:\b)WebCollage(?:\b)" 3; + "~*(?:\b)Web\ Collage(?:\b)" 3; + "~*(?:\b)WebCopier(?:\b)" 3; + "~*(?:\b)WEBDAV(?:\b)" 3; + "~*(?:\b)WebEnhancer(?:\b)" 3; + "~*(?:\b)Web\ Enhancer(?:\b)" 3; + "~*(?:\b)WebFetch(?:\b)" 3; + "~*(?:\b)Web\ Fetch(?:\b)" 3; + "~*(?:\b)WebFuck(?:\b)" 3; + "~*(?:\b)Web\ Fuck(?:\b)" 3; + "~*(?:\b)WebGo\ IS(?:\b)" 3; + "~*(?:\b)WebImageCollector(?:\b)" 3; + "~*(?:\b)WebLeacher(?:\b)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; + "~*(?:\b)webmeup-crawler(?:\b)" 3; + "~*(?:\b)WebPix(?:\b)" 3; + "~*(?:\b)Web\ Pix(?:\b)" 3; + "~*(?:\b)WebReaper(?:\b)" 3; + "~*(?:\b)WebSauger(?:\b)" 3; + "~*(?:\b)Web\ Sauger(?:\b)" 3; + "~*(?:\b)Webshag(?:\b)" 3; + "~*(?:\b)WebsiteExtractor(?:\b)" 3; + "~*(?:\b)WebsiteQuester(?:\b)" 3; + "~*(?:\b)Website\ Quester(?:\b)" 3; + "~*(?:\b)Webster(?:\b)" 3; + "~*(?:\b)WebStripper(?:\b)" 3; + "~*(?:\b)WebSucker(?:\b)" 3; + "~*(?:\b)Web\ Sucker(?:\b)" 3; + "~*(?:\b)WebWhacker(?:\b)" 3; + "~*(?:\b)WebZIP(?:\b)" 3; + "~*(?:\b)WeSEE(?:\b)" 3; + "~*(?:\b)Whack(?:\b)" 3; + "~*(?:\b)Whacker(?:\b)" 3; + "~*(?:\b)Whatweb(?:\b)" 3; + "~*(?:\b)Who.is\ Bot(?:\b)" 3; + "~*(?:\b)Widow(?:\b)" 3; + "~*(?:\b)WinHTTrack(?:\b)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; + "~*(?:\b)WISENutbot(?:\b)" 3; + "~*(?:\b)Wonderbot(?:\b)" 3; + "~*(?:\b)Woobot(?:\b)" 3; + "~*(?:\b)Wotbox(?:\b)" 3; + "~*(?:\b)Wprecon(?:\b)" 3; + "~*(?:\b)WPScan(?:\b)" 3; + "~*(?:\b)WWW-Collector-E(?:\b)" 3; + "~*(?:\b)WWW-Mechanize(?:\b)" 3; + "~*(?:\b)WWW::Mechanize(?:\b)" 3; + "~*(?:\b)WWWOFFLE(?:\b)" 3; + "~*(?:\b)x09Mozilla(?:\b)" 3; + "~*(?:\b)x22Mozilla(?:\b)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; + "~*(?:\b)Xenu(?:\b)" 3; + "~*(?:\b)xpymep1.exe(?:\b)" 3; + "~*(?:\b)YoudaoBot(?:\b)" 3; + "~*(?:\b)Zade(?:\b)" 3; + "~*(?:\b)Zauba(?:\b)" 3; + "~*(?:\b)zauba.io(?:\b)" 3; + "~*(?:\b)Zermelo(?:\b)" 3; + "~*(?:\b)Zeus(?:\b)" 3; + "~*(?:\b)zgrab(?:\b)" 3; + "~*(?:\b)Zitebot(?:\b)" 3; + "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZumBot(?:\b)" 3; + "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------- @@ -717,38 +717,38 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)adidxbot(?:\b|)" 0; - "~*(?:\b)AdsBot-Google(?:\b|)" 0; - "~*(?:\b)aolbuild(?:\b|)" 0; - "~*(?:\b)bingbot(?:\b|)" 0; - "~*(?:\b)bingpreview(?:\b|)" 0; - "~*(?:\b)DoCoMo(?:\b|)" 0; - "~*(?:\b)duckduckgo(?:\b|)" 0; - "~*(?:\b)facebookexternalhit(?:\b|)" 0; - "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; - "~*(?:\b)Googlebot(?:\b|)" 0; - "~*(?:\b)Googlebot-Image(?:\b|)" 0; - "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; - "~*(?:\b)Googlebot-News(?:\b|)" 0; - "~*(?:\b)Googlebot/Test(?:\b|)" 0; - "~*(?:\b)Googlebot-Video(?:\b|)" 0; - "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; - "~*(?:\b)Gravityscan(?:\b|)" 0; - "~*(?:\b)gsa-crawler(?:\b|)" 0; - "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; - "~*(?:\b)Kraken/0.1(?:\b|)" 0; - "~*(?:\b)LinkedInBot(?:\b|)" 0; - "~*(?:\b)Mediapartners-Google(?:\b|)" 0; - "~*(?:\b)msnbot(?:\b|)" 0; - "~*(?:\b)msnbot-media(?:\b|)" 0; - "~*(?:\b)SAMSUNG(?:\b|)" 0; - "~*(?:\b)Slackbot(?:\b|)" 0; - "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; - "~*(?:\b)slurp(?:\b|)" 0; - "~*(?:\b)teoma(?:\b|)" 0; - "~*(?:\b)TwitterBot(?:\b|)" 0; - "~*(?:\b)Wordpress(?:\b|)" 0; - "~*(?:\b)yahoo(?:\b|)" 0; + "~*(?:\b)adidxbot(?:\b)" 0; + "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bingbot(?:\b)" 0; + "~*(?:\b)bingpreview(?:\b)" 0; + "~*(?:\b)DoCoMo(?:\b)" 0; + "~*(?:\b)duckduckgo(?:\b)" 0; + "~*(?:\b)facebookexternalhit(?:\b)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b)" 0; + "~*(?:\b)Googlebot(?:\b)" 0; + "~*(?:\b)Googlebot-Image(?:\b)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b)" 0; + "~*(?:\b)Googlebot-News(?:\b)" 0; + "~*(?:\b)Googlebot/Test(?:\b)" 0; + "~*(?:\b)Googlebot-Video(?:\b)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; + "~*(?:\b)Gravityscan(?:\b)" 0; + "~*(?:\b)gsa-crawler(?:\b)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b)" 0; + "~*(?:\b)Kraken/0.1(?:\b)" 0; + "~*(?:\b)LinkedInBot(?:\b)" 0; + "~*(?:\b)Mediapartners-Google(?:\b)" 0; + "~*(?:\b)msnbot(?:\b)" 0; + "~*(?:\b)msnbot-media(?:\b)" 0; + "~*(?:\b)SAMSUNG(?:\b)" 0; + "~*(?:\b)Slackbot(?:\b)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 0; + "~*(?:\b)slurp(?:\b)" 0; + "~*(?:\b)teoma(?:\b)" 0; + "~*(?:\b)TwitterBot(?:\b)" 0; + "~*(?:\b)Wordpress(?:\b)" 0; + "~*(?:\b)yahoo(?:\b)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------------------- @@ -760,12 +760,12 @@ map $http_user_agent $bad_bot { #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)jetmon(?:\b|)" 1; - "~*(?:\b)libwww-perl(?:\b|)" 1; - "~*(?:\b)Lynx(?:\b|)" 1; - "~*(?:\b)munin(?:\b|)" 1; - "~*(?:\b)Presto(?:\b|)" 1; - "~*(?:\b)Wget/1.15(?:\b|)" 1; + "~*(?:\b)jetmon(?:\b)" 1; + "~*(?:\b)libwww-perl(?:\b)" 1; + "~*(?:\b)Lynx(?:\b)" 1; + "~*(?:\b)munin(?:\b)" 1; + "~*(?:\b)Presto(?:\b)" 1; + "~*(?:\b)Wget/1.15(?:\b)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ------------------------------------------------------- @@ -778,23 +778,23 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)Alexa(?:\b|)" 2; - "~*(?:\b)archive.org(?:\b|)" 2; - "~*(?:\b)Baidu(?:\b|)" 2; - "~*(?:\b)BUbiNG(?:\b|)" 2; - "~*(?:\b)FlipboardProxy(?:\b|)" 2; - "~*(?:\b)ia_archiver(?:\b|)" 2; - "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; - "~*(?:\b)Proximic(?:\b|)" 2; - "~*(?:\b)R6_CommentReader(?:\b|)" 2; - "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; - "~*(?:\b)RED/1(?:\b|)" 2; - "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; - "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; - "~*(?:\b)Spaidu(?:\b|)" 2; - "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; - "~*(?:\b)YandexBot(?:\b|)" 2; - "~*(?:\b)YandexImages(?:\b|)" 2; + "~*(?:\b)Alexa(?:\b)" 2; + "~*(?:\b)archive.org(?:\b)" 2; + "~*(?:\b)Baidu(?:\b)" 2; + "~*(?:\b)BUbiNG(?:\b)" 2; + "~*(?:\b)FlipboardProxy(?:\b)" 2; + "~*(?:\b)ia_archiver(?:\b)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b)" 2; + "~*(?:\b)Proximic(?:\b)" 2; + "~*(?:\b)R6_CommentReader(?:\b)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b)" 2; + "~*(?:\b)RED/1(?:\b)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b)" 2; + "~*(?:\b)Spaidu(?:\b)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b)" 2; + "~*(?:\b)YandexBot(?:\b)" 2; + "~*(?:\b)YandexImages(?:\b)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf index cf6daef34..d21a1eaca 100644 --- a/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf +++ b/.dev-tools/test5_conf_files/bots.d/whitelist-ips.conf @@ -45,6 +45,6 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.198.131.58 0; -35.192.136.167 0; +35.193.14.140 0; 34.66.178.120 0; +104.198.131.58 0; diff --git a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf index f26e52add..a1999739a 100644 --- a/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf +++ b/.dev-tools/test5_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2019.07.1709 -### Updated: Mon Jul 1 10:37:04 SAST 2019 +### Version: V4.2019.07.1712 +### Updated: Mon Jul 1 11:24:52 SAST 2019 ### Bad Referrer Count: 6717 ### Bad Bot Count: 556 ################################################### @@ -147,562 +147,562 @@ map $http_user_agent $bad_bot { # -------------------------------------------------- # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)360Spider(?:\b|)" 3; - "~*(?:\b)404checker(?:\b|)" 3; - "~*(?:\b)404enemy(?:\b|)" 3; - "~*(?:\b)80legs(?:\b|)" 3; - "~*(?:\b)Abonti(?:\b|)" 3; - "~*(?:\b)Aboundex(?:\b|)" 3; - "~*(?:\b)Aboundexbot(?:\b|)" 3; - "~*(?:\b)Acunetix(?:\b|)" 3; - "~*(?:\b)ADmantX(?:\b|)" 3; - "~*(?:\b)AfD-Verbotsverfahren(?:\b|)" 3; - "~*(?:\b)AhrefsBot(?:\b|)" 3; - "~*(?:\b)AIBOT(?:\b|)" 3; - "~*(?:\b)AiHitBot(?:\b|)" 3; - "~*(?:\b)Aipbot(?:\b|)" 3; - "~*(?:\b)Alexibot(?:\b|)" 3; - "~*(?:\b)Alligator(?:\b|)" 3; - "~*(?:\b)AllSubmitter(?:\b|)" 3; - "~*(?:\b)AlphaBot(?:\b|)" 3; - "~*(?:\b)Anarchie(?:\b|)" 3; - "~*(?:\b)Apexoo(?:\b|)" 3; - "~*(?:\b)archive.org_bot(?:\b|)" 3; - "~*(?:\b)ASPSeek(?:\b|)" 3; - "~*(?:\b)Asterias(?:\b|)" 3; - "~*(?:\b)Attach(?:\b|)" 3; - "~*(?:\b)autoemailspider(?:\b|)" 3; - "~*(?:\b)BackDoorBot(?:\b|)" 3; - "~*(?:\b)Backlink-Ceck(?:\b|)" 3; - "~*(?:\b)backlink-check(?:\b|)" 3; - "~*(?:\b)BacklinkCrawler(?:\b|)" 3; - "~*(?:\b)BackStreet(?:\b|)" 3; - "~*(?:\b)BackWeb(?:\b|)" 3; - "~*(?:\b)Badass(?:\b|)" 3; - "~*(?:\b)Bandit(?:\b|)" 3; - "~*(?:\b)Barkrowler(?:\b|)" 3; - "~*(?:\b)BatchFTP(?:\b|)" 3; - "~*(?:\b)Battleztar\ Bazinga(?:\b|)" 3; - "~*(?:\b)BBBike(?:\b|)" 3; - "~*(?:\b)BDCbot(?:\b|)" 3; - "~*(?:\b)BDFetch(?:\b|)" 3; - "~*(?:\b)BetaBot(?:\b|)" 3; - "~*(?:\b)Bigfoot(?:\b|)" 3; - "~*(?:\b)Bitacle(?:\b|)" 3; - "~*(?:\b)Blackboard(?:\b|)" 3; - "~*(?:\b)Black\ Hole(?:\b|)" 3; - "~*(?:\b)BlackWidow(?:\b|)" 3; - "~*(?:\b)BLEXBot(?:\b|)" 3; - "~*(?:\b)Blow(?:\b|)" 3; - "~*(?:\b)BlowFish(?:\b|)" 3; - "~*(?:\b)Boardreader(?:\b|)" 3; - "~*(?:\b)Bolt(?:\b|)" 3; - "~*(?:\b)BotALot(?:\b|)" 3; - "~*(?:\b)Brandprotect(?:\b|)" 3; - "~*(?:\b)Brandwatch(?:\b|)" 3; - "~*(?:\b)Buddy(?:\b|)" 3; - "~*(?:\b)BuiltBotTough(?:\b|)" 3; - "~*(?:\b)BuiltWith(?:\b|)" 3; - "~*(?:\b)Bullseye(?:\b|)" 3; - "~*(?:\b)BunnySlippers(?:\b|)" 3; - "~*(?:\b)BuzzSumo(?:\b|)" 3; - "~*(?:\b)Calculon(?:\b|)" 3; - "~*(?:\b)CATExplorador(?:\b|)" 3; - "~*(?:\b)CazoodleBot(?:\b|)" 3; - "~*(?:\b)CCBot(?:\b|)" 3; - "~*(?:\b)Cegbfeieh(?:\b|)" 3; - "~*(?:\b)CheeseBot(?:\b|)" 3; - "~*(?:\b)CherryPicker(?:\b|)" 3; - "~*(?:\b)CheTeam(?:\b|)" 3; - "~*(?:\b)ChinaClaw(?:\b|)" 3; - "~*(?:\b)Chlooe(?:\b|)" 3; - "~*(?:\b)Claritybot(?:\b|)" 3; - "~*(?:\b)Cliqzbot(?:\b|)" 3; - "~*(?:\b)Cloud\ mapping(?:\b|)" 3; - "~*(?:\b)coccocbot-web(?:\b|)" 3; - "~*(?:\b)Cogentbot(?:\b|)" 3; - "~*(?:\b)cognitiveseo(?:\b|)" 3; - "~*(?:\b)Collector(?:\b|)" 3; - "~*(?:\b)com.plumanalytics(?:\b|)" 3; - "~*(?:\b)Copier(?:\b|)" 3; - "~*(?:\b)CopyRightCheck(?:\b|)" 3; - "~*(?:\b)Copyscape(?:\b|)" 3; - "~*(?:\b)Cosmos(?:\b|)" 3; - "~*(?:\b)Craftbot(?:\b|)" 3; - "~*(?:\b)crawler4j(?:\b|)" 3; - "~*(?:\b)crawler.feedback(?:\b|)" 3; - "~*(?:\b)crawl.sogou.com(?:\b|)" 3; - "~*(?:\b)CrazyWebCrawler(?:\b|)" 3; - "~*(?:\b)Crescent(?:\b|)" 3; - "~*(?:\b)CrunchBot(?:\b|)" 3; - "~*(?:\b)CSHttp(?:\b|)" 3; - "~*(?:\b)Curious(?:\b|)" 3; - "~*(?:\b)Custo(?:\b|)" 3; - "~*(?:\b)DatabaseDriverMysqli(?:\b|)" 3; - "~*(?:\b)DataCha0s(?:\b|)" 3; - "~*(?:\b)DBLBot(?:\b|)" 3; - "~*(?:\b)demandbase-bot(?:\b|)" 3; - "~*(?:\b)Demon(?:\b|)" 3; - "~*(?:\b)Deusu(?:\b|)" 3; - "~*(?:\b)Devil(?:\b|)" 3; - "~*(?:\b)Digincore(?:\b|)" 3; - "~*(?:\b)DigitalPebble(?:\b|)" 3; - "~*(?:\b)DIIbot(?:\b|)" 3; - "~*(?:\b)Dirbuster(?:\b|)" 3; - "~*(?:\b)Disco(?:\b|)" 3; - "~*(?:\b)Discobot(?:\b|)" 3; - "~*(?:\b)Discoverybot(?:\b|)" 3; - "~*(?:\b)Dispatch(?:\b|)" 3; - "~*(?:\b)DittoSpyder(?:\b|)" 3; - "~*(?:\b)DnyzBot(?:\b|)" 3; - "~*(?:\b)DomainAppender(?:\b|)" 3; - "~*(?:\b)DomainCrawler(?:\b|)" 3; - "~*(?:\b)DomainSigmaCrawler(?:\b|)" 3; - "~*(?:\b)DomainStatsBot(?:\b|)" 3; - "~*(?:\b)Dotbot(?:\b|)" 3; - "~*(?:\b)Download\ Wonder(?:\b|)" 3; - "~*(?:\b)Dragonfly(?:\b|)" 3; - "~*(?:\b)Drip(?:\b|)" 3; - "~*(?:\b)DSearch(?:\b|)" 3; - "~*(?:\b)DTS\ Agent(?:\b|)" 3; - "~*(?:\b)EasyDL(?:\b|)" 3; - "~*(?:\b)Ebingbong(?:\b|)" 3; - "~*(?:\b)eCatch(?:\b|)" 3; - "~*(?:\b)ECCP/1.0(?:\b|)" 3; - "~*(?:\b)Ecxi(?:\b|)" 3; - "~*(?:\b)EirGrabber(?:\b|)" 3; - "~*(?:\b)EMail\ Siphon(?:\b|)" 3; - "~*(?:\b)EMail\ Wolf(?:\b|)" 3; - "~*(?:\b)EroCrawler(?:\b|)" 3; - "~*(?:\b)evc-batch(?:\b|)" 3; - "~*(?:\b)Evil(?:\b|)" 3; - "~*(?:\b)Exabot(?:\b|)" 3; - "~*(?:\b)Express\ WebPictures(?:\b|)" 3; - "~*(?:\b)ExtLinksBot(?:\b|)" 3; - "~*(?:\b)Extractor(?:\b|)" 3; - "~*(?:\b)ExtractorPro(?:\b|)" 3; - "~*(?:\b)Extreme\ Picture\ Finder(?:\b|)" 3; - "~*(?:\b)EyeNetIE(?:\b|)" 3; - "~*(?:\b)Ezooms(?:\b|)" 3; - "~*(?:\b)facebookscraper(?:\b|)" 3; - "~*(?:\b)FDM(?:\b|)" 3; - "~*(?:\b)FemtosearchBot(?:\b|)" 3; - "~*(?:\b)FHscan(?:\b|)" 3; - "~*(?:\b)Fimap(?:\b|)" 3; - "~*(?:\b)Firefox/7.0(?:\b|)" 3; - "~*(?:\b)FlashGet(?:\b|)" 3; - "~*(?:\b)Flunky(?:\b|)" 3; - "~*(?:\b)Foobot(?:\b|)" 3; - "~*(?:\b)Freeuploader(?:\b|)" 3; - "~*(?:\b)FrontPage(?:\b|)" 3; - "~*(?:\b)FyberSpider(?:\b|)" 3; - "~*(?:\b)Fyrebot(?:\b|)" 3; - "~*(?:\b)GalaxyBot(?:\b|)" 3; - "~*(?:\b)Genieo(?:\b|)" 3; - "~*(?:\b)GermCrawler(?:\b|)" 3; - "~*(?:\b)Getintent(?:\b|)" 3; - "~*(?:\b)GetRight(?:\b|)" 3; - "~*(?:\b)GetWeb(?:\b|)" 3; - "~*(?:\b)Gigablast(?:\b|)" 3; - "~*(?:\b)Gigabot(?:\b|)" 3; - "~*(?:\b)G-i-g-a-b-o-t(?:\b|)" 3; - "~*(?:\b)Go-Ahead-Got-It(?:\b|)" 3; - "~*(?:\b)Gotit(?:\b|)" 3; - "~*(?:\b)GoZilla(?:\b|)" 3; - "~*(?:\b)Go!Zilla(?:\b|)" 3; - "~*(?:\b)Grabber(?:\b|)" 3; - "~*(?:\b)GrabNet(?:\b|)" 3; - "~*(?:\b)Grafula(?:\b|)" 3; - "~*(?:\b)GrapeFX(?:\b|)" 3; - "~*(?:\b)GrapeshotCrawler(?:\b|)" 3; - "~*(?:\b)GridBot(?:\b|)" 3; - "~*(?:\b)GT::WWW(?:\b|)" 3; - "~*(?:\b)Haansoft(?:\b|)" 3; - "~*(?:\b)HaosouSpider(?:\b|)" 3; - "~*(?:\b)Harvest(?:\b|)" 3; - "~*(?:\b)Havij(?:\b|)" 3; - "~*(?:\b)HEADMasterSEO(?:\b|)" 3; - "~*(?:\b)Heritrix(?:\b|)" 3; - "~*(?:\b)Hloader(?:\b|)" 3; - "~*(?:\b)HMView(?:\b|)" 3; - "~*(?:\b)HTMLparser(?:\b|)" 3; - "~*(?:\b)HTTP::Lite(?:\b|)" 3; - "~*(?:\b)HTTrack(?:\b|)" 3; - "~*(?:\b)Humanlinks(?:\b|)" 3; - "~*(?:\b)HybridBot(?:\b|)" 3; - "~*(?:\b)Iblog(?:\b|)" 3; - "~*(?:\b)IDBot(?:\b|)" 3; - "~*(?:\b)Id-search(?:\b|)" 3; - "~*(?:\b)IlseBot(?:\b|)" 3; - "~*(?:\b)Image\ Fetch(?:\b|)" 3; - "~*(?:\b)Image\ Sucker(?:\b|)" 3; - "~*(?:\b)IndeedBot(?:\b|)" 3; - "~*(?:\b)Indy\ Library(?:\b|)" 3; - "~*(?:\b)InfoNaviRobot(?:\b|)" 3; - "~*(?:\b)InfoTekies(?:\b|)" 3; - "~*(?:\b)instabid(?:\b|)" 3; - "~*(?:\b)Intelliseek(?:\b|)" 3; - "~*(?:\b)InterGET(?:\b|)" 3; - "~*(?:\b)Internet\ Ninja(?:\b|)" 3; - "~*(?:\b)InternetSeer(?:\b|)" 3; - "~*(?:\b)internetVista\ monitor(?:\b|)" 3; - "~*(?:\b)ips-agent(?:\b|)" 3; - "~*(?:\b)Iria(?:\b|)" 3; - "~*(?:\b)IRLbot(?:\b|)" 3; - "~*(?:\b)Iskanie(?:\b|)" 3; - "~*(?:\b)IstellaBot(?:\b|)" 3; - "~*(?:\b)JamesBOT(?:\b|)" 3; - "~*(?:\b)Jbrofuzz(?:\b|)" 3; - "~*(?:\b)JennyBot(?:\b|)" 3; - "~*(?:\b)JetCar(?:\b|)" 3; - "~*(?:\b)Jetty(?:\b|)" 3; - "~*(?:\b)JikeSpider(?:\b|)" 3; - "~*(?:\b)JOC\ Web\ Spider(?:\b|)" 3; - "~*(?:\b)Joomla(?:\b|)" 3; - "~*(?:\b)Jorgee(?:\b|)" 3; - "~*(?:\b)JustView(?:\b|)" 3; - "~*(?:\b)Jyxobot(?:\b|)" 3; - "~*(?:\b)Kenjin\ Spider(?:\b|)" 3; - "~*(?:\b)Keyword\ Density(?:\b|)" 3; - "~*(?:\b)Kozmosbot(?:\b|)" 3; - "~*(?:\b)Lanshanbot(?:\b|)" 3; - "~*(?:\b)Larbin(?:\b|)" 3; - "~*(?:\b)LeechFTP(?:\b|)" 3; - "~*(?:\b)LeechGet(?:\b|)" 3; - "~*(?:\b)LexiBot(?:\b|)" 3; - "~*(?:\b)Lftp(?:\b|)" 3; - "~*(?:\b)LibWeb(?:\b|)" 3; - "~*(?:\b)Libwhisker(?:\b|)" 3; - "~*(?:\b)Lightspeedsystems(?:\b|)" 3; - "~*(?:\b)Likse(?:\b|)" 3; - "~*(?:\b)Linkdexbot(?:\b|)" 3; - "~*(?:\b)LinkextractorPro(?:\b|)" 3; - "~*(?:\b)LinkpadBot(?:\b|)" 3; - "~*(?:\b)LinkScan(?:\b|)" 3; - "~*(?:\b)LinksManager(?:\b|)" 3; - "~*(?:\b)LinkWalker(?:\b|)" 3; - "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b|)" 3; - "~*(?:\b)LinqiaRSSBot(?:\b|)" 3; - "~*(?:\b)LinqiaScrapeBot(?:\b|)" 3; - "~*(?:\b)Lipperhey(?:\b|)" 3; - "~*(?:\b)Lipperhey\ Spider(?:\b|)" 3; - "~*(?:\b)Litemage_walker(?:\b|)" 3; - "~*(?:\b)Lmspider(?:\b|)" 3; - "~*(?:\b)LNSpiderguy(?:\b|)" 3; - "~*(?:\b)Ltx71(?:\b|)" 3; - "~*(?:\b)lwp-request(?:\b|)" 3; - "~*(?:\b)LWP::Simple(?:\b|)" 3; - "~*(?:\b)lwp-trivial(?:\b|)" 3; - "~*(?:\b)Magnet(?:\b|)" 3; - "~*(?:\b)Mag-Net(?:\b|)" 3; - "~*(?:\b)magpie-crawler(?:\b|)" 3; - "~*(?:\b)Mail.RU_Bot(?:\b|)" 3; - "~*(?:\b)Majestic12(?:\b|)" 3; - "~*(?:\b)Majestic-SEO(?:\b|)" 3; - "~*(?:\b)Majestic\ SEO(?:\b|)" 3; - "~*(?:\b)MarkMonitor(?:\b|)" 3; - "~*(?:\b)MarkWatch(?:\b|)" 3; - "~*(?:\b)Masscan(?:\b|)" 3; - "~*(?:\b)Mass\ Downloader(?:\b|)" 3; - "~*(?:\b)Mata\ Hari(?:\b|)" 3; - "~*(?:\b)MauiBot(?:\b|)" 3; - "~*(?:\b)meanpathbot(?:\b|)" 3; - "~*(?:\b)Meanpathbot(?:\b|)" 3; - "~*(?:\b)MeanPath\ Bot(?:\b|)" 3; - "~*(?:\b)Mediatoolkitbot(?:\b|)" 3; - "~*(?:\b)mediawords(?:\b|)" 3; - "~*(?:\b)MegaIndex.ru(?:\b|)" 3; - "~*(?:\b)Metauri(?:\b|)" 3; - "~*(?:\b)MFC_Tear_Sample(?:\b|)" 3; - "~*(?:\b)Microsoft\ Data\ Access(?:\b|)" 3; - "~*(?:\b)Microsoft\ URL\ Control(?:\b|)" 3; - "~*(?:\b)MIDown\ tool(?:\b|)" 3; - "~*(?:\b)MIIxpc(?:\b|)" 3; - "~*(?:\b)Mister\ PiX(?:\b|)" 3; - "~*(?:\b)MJ12bot(?:\b|)" 3; - "~*(?:\b)Mojeek(?:\b|)" 3; - "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b|)" 3; - "~*(?:\b)Mr.4x3(?:\b|)" 3; - "~*(?:\b)MSFrontPage(?:\b|)" 3; - "~*(?:\b)MSIECrawler(?:\b|)" 3; - "~*(?:\b)Msrabot(?:\b|)" 3; - "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b|)" 3; - "~*(?:\b)muhstik-scan(?:\b|)" 3; - "~*(?:\b)Musobot(?:\b|)" 3; - "~*(?:\b)Name\ Intelligence(?:\b|)" 3; - "~*(?:\b)Nameprotect(?:\b|)" 3; - "~*(?:\b)Navroad(?:\b|)" 3; - "~*(?:\b)NearSite(?:\b|)" 3; - "~*(?:\b)Needle(?:\b|)" 3; - "~*(?:\b)Nessus(?:\b|)" 3; - "~*(?:\b)NetAnts(?:\b|)" 3; - "~*(?:\b)Netcraft(?:\b|)" 3; - "~*(?:\b)netEstate\ NE\ Crawler(?:\b|)" 3; - "~*(?:\b)NetLyzer(?:\b|)" 3; - "~*(?:\b)NetMechanic(?:\b|)" 3; - "~*(?:\b)NetSpider(?:\b|)" 3; - "~*(?:\b)Nettrack(?:\b|)" 3; - "~*(?:\b)Net\ Vampire(?:\b|)" 3; - "~*(?:\b)Netvibes(?:\b|)" 3; - "~*(?:\b)NetZIP(?:\b|)" 3; - "~*(?:\b)NextGenSearchBot(?:\b|)" 3; - "~*(?:\b)Nibbler(?:\b|)" 3; - "~*(?:\b)NICErsPRO(?:\b|)" 3; - "~*(?:\b)Niki-bot(?:\b|)" 3; - "~*(?:\b)Nikto(?:\b|)" 3; - "~*(?:\b)NimbleCrawler(?:\b|)" 3; - "~*(?:\b)Nimbostratus(?:\b|)" 3; - "~*(?:\b)Ninja(?:\b|)" 3; - "~*(?:\b)Nmap(?:\b|)" 3; - "~*(?:\b)NPbot(?:\b|)" 3; - "~*(?:\b)Nutch(?:\b|)" 3; - "~*(?:\b)oBot(?:\b|)" 3; - "~*(?:\b)Octopus(?:\b|)" 3; - "~*(?:\b)Offline\ Explorer(?:\b|)" 3; - "~*(?:\b)Offline\ Navigator(?:\b|)" 3; - "~*(?:\b)OnCrawl(?:\b|)" 3; - "~*(?:\b)Openfind(?:\b|)" 3; - "~*(?:\b)OpenLinkProfiler(?:\b|)" 3; - "~*(?:\b)Openvas(?:\b|)" 3; - "~*(?:\b)OpenVAS(?:\b|)" 3; - "~*(?:\b)OrangeBot(?:\b|)" 3; - "~*(?:\b)OrangeSpider(?:\b|)" 3; - "~*(?:\b)OutclicksBot(?:\b|)" 3; - "~*(?:\b)OutfoxBot(?:\b|)" 3; - "~*(?:\b)PageAnalyzer(?:\b|)" 3; - "~*(?:\b)Page\ Analyzer(?:\b|)" 3; - "~*(?:\b)PageGrabber(?:\b|)" 3; - "~*(?:\b)page\ scorer(?:\b|)" 3; - "~*(?:\b)PageScorer(?:\b|)" 3; - "~*(?:\b)Pandalytics(?:\b|)" 3; - "~*(?:\b)Panscient(?:\b|)" 3; - "~*(?:\b)Papa\ Foto(?:\b|)" 3; - "~*(?:\b)Pavuk(?:\b|)" 3; - "~*(?:\b)pcBrowser(?:\b|)" 3; - "~*(?:\b)PECL::HTTP(?:\b|)" 3; - "~*(?:\b)PeoplePal(?:\b|)" 3; - "~*(?:\b)PHPCrawl(?:\b|)" 3; - "~*(?:\b)Picscout(?:\b|)" 3; - "~*(?:\b)Picsearch(?:\b|)" 3; - "~*(?:\b)PictureFinder(?:\b|)" 3; - "~*(?:\b)Pimonster(?:\b|)" 3; - "~*(?:\b)Pi-Monster(?:\b|)" 3; - "~*(?:\b)Pixray(?:\b|)" 3; - "~*(?:\b)PleaseCrawl(?:\b|)" 3; - "~*(?:\b)plumanalytics(?:\b|)" 3; - "~*(?:\b)Pockey(?:\b|)" 3; - "~*(?:\b)POE-Component-Client-HTTP(?:\b|)" 3; - "~*(?:\b)Probethenet(?:\b|)" 3; - "~*(?:\b)ProPowerBot(?:\b|)" 3; - "~*(?:\b)ProWebWalker(?:\b|)" 3; - "~*(?:\b)Psbot(?:\b|)" 3; - "~*(?:\b)Pump(?:\b|)" 3; - "~*(?:\b)PxBroker(?:\b|)" 3; - "~*(?:\b)PyCurl(?:\b|)" 3; - "~*(?:\b)QueryN\ Metasearch(?:\b|)" 3; - "~*(?:\b)Quick-Crawler(?:\b|)" 3; - "~*(?:\b)RankActive(?:\b|)" 3; - "~*(?:\b)RankActiveLinkBot(?:\b|)" 3; - "~*(?:\b)RankFlex(?:\b|)" 3; - "~*(?:\b)RankingBot(?:\b|)" 3; - "~*(?:\b)RankingBot2(?:\b|)" 3; - "~*(?:\b)Rankivabot(?:\b|)" 3; - "~*(?:\b)RankurBot(?:\b|)" 3; - "~*(?:\b)RealDownload(?:\b|)" 3; - "~*(?:\b)Reaper(?:\b|)" 3; - "~*(?:\b)RebelMouse(?:\b|)" 3; - "~*(?:\b)Recorder(?:\b|)" 3; - "~*(?:\b)RedesScrapy(?:\b|)" 3; - "~*(?:\b)ReGet(?:\b|)" 3; - "~*(?:\b)RepoMonkey(?:\b|)" 3; - "~*(?:\b)Ripper(?:\b|)" 3; - "~*(?:\b)RocketCrawler(?:\b|)" 3; - "~*(?:\b)Rogerbot(?:\b|)" 3; - "~*(?:\b)s1z.ru(?:\b|)" 3; - "~*(?:\b)SalesIntelligent(?:\b|)" 3; - "~*(?:\b)SBIder(?:\b|)" 3; - "~*(?:\b)ScanAlert(?:\b|)" 3; - "~*(?:\b)Scanbot(?:\b|)" 3; - "~*(?:\b)scan.lol(?:\b|)" 3; - "~*(?:\b)ScoutJet(?:\b|)" 3; - "~*(?:\b)Scrapy(?:\b|)" 3; - "~*(?:\b)Screaming(?:\b|)" 3; - "~*(?:\b)ScreenerBot(?:\b|)" 3; - "~*(?:\b)Searchestate(?:\b|)" 3; - "~*(?:\b)SearchmetricsBot(?:\b|)" 3; - "~*(?:\b)Semrush(?:\b|)" 3; - "~*(?:\b)SemrushBot(?:\b|)" 3; - "~*(?:\b)SEOkicks(?:\b|)" 3; - "~*(?:\b)SEOkicks-Robot(?:\b|)" 3; - "~*(?:\b)SEOlyticsCrawler(?:\b|)" 3; - "~*(?:\b)Seomoz(?:\b|)" 3; - "~*(?:\b)SEOprofiler(?:\b|)" 3; - "~*(?:\b)seoscanners(?:\b|)" 3; - "~*(?:\b)SeoSiteCheckup(?:\b|)" 3; - "~*(?:\b)SEOstats(?:\b|)" 3; - "~*(?:\b)serpstatbot(?:\b|)" 3; - "~*(?:\b)sexsearcher(?:\b|)" 3; - "~*(?:\b)Shodan(?:\b|)" 3; - "~*(?:\b)Siphon(?:\b|)" 3; - "~*(?:\b)SISTRIX(?:\b|)" 3; - "~*(?:\b)Sitebeam(?:\b|)" 3; - "~*(?:\b)SiteExplorer(?:\b|)" 3; - "~*(?:\b)Siteimprove(?:\b|)" 3; - "~*(?:\b)SiteLockSpider(?:\b|)" 3; - "~*(?:\b)SiteSnagger(?:\b|)" 3; - "~*(?:\b)SiteSucker(?:\b|)" 3; - "~*(?:\b)Site\ Sucker(?:\b|)" 3; - "~*(?:\b)Sitevigil(?:\b|)" 3; - "~*(?:\b)SlySearch(?:\b|)" 3; - "~*(?:\b)SmartDownload(?:\b|)" 3; - "~*(?:\b)SMTBot(?:\b|)" 3; - "~*(?:\b)Snake(?:\b|)" 3; - "~*(?:\b)Snapbot(?:\b|)" 3; - "~*(?:\b)Snoopy(?:\b|)" 3; - "~*(?:\b)SocialRankIOBot(?:\b|)" 3; - "~*(?:\b)Sociscraper(?:\b|)" 3; - "~*(?:\b)sogouspider(?:\b|)" 3; - "~*(?:\b)Sogou\ web\ spider(?:\b|)" 3; - "~*(?:\b)Sosospider(?:\b|)" 3; - "~*(?:\b)Sottopop(?:\b|)" 3; - "~*(?:\b)SpaceBison(?:\b|)" 3; - "~*(?:\b)Spammen(?:\b|)" 3; - "~*(?:\b)SpankBot(?:\b|)" 3; - "~*(?:\b)Spanner(?:\b|)" 3; - "~*(?:\b)sp_auditbot(?:\b|)" 3; - "~*(?:\b)Spbot(?:\b|)" 3; - "~*(?:\b)Spinn3r(?:\b|)" 3; - "~*(?:\b)SputnikBot(?:\b|)" 3; - "~*(?:\b)spyfu(?:\b|)" 3; - "~*(?:\b)Sqlmap(?:\b|)" 3; - "~*(?:\b)Sqlworm(?:\b|)" 3; - "~*(?:\b)Sqworm(?:\b|)" 3; - "~*(?:\b)Steeler(?:\b|)" 3; - "~*(?:\b)Stripper(?:\b|)" 3; - "~*(?:\b)Sucker(?:\b|)" 3; - "~*(?:\b)Sucuri(?:\b|)" 3; - "~*(?:\b)SuperBot(?:\b|)" 3; - "~*(?:\b)SuperHTTP(?:\b|)" 3; - "~*(?:\b)Surfbot(?:\b|)" 3; - "~*(?:\b)SurveyBot(?:\b|)" 3; - "~*(?:\b)Suzuran(?:\b|)" 3; - "~*(?:\b)Swiftbot(?:\b|)" 3; - "~*(?:\b)sysscan(?:\b|)" 3; - "~*(?:\b)Szukacz(?:\b|)" 3; - "~*(?:\b)T0PHackTeam(?:\b|)" 3; - "~*(?:\b)T8Abot(?:\b|)" 3; - "~*(?:\b)tAkeOut(?:\b|)" 3; - "~*(?:\b)Teleport(?:\b|)" 3; - "~*(?:\b)TeleportPro(?:\b|)" 3; - "~*(?:\b)Telesoft(?:\b|)" 3; - "~*(?:\b)Telesphoreo(?:\b|)" 3; - "~*(?:\b)Telesphorep(?:\b|)" 3; - "~*(?:\b)The\ Intraformant(?:\b|)" 3; - "~*(?:\b)TheNomad(?:\b|)" 3; - "~*(?:\b)Thumbor(?:\b|)" 3; - "~*(?:\b)TightTwatBot(?:\b|)" 3; - "~*(?:\b)Titan(?:\b|)" 3; - "~*(?:\b)Toata(?:\b|)" 3; - "~*(?:\b)Toweyabot(?:\b|)" 3; - "~*(?:\b)Tracemyfile(?:\b|)" 3; - "~*(?:\b)Trendiction(?:\b|)" 3; - "~*(?:\b)Trendictionbot(?:\b|)" 3; - "~*(?:\b)trendiction.com(?:\b|)" 3; - "~*(?:\b)trendiction.de(?:\b|)" 3; - "~*(?:\b)True_Robot(?:\b|)" 3; - "~*(?:\b)Turingos(?:\b|)" 3; - "~*(?:\b)Turnitin(?:\b|)" 3; - "~*(?:\b)TurnitinBot(?:\b|)" 3; - "~*(?:\b)TwengaBot(?:\b|)" 3; - "~*(?:\b)Twice(?:\b|)" 3; - "~*(?:\b)Typhoeus(?:\b|)" 3; - "~*(?:\b)UnisterBot(?:\b|)" 3; - "~*(?:\b)Upflow(?:\b|)" 3; - "~*(?:\b)URLy.Warning(?:\b|)" 3; - "~*(?:\b)URLy\ Warning(?:\b|)" 3; - "~*(?:\b)Vacuum(?:\b|)" 3; - "~*(?:\b)Vagabondo(?:\b|)" 3; - "~*(?:\b)VB\ Project(?:\b|)" 3; - "~*(?:\b)VCI(?:\b|)" 3; - "~*(?:\b)VeriCiteCrawler(?:\b|)" 3; - "~*(?:\b)VidibleScraper(?:\b|)" 3; - "~*(?:\b)Virusdie(?:\b|)" 3; - "~*(?:\b)VoidEYE(?:\b|)" 3; - "~*(?:\b)Voil(?:\b|)" 3; - "~*(?:\b)Voltron(?:\b|)" 3; - "~*(?:\b)Wallpapers/3.0(?:\b|)" 3; - "~*(?:\b)WallpapersHD(?:\b|)" 3; - "~*(?:\b)WASALive-Bot(?:\b|)" 3; - "~*(?:\b)WBSearchBot(?:\b|)" 3; - "~*(?:\b)Webalta(?:\b|)" 3; - "~*(?:\b)WebAuto(?:\b|)" 3; - "~*(?:\b)Web\ Auto(?:\b|)" 3; - "~*(?:\b)WebBandit(?:\b|)" 3; - "~*(?:\b)WebCollage(?:\b|)" 3; - "~*(?:\b)Web\ Collage(?:\b|)" 3; - "~*(?:\b)WebCopier(?:\b|)" 3; - "~*(?:\b)WEBDAV(?:\b|)" 3; - "~*(?:\b)WebEnhancer(?:\b|)" 3; - "~*(?:\b)Web\ Enhancer(?:\b|)" 3; - "~*(?:\b)WebFetch(?:\b|)" 3; - "~*(?:\b)Web\ Fetch(?:\b|)" 3; - "~*(?:\b)WebFuck(?:\b|)" 3; - "~*(?:\b)Web\ Fuck(?:\b|)" 3; - "~*(?:\b)WebGo\ IS(?:\b|)" 3; - "~*(?:\b)WebImageCollector(?:\b|)" 3; - "~*(?:\b)WebLeacher(?:\b|)" 3; - "~*(?:\b)WebmasterWorldForumBot(?:\b|)" 3; - "~*(?:\b)webmeup-crawler(?:\b|)" 3; - "~*(?:\b)WebPix(?:\b|)" 3; - "~*(?:\b)Web\ Pix(?:\b|)" 3; - "~*(?:\b)WebReaper(?:\b|)" 3; - "~*(?:\b)WebSauger(?:\b|)" 3; - "~*(?:\b)Web\ Sauger(?:\b|)" 3; - "~*(?:\b)Webshag(?:\b|)" 3; - "~*(?:\b)WebsiteExtractor(?:\b|)" 3; - "~*(?:\b)WebsiteQuester(?:\b|)" 3; - "~*(?:\b)Website\ Quester(?:\b|)" 3; - "~*(?:\b)Webster(?:\b|)" 3; - "~*(?:\b)WebStripper(?:\b|)" 3; - "~*(?:\b)WebSucker(?:\b|)" 3; - "~*(?:\b)Web\ Sucker(?:\b|)" 3; - "~*(?:\b)WebWhacker(?:\b|)" 3; - "~*(?:\b)WebZIP(?:\b|)" 3; - "~*(?:\b)WeSEE(?:\b|)" 3; - "~*(?:\b)Whack(?:\b|)" 3; - "~*(?:\b)Whacker(?:\b|)" 3; - "~*(?:\b)Whatweb(?:\b|)" 3; - "~*(?:\b)Who.is\ Bot(?:\b|)" 3; - "~*(?:\b)Widow(?:\b|)" 3; - "~*(?:\b)WinHTTrack(?:\b|)" 3; - "~*(?:\b)WiseGuys\ Robot(?:\b|)" 3; - "~*(?:\b)WISENutbot(?:\b|)" 3; - "~*(?:\b)Wonderbot(?:\b|)" 3; - "~*(?:\b)Woobot(?:\b|)" 3; - "~*(?:\b)Wotbox(?:\b|)" 3; - "~*(?:\b)Wprecon(?:\b|)" 3; - "~*(?:\b)WPScan(?:\b|)" 3; - "~*(?:\b)WWW-Collector-E(?:\b|)" 3; - "~*(?:\b)WWW-Mechanize(?:\b|)" 3; - "~*(?:\b)WWW::Mechanize(?:\b|)" 3; - "~*(?:\b)WWWOFFLE(?:\b|)" 3; - "~*(?:\b)x09Mozilla(?:\b|)" 3; - "~*(?:\b)x22Mozilla(?:\b|)" 3; - "~*(?:\b)Xaldon_WebSpider(?:\b|)" 3; - "~*(?:\b)Xaldon\ WebSpider(?:\b|)" 3; - "~*(?:\b)Xenu(?:\b|)" 3; - "~*(?:\b)xpymep1.exe(?:\b|)" 3; - "~*(?:\b)YoudaoBot(?:\b|)" 3; - "~*(?:\b)Zade(?:\b|)" 3; - "~*(?:\b)Zauba(?:\b|)" 3; - "~*(?:\b)zauba.io(?:\b|)" 3; - "~*(?:\b)Zermelo(?:\b|)" 3; - "~*(?:\b)Zeus(?:\b|)" 3; - "~*(?:\b)zgrab(?:\b|)" 3; - "~*(?:\b)Zitebot(?:\b|)" 3; - "~*(?:\b)ZmEu(?:\b|)" 3; - "~*(?:\b)ZumBot(?:\b|)" 3; - "~*(?:\b)ZyBorg(?:\b|)" 3; + "~*(?:\b)360Spider(?:\b)" 3; + "~*(?:\b)404checker(?:\b)" 3; + "~*(?:\b)404enemy(?:\b)" 3; + "~*(?:\b)80legs(?:\b)" 3; + "~*(?:\b)Abonti(?:\b)" 3; + "~*(?:\b)Aboundex(?:\b)" 3; + "~*(?:\b)Aboundexbot(?:\b)" 3; + "~*(?:\b)Acunetix(?:\b)" 3; + "~*(?:\b)ADmantX(?:\b)" 3; + "~*(?:\b)AfD-Verbotsverfahren(?:\b)" 3; + "~*(?:\b)AhrefsBot(?:\b)" 3; + "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)AiHitBot(?:\b)" 3; + "~*(?:\b)Aipbot(?:\b)" 3; + "~*(?:\b)Alexibot(?:\b)" 3; + "~*(?:\b)Alligator(?:\b)" 3; + "~*(?:\b)AllSubmitter(?:\b)" 3; + "~*(?:\b)AlphaBot(?:\b)" 3; + "~*(?:\b)Anarchie(?:\b)" 3; + "~*(?:\b)Apexoo(?:\b)" 3; + "~*(?:\b)archive.org_bot(?:\b)" 3; + "~*(?:\b)ASPSeek(?:\b)" 3; + "~*(?:\b)Asterias(?:\b)" 3; + "~*(?:\b)Attach(?:\b)" 3; + "~*(?:\b)autoemailspider(?:\b)" 3; + "~*(?:\b)BackDoorBot(?:\b)" 3; + "~*(?:\b)Backlink-Ceck(?:\b)" 3; + "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)BacklinkCrawler(?:\b)" 3; + "~*(?:\b)BackStreet(?:\b)" 3; + "~*(?:\b)BackWeb(?:\b)" 3; + "~*(?:\b)Badass(?:\b)" 3; + "~*(?:\b)Bandit(?:\b)" 3; + "~*(?:\b)Barkrowler(?:\b)" 3; + "~*(?:\b)BatchFTP(?:\b)" 3; + "~*(?:\b)Battleztar\ Bazinga(?:\b)" 3; + "~*(?:\b)BBBike(?:\b)" 3; + "~*(?:\b)BDCbot(?:\b)" 3; + "~*(?:\b)BDFetch(?:\b)" 3; + "~*(?:\b)BetaBot(?:\b)" 3; + "~*(?:\b)Bigfoot(?:\b)" 3; + "~*(?:\b)Bitacle(?:\b)" 3; + "~*(?:\b)Blackboard(?:\b)" 3; + "~*(?:\b)Black\ Hole(?:\b)" 3; + "~*(?:\b)BlackWidow(?:\b)" 3; + "~*(?:\b)BLEXBot(?:\b)" 3; + "~*(?:\b)Blow(?:\b)" 3; + "~*(?:\b)BlowFish(?:\b)" 3; + "~*(?:\b)Boardreader(?:\b)" 3; + "~*(?:\b)Bolt(?:\b)" 3; + "~*(?:\b)BotALot(?:\b)" 3; + "~*(?:\b)Brandprotect(?:\b)" 3; + "~*(?:\b)Brandwatch(?:\b)" 3; + "~*(?:\b)Buddy(?:\b)" 3; + "~*(?:\b)BuiltBotTough(?:\b)" 3; + "~*(?:\b)BuiltWith(?:\b)" 3; + "~*(?:\b)Bullseye(?:\b)" 3; + "~*(?:\b)BunnySlippers(?:\b)" 3; + "~*(?:\b)BuzzSumo(?:\b)" 3; + "~*(?:\b)Calculon(?:\b)" 3; + "~*(?:\b)CATExplorador(?:\b)" 3; + "~*(?:\b)CazoodleBot(?:\b)" 3; + "~*(?:\b)CCBot(?:\b)" 3; + "~*(?:\b)Cegbfeieh(?:\b)" 3; + "~*(?:\b)CheeseBot(?:\b)" 3; + "~*(?:\b)CherryPicker(?:\b)" 3; + "~*(?:\b)CheTeam(?:\b)" 3; + "~*(?:\b)ChinaClaw(?:\b)" 3; + "~*(?:\b)Chlooe(?:\b)" 3; + "~*(?:\b)Claritybot(?:\b)" 3; + "~*(?:\b)Cliqzbot(?:\b)" 3; + "~*(?:\b)Cloud\ mapping(?:\b)" 3; + "~*(?:\b)coccocbot-web(?:\b)" 3; + "~*(?:\b)Cogentbot(?:\b)" 3; + "~*(?:\b)cognitiveseo(?:\b)" 3; + "~*(?:\b)Collector(?:\b)" 3; + "~*(?:\b)com.plumanalytics(?:\b)" 3; + "~*(?:\b)Copier(?:\b)" 3; + "~*(?:\b)CopyRightCheck(?:\b)" 3; + "~*(?:\b)Copyscape(?:\b)" 3; + "~*(?:\b)Cosmos(?:\b)" 3; + "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)crawler4j(?:\b)" 3; + "~*(?:\b)crawler.feedback(?:\b)" 3; + "~*(?:\b)crawl.sogou.com(?:\b)" 3; + "~*(?:\b)CrazyWebCrawler(?:\b)" 3; + "~*(?:\b)Crescent(?:\b)" 3; + "~*(?:\b)CrunchBot(?:\b)" 3; + "~*(?:\b)CSHttp(?:\b)" 3; + "~*(?:\b)Curious(?:\b)" 3; + "~*(?:\b)Custo(?:\b)" 3; + "~*(?:\b)DatabaseDriverMysqli(?:\b)" 3; + "~*(?:\b)DataCha0s(?:\b)" 3; + "~*(?:\b)DBLBot(?:\b)" 3; + "~*(?:\b)demandbase-bot(?:\b)" 3; + "~*(?:\b)Demon(?:\b)" 3; + "~*(?:\b)Deusu(?:\b)" 3; + "~*(?:\b)Devil(?:\b)" 3; + "~*(?:\b)Digincore(?:\b)" 3; + "~*(?:\b)DigitalPebble(?:\b)" 3; + "~*(?:\b)DIIbot(?:\b)" 3; + "~*(?:\b)Dirbuster(?:\b)" 3; + "~*(?:\b)Disco(?:\b)" 3; + "~*(?:\b)Discobot(?:\b)" 3; + "~*(?:\b)Discoverybot(?:\b)" 3; + "~*(?:\b)Dispatch(?:\b)" 3; + "~*(?:\b)DittoSpyder(?:\b)" 3; + "~*(?:\b)DnyzBot(?:\b)" 3; + "~*(?:\b)DomainAppender(?:\b)" 3; + "~*(?:\b)DomainCrawler(?:\b)" 3; + "~*(?:\b)DomainSigmaCrawler(?:\b)" 3; + "~*(?:\b)DomainStatsBot(?:\b)" 3; + "~*(?:\b)Dotbot(?:\b)" 3; + "~*(?:\b)Download\ Wonder(?:\b)" 3; + "~*(?:\b)Dragonfly(?:\b)" 3; + "~*(?:\b)Drip(?:\b)" 3; + "~*(?:\b)DSearch(?:\b)" 3; + "~*(?:\b)DTS\ Agent(?:\b)" 3; + "~*(?:\b)EasyDL(?:\b)" 3; + "~*(?:\b)Ebingbong(?:\b)" 3; + "~*(?:\b)eCatch(?:\b)" 3; + "~*(?:\b)ECCP/1.0(?:\b)" 3; + "~*(?:\b)Ecxi(?:\b)" 3; + "~*(?:\b)EirGrabber(?:\b)" 3; + "~*(?:\b)EMail\ Siphon(?:\b)" 3; + "~*(?:\b)EMail\ Wolf(?:\b)" 3; + "~*(?:\b)EroCrawler(?:\b)" 3; + "~*(?:\b)evc-batch(?:\b)" 3; + "~*(?:\b)Evil(?:\b)" 3; + "~*(?:\b)Exabot(?:\b)" 3; + "~*(?:\b)Express\ WebPictures(?:\b)" 3; + "~*(?:\b)ExtLinksBot(?:\b)" 3; + "~*(?:\b)Extractor(?:\b)" 3; + "~*(?:\b)ExtractorPro(?:\b)" 3; + "~*(?:\b)Extreme\ Picture\ Finder(?:\b)" 3; + "~*(?:\b)EyeNetIE(?:\b)" 3; + "~*(?:\b)Ezooms(?:\b)" 3; + "~*(?:\b)facebookscraper(?:\b)" 3; + "~*(?:\b)FDM(?:\b)" 3; + "~*(?:\b)FemtosearchBot(?:\b)" 3; + "~*(?:\b)FHscan(?:\b)" 3; + "~*(?:\b)Fimap(?:\b)" 3; + "~*(?:\b)Firefox/7.0(?:\b)" 3; + "~*(?:\b)FlashGet(?:\b)" 3; + "~*(?:\b)Flunky(?:\b)" 3; + "~*(?:\b)Foobot(?:\b)" 3; + "~*(?:\b)Freeuploader(?:\b)" 3; + "~*(?:\b)FrontPage(?:\b)" 3; + "~*(?:\b)FyberSpider(?:\b)" 3; + "~*(?:\b)Fyrebot(?:\b)" 3; + "~*(?:\b)GalaxyBot(?:\b)" 3; + "~*(?:\b)Genieo(?:\b)" 3; + "~*(?:\b)GermCrawler(?:\b)" 3; + "~*(?:\b)Getintent(?:\b)" 3; + "~*(?:\b)GetRight(?:\b)" 3; + "~*(?:\b)GetWeb(?:\b)" 3; + "~*(?:\b)Gigablast(?:\b)" 3; + "~*(?:\b)Gigabot(?:\b)" 3; + "~*(?:\b)G-i-g-a-b-o-t(?:\b)" 3; + "~*(?:\b)Go-Ahead-Got-It(?:\b)" 3; + "~*(?:\b)Gotit(?:\b)" 3; + "~*(?:\b)GoZilla(?:\b)" 3; + "~*(?:\b)Go!Zilla(?:\b)" 3; + "~*(?:\b)Grabber(?:\b)" 3; + "~*(?:\b)GrabNet(?:\b)" 3; + "~*(?:\b)Grafula(?:\b)" 3; + "~*(?:\b)GrapeFX(?:\b)" 3; + "~*(?:\b)GrapeshotCrawler(?:\b)" 3; + "~*(?:\b)GridBot(?:\b)" 3; + "~*(?:\b)GT::WWW(?:\b)" 3; + "~*(?:\b)Haansoft(?:\b)" 3; + "~*(?:\b)HaosouSpider(?:\b)" 3; + "~*(?:\b)Harvest(?:\b)" 3; + "~*(?:\b)Havij(?:\b)" 3; + "~*(?:\b)HEADMasterSEO(?:\b)" 3; + "~*(?:\b)Heritrix(?:\b)" 3; + "~*(?:\b)Hloader(?:\b)" 3; + "~*(?:\b)HMView(?:\b)" 3; + "~*(?:\b)HTMLparser(?:\b)" 3; + "~*(?:\b)HTTP::Lite(?:\b)" 3; + "~*(?:\b)HTTrack(?:\b)" 3; + "~*(?:\b)Humanlinks(?:\b)" 3; + "~*(?:\b)HybridBot(?:\b)" 3; + "~*(?:\b)Iblog(?:\b)" 3; + "~*(?:\b)IDBot(?:\b)" 3; + "~*(?:\b)Id-search(?:\b)" 3; + "~*(?:\b)IlseBot(?:\b)" 3; + "~*(?:\b)Image\ Fetch(?:\b)" 3; + "~*(?:\b)Image\ Sucker(?:\b)" 3; + "~*(?:\b)IndeedBot(?:\b)" 3; + "~*(?:\b)Indy\ Library(?:\b)" 3; + "~*(?:\b)InfoNaviRobot(?:\b)" 3; + "~*(?:\b)InfoTekies(?:\b)" 3; + "~*(?:\b)instabid(?:\b)" 3; + "~*(?:\b)Intelliseek(?:\b)" 3; + "~*(?:\b)InterGET(?:\b)" 3; + "~*(?:\b)Internet\ Ninja(?:\b)" 3; + "~*(?:\b)InternetSeer(?:\b)" 3; + "~*(?:\b)internetVista\ monitor(?:\b)" 3; + "~*(?:\b)ips-agent(?:\b)" 3; + "~*(?:\b)Iria(?:\b)" 3; + "~*(?:\b)IRLbot(?:\b)" 3; + "~*(?:\b)Iskanie(?:\b)" 3; + "~*(?:\b)IstellaBot(?:\b)" 3; + "~*(?:\b)JamesBOT(?:\b)" 3; + "~*(?:\b)Jbrofuzz(?:\b)" 3; + "~*(?:\b)JennyBot(?:\b)" 3; + "~*(?:\b)JetCar(?:\b)" 3; + "~*(?:\b)Jetty(?:\b)" 3; + "~*(?:\b)JikeSpider(?:\b)" 3; + "~*(?:\b)JOC\ Web\ Spider(?:\b)" 3; + "~*(?:\b)Joomla(?:\b)" 3; + "~*(?:\b)Jorgee(?:\b)" 3; + "~*(?:\b)JustView(?:\b)" 3; + "~*(?:\b)Jyxobot(?:\b)" 3; + "~*(?:\b)Kenjin\ Spider(?:\b)" 3; + "~*(?:\b)Keyword\ Density(?:\b)" 3; + "~*(?:\b)Kozmosbot(?:\b)" 3; + "~*(?:\b)Lanshanbot(?:\b)" 3; + "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)LeechFTP(?:\b)" 3; + "~*(?:\b)LeechGet(?:\b)" 3; + "~*(?:\b)LexiBot(?:\b)" 3; + "~*(?:\b)Lftp(?:\b)" 3; + "~*(?:\b)LibWeb(?:\b)" 3; + "~*(?:\b)Libwhisker(?:\b)" 3; + "~*(?:\b)Lightspeedsystems(?:\b)" 3; + "~*(?:\b)Likse(?:\b)" 3; + "~*(?:\b)Linkdexbot(?:\b)" 3; + "~*(?:\b)LinkextractorPro(?:\b)" 3; + "~*(?:\b)LinkpadBot(?:\b)" 3; + "~*(?:\b)LinkScan(?:\b)" 3; + "~*(?:\b)LinksManager(?:\b)" 3; + "~*(?:\b)LinkWalker(?:\b)" 3; + "~*(?:\b)LinqiaMetadataDownloaderBot(?:\b)" 3; + "~*(?:\b)LinqiaRSSBot(?:\b)" 3; + "~*(?:\b)LinqiaScrapeBot(?:\b)" 3; + "~*(?:\b)Lipperhey(?:\b)" 3; + "~*(?:\b)Lipperhey\ Spider(?:\b)" 3; + "~*(?:\b)Litemage_walker(?:\b)" 3; + "~*(?:\b)Lmspider(?:\b)" 3; + "~*(?:\b)LNSpiderguy(?:\b)" 3; + "~*(?:\b)Ltx71(?:\b)" 3; + "~*(?:\b)lwp-request(?:\b)" 3; + "~*(?:\b)LWP::Simple(?:\b)" 3; + "~*(?:\b)lwp-trivial(?:\b)" 3; + "~*(?:\b)Magnet(?:\b)" 3; + "~*(?:\b)Mag-Net(?:\b)" 3; + "~*(?:\b)magpie-crawler(?:\b)" 3; + "~*(?:\b)Mail.RU_Bot(?:\b)" 3; + "~*(?:\b)Majestic12(?:\b)" 3; + "~*(?:\b)Majestic-SEO(?:\b)" 3; + "~*(?:\b)Majestic\ SEO(?:\b)" 3; + "~*(?:\b)MarkMonitor(?:\b)" 3; + "~*(?:\b)MarkWatch(?:\b)" 3; + "~*(?:\b)Masscan(?:\b)" 3; + "~*(?:\b)Mass\ Downloader(?:\b)" 3; + "~*(?:\b)Mata\ Hari(?:\b)" 3; + "~*(?:\b)MauiBot(?:\b)" 3; + "~*(?:\b)meanpathbot(?:\b)" 3; + "~*(?:\b)Meanpathbot(?:\b)" 3; + "~*(?:\b)MeanPath\ Bot(?:\b)" 3; + "~*(?:\b)Mediatoolkitbot(?:\b)" 3; + "~*(?:\b)mediawords(?:\b)" 3; + "~*(?:\b)MegaIndex.ru(?:\b)" 3; + "~*(?:\b)Metauri(?:\b)" 3; + "~*(?:\b)MFC_Tear_Sample(?:\b)" 3; + "~*(?:\b)Microsoft\ Data\ Access(?:\b)" 3; + "~*(?:\b)Microsoft\ URL\ Control(?:\b)" 3; + "~*(?:\b)MIDown\ tool(?:\b)" 3; + "~*(?:\b)MIIxpc(?:\b)" 3; + "~*(?:\b)Mister\ PiX(?:\b)" 3; + "~*(?:\b)MJ12bot(?:\b)" 3; + "~*(?:\b)Mojeek(?:\b)" 3; + "~*(?:\b)Morfeus\ Fucking\ Scanner(?:\b)" 3; + "~*(?:\b)Mr.4x3(?:\b)" 3; + "~*(?:\b)MSFrontPage(?:\b)" 3; + "~*(?:\b)MSIECrawler(?:\b)" 3; + "~*(?:\b)Msrabot(?:\b)" 3; + "~*(?:\b)MS\ Web\ Services\ Client\ Protocol(?:\b)" 3; + "~*(?:\b)muhstik-scan(?:\b)" 3; + "~*(?:\b)Musobot(?:\b)" 3; + "~*(?:\b)Name\ Intelligence(?:\b)" 3; + "~*(?:\b)Nameprotect(?:\b)" 3; + "~*(?:\b)Navroad(?:\b)" 3; + "~*(?:\b)NearSite(?:\b)" 3; + "~*(?:\b)Needle(?:\b)" 3; + "~*(?:\b)Nessus(?:\b)" 3; + "~*(?:\b)NetAnts(?:\b)" 3; + "~*(?:\b)Netcraft(?:\b)" 3; + "~*(?:\b)netEstate\ NE\ Crawler(?:\b)" 3; + "~*(?:\b)NetLyzer(?:\b)" 3; + "~*(?:\b)NetMechanic(?:\b)" 3; + "~*(?:\b)NetSpider(?:\b)" 3; + "~*(?:\b)Nettrack(?:\b)" 3; + "~*(?:\b)Net\ Vampire(?:\b)" 3; + "~*(?:\b)Netvibes(?:\b)" 3; + "~*(?:\b)NetZIP(?:\b)" 3; + "~*(?:\b)NextGenSearchBot(?:\b)" 3; + "~*(?:\b)Nibbler(?:\b)" 3; + "~*(?:\b)NICErsPRO(?:\b)" 3; + "~*(?:\b)Niki-bot(?:\b)" 3; + "~*(?:\b)Nikto(?:\b)" 3; + "~*(?:\b)NimbleCrawler(?:\b)" 3; + "~*(?:\b)Nimbostratus(?:\b)" 3; + "~*(?:\b)Ninja(?:\b)" 3; + "~*(?:\b)Nmap(?:\b)" 3; + "~*(?:\b)NPbot(?:\b)" 3; + "~*(?:\b)Nutch(?:\b)" 3; + "~*(?:\b)oBot(?:\b)" 3; + "~*(?:\b)Octopus(?:\b)" 3; + "~*(?:\b)Offline\ Explorer(?:\b)" 3; + "~*(?:\b)Offline\ Navigator(?:\b)" 3; + "~*(?:\b)OnCrawl(?:\b)" 3; + "~*(?:\b)Openfind(?:\b)" 3; + "~*(?:\b)OpenLinkProfiler(?:\b)" 3; + "~*(?:\b)Openvas(?:\b)" 3; + "~*(?:\b)OpenVAS(?:\b)" 3; + "~*(?:\b)OrangeBot(?:\b)" 3; + "~*(?:\b)OrangeSpider(?:\b)" 3; + "~*(?:\b)OutclicksBot(?:\b)" 3; + "~*(?:\b)OutfoxBot(?:\b)" 3; + "~*(?:\b)PageAnalyzer(?:\b)" 3; + "~*(?:\b)Page\ Analyzer(?:\b)" 3; + "~*(?:\b)PageGrabber(?:\b)" 3; + "~*(?:\b)page\ scorer(?:\b)" 3; + "~*(?:\b)PageScorer(?:\b)" 3; + "~*(?:\b)Pandalytics(?:\b)" 3; + "~*(?:\b)Panscient(?:\b)" 3; + "~*(?:\b)Papa\ Foto(?:\b)" 3; + "~*(?:\b)Pavuk(?:\b)" 3; + "~*(?:\b)pcBrowser(?:\b)" 3; + "~*(?:\b)PECL::HTTP(?:\b)" 3; + "~*(?:\b)PeoplePal(?:\b)" 3; + "~*(?:\b)PHPCrawl(?:\b)" 3; + "~*(?:\b)Picscout(?:\b)" 3; + "~*(?:\b)Picsearch(?:\b)" 3; + "~*(?:\b)PictureFinder(?:\b)" 3; + "~*(?:\b)Pimonster(?:\b)" 3; + "~*(?:\b)Pi-Monster(?:\b)" 3; + "~*(?:\b)Pixray(?:\b)" 3; + "~*(?:\b)PleaseCrawl(?:\b)" 3; + "~*(?:\b)plumanalytics(?:\b)" 3; + "~*(?:\b)Pockey(?:\b)" 3; + "~*(?:\b)POE-Component-Client-HTTP(?:\b)" 3; + "~*(?:\b)Probethenet(?:\b)" 3; + "~*(?:\b)ProPowerBot(?:\b)" 3; + "~*(?:\b)ProWebWalker(?:\b)" 3; + "~*(?:\b)Psbot(?:\b)" 3; + "~*(?:\b)Pump(?:\b)" 3; + "~*(?:\b)PxBroker(?:\b)" 3; + "~*(?:\b)PyCurl(?:\b)" 3; + "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; + "~*(?:\b)Quick-Crawler(?:\b)" 3; + "~*(?:\b)RankActive(?:\b)" 3; + "~*(?:\b)RankActiveLinkBot(?:\b)" 3; + "~*(?:\b)RankFlex(?:\b)" 3; + "~*(?:\b)RankingBot(?:\b)" 3; + "~*(?:\b)RankingBot2(?:\b)" 3; + "~*(?:\b)Rankivabot(?:\b)" 3; + "~*(?:\b)RankurBot(?:\b)" 3; + "~*(?:\b)RealDownload(?:\b)" 3; + "~*(?:\b)Reaper(?:\b)" 3; + "~*(?:\b)RebelMouse(?:\b)" 3; + "~*(?:\b)Recorder(?:\b)" 3; + "~*(?:\b)RedesScrapy(?:\b)" 3; + "~*(?:\b)ReGet(?:\b)" 3; + "~*(?:\b)RepoMonkey(?:\b)" 3; + "~*(?:\b)Ripper(?:\b)" 3; + "~*(?:\b)RocketCrawler(?:\b)" 3; + "~*(?:\b)Rogerbot(?:\b)" 3; + "~*(?:\b)s1z.ru(?:\b)" 3; + "~*(?:\b)SalesIntelligent(?:\b)" 3; + "~*(?:\b)SBIder(?:\b)" 3; + "~*(?:\b)ScanAlert(?:\b)" 3; + "~*(?:\b)Scanbot(?:\b)" 3; + "~*(?:\b)scan.lol(?:\b)" 3; + "~*(?:\b)ScoutJet(?:\b)" 3; + "~*(?:\b)Scrapy(?:\b)" 3; + "~*(?:\b)Screaming(?:\b)" 3; + "~*(?:\b)ScreenerBot(?:\b)" 3; + "~*(?:\b)Searchestate(?:\b)" 3; + "~*(?:\b)SearchmetricsBot(?:\b)" 3; + "~*(?:\b)Semrush(?:\b)" 3; + "~*(?:\b)SemrushBot(?:\b)" 3; + "~*(?:\b)SEOkicks(?:\b)" 3; + "~*(?:\b)SEOkicks-Robot(?:\b)" 3; + "~*(?:\b)SEOlyticsCrawler(?:\b)" 3; + "~*(?:\b)Seomoz(?:\b)" 3; + "~*(?:\b)SEOprofiler(?:\b)" 3; + "~*(?:\b)seoscanners(?:\b)" 3; + "~*(?:\b)SeoSiteCheckup(?:\b)" 3; + "~*(?:\b)SEOstats(?:\b)" 3; + "~*(?:\b)serpstatbot(?:\b)" 3; + "~*(?:\b)sexsearcher(?:\b)" 3; + "~*(?:\b)Shodan(?:\b)" 3; + "~*(?:\b)Siphon(?:\b)" 3; + "~*(?:\b)SISTRIX(?:\b)" 3; + "~*(?:\b)Sitebeam(?:\b)" 3; + "~*(?:\b)SiteExplorer(?:\b)" 3; + "~*(?:\b)Siteimprove(?:\b)" 3; + "~*(?:\b)SiteLockSpider(?:\b)" 3; + "~*(?:\b)SiteSnagger(?:\b)" 3; + "~*(?:\b)SiteSucker(?:\b)" 3; + "~*(?:\b)Site\ Sucker(?:\b)" 3; + "~*(?:\b)Sitevigil(?:\b)" 3; + "~*(?:\b)SlySearch(?:\b)" 3; + "~*(?:\b)SmartDownload(?:\b)" 3; + "~*(?:\b)SMTBot(?:\b)" 3; + "~*(?:\b)Snake(?:\b)" 3; + "~*(?:\b)Snapbot(?:\b)" 3; + "~*(?:\b)Snoopy(?:\b)" 3; + "~*(?:\b)SocialRankIOBot(?:\b)" 3; + "~*(?:\b)Sociscraper(?:\b)" 3; + "~*(?:\b)sogouspider(?:\b)" 3; + "~*(?:\b)Sogou\ web\ spider(?:\b)" 3; + "~*(?:\b)Sosospider(?:\b)" 3; + "~*(?:\b)Sottopop(?:\b)" 3; + "~*(?:\b)SpaceBison(?:\b)" 3; + "~*(?:\b)Spammen(?:\b)" 3; + "~*(?:\b)SpankBot(?:\b)" 3; + "~*(?:\b)Spanner(?:\b)" 3; + "~*(?:\b)sp_auditbot(?:\b)" 3; + "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spinn3r(?:\b)" 3; + "~*(?:\b)SputnikBot(?:\b)" 3; + "~*(?:\b)spyfu(?:\b)" 3; + "~*(?:\b)Sqlmap(?:\b)" 3; + "~*(?:\b)Sqlworm(?:\b)" 3; + "~*(?:\b)Sqworm(?:\b)" 3; + "~*(?:\b)Steeler(?:\b)" 3; + "~*(?:\b)Stripper(?:\b)" 3; + "~*(?:\b)Sucker(?:\b)" 3; + "~*(?:\b)Sucuri(?:\b)" 3; + "~*(?:\b)SuperBot(?:\b)" 3; + "~*(?:\b)SuperHTTP(?:\b)" 3; + "~*(?:\b)Surfbot(?:\b)" 3; + "~*(?:\b)SurveyBot(?:\b)" 3; + "~*(?:\b)Suzuran(?:\b)" 3; + "~*(?:\b)Swiftbot(?:\b)" 3; + "~*(?:\b)sysscan(?:\b)" 3; + "~*(?:\b)Szukacz(?:\b)" 3; + "~*(?:\b)T0PHackTeam(?:\b)" 3; + "~*(?:\b)T8Abot(?:\b)" 3; + "~*(?:\b)tAkeOut(?:\b)" 3; + "~*(?:\b)Teleport(?:\b)" 3; + "~*(?:\b)TeleportPro(?:\b)" 3; + "~*(?:\b)Telesoft(?:\b)" 3; + "~*(?:\b)Telesphoreo(?:\b)" 3; + "~*(?:\b)Telesphorep(?:\b)" 3; + "~*(?:\b)The\ Intraformant(?:\b)" 3; + "~*(?:\b)TheNomad(?:\b)" 3; + "~*(?:\b)Thumbor(?:\b)" 3; + "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)Titan(?:\b)" 3; + "~*(?:\b)Toata(?:\b)" 3; + "~*(?:\b)Toweyabot(?:\b)" 3; + "~*(?:\b)Tracemyfile(?:\b)" 3; + "~*(?:\b)Trendiction(?:\b)" 3; + "~*(?:\b)Trendictionbot(?:\b)" 3; + "~*(?:\b)trendiction.com(?:\b)" 3; + "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)True_Robot(?:\b)" 3; + "~*(?:\b)Turingos(?:\b)" 3; + "~*(?:\b)Turnitin(?:\b)" 3; + "~*(?:\b)TurnitinBot(?:\b)" 3; + "~*(?:\b)TwengaBot(?:\b)" 3; + "~*(?:\b)Twice(?:\b)" 3; + "~*(?:\b)Typhoeus(?:\b)" 3; + "~*(?:\b)UnisterBot(?:\b)" 3; + "~*(?:\b)Upflow(?:\b)" 3; + "~*(?:\b)URLy.Warning(?:\b)" 3; + "~*(?:\b)URLy\ Warning(?:\b)" 3; + "~*(?:\b)Vacuum(?:\b)" 3; + "~*(?:\b)Vagabondo(?:\b)" 3; + "~*(?:\b)VB\ Project(?:\b)" 3; + "~*(?:\b)VCI(?:\b)" 3; + "~*(?:\b)VeriCiteCrawler(?:\b)" 3; + "~*(?:\b)VidibleScraper(?:\b)" 3; + "~*(?:\b)Virusdie(?:\b)" 3; + "~*(?:\b)VoidEYE(?:\b)" 3; + "~*(?:\b)Voil(?:\b)" 3; + "~*(?:\b)Voltron(?:\b)" 3; + "~*(?:\b)Wallpapers/3.0(?:\b)" 3; + "~*(?:\b)WallpapersHD(?:\b)" 3; + "~*(?:\b)WASALive-Bot(?:\b)" 3; + "~*(?:\b)WBSearchBot(?:\b)" 3; + "~*(?:\b)Webalta(?:\b)" 3; + "~*(?:\b)WebAuto(?:\b)" 3; + "~*(?:\b)Web\ Auto(?:\b)" 3; + "~*(?:\b)WebBandit(?:\b)" 3; + "~*(?:\b)WebCollage(?:\b)" 3; + "~*(?:\b)Web\ Collage(?:\b)" 3; + "~*(?:\b)WebCopier(?:\b)" 3; + "~*(?:\b)WEBDAV(?:\b)" 3; + "~*(?:\b)WebEnhancer(?:\b)" 3; + "~*(?:\b)Web\ Enhancer(?:\b)" 3; + "~*(?:\b)WebFetch(?:\b)" 3; + "~*(?:\b)Web\ Fetch(?:\b)" 3; + "~*(?:\b)WebFuck(?:\b)" 3; + "~*(?:\b)Web\ Fuck(?:\b)" 3; + "~*(?:\b)WebGo\ IS(?:\b)" 3; + "~*(?:\b)WebImageCollector(?:\b)" 3; + "~*(?:\b)WebLeacher(?:\b)" 3; + "~*(?:\b)WebmasterWorldForumBot(?:\b)" 3; + "~*(?:\b)webmeup-crawler(?:\b)" 3; + "~*(?:\b)WebPix(?:\b)" 3; + "~*(?:\b)Web\ Pix(?:\b)" 3; + "~*(?:\b)WebReaper(?:\b)" 3; + "~*(?:\b)WebSauger(?:\b)" 3; + "~*(?:\b)Web\ Sauger(?:\b)" 3; + "~*(?:\b)Webshag(?:\b)" 3; + "~*(?:\b)WebsiteExtractor(?:\b)" 3; + "~*(?:\b)WebsiteQuester(?:\b)" 3; + "~*(?:\b)Website\ Quester(?:\b)" 3; + "~*(?:\b)Webster(?:\b)" 3; + "~*(?:\b)WebStripper(?:\b)" 3; + "~*(?:\b)WebSucker(?:\b)" 3; + "~*(?:\b)Web\ Sucker(?:\b)" 3; + "~*(?:\b)WebWhacker(?:\b)" 3; + "~*(?:\b)WebZIP(?:\b)" 3; + "~*(?:\b)WeSEE(?:\b)" 3; + "~*(?:\b)Whack(?:\b)" 3; + "~*(?:\b)Whacker(?:\b)" 3; + "~*(?:\b)Whatweb(?:\b)" 3; + "~*(?:\b)Who.is\ Bot(?:\b)" 3; + "~*(?:\b)Widow(?:\b)" 3; + "~*(?:\b)WinHTTrack(?:\b)" 3; + "~*(?:\b)WiseGuys\ Robot(?:\b)" 3; + "~*(?:\b)WISENutbot(?:\b)" 3; + "~*(?:\b)Wonderbot(?:\b)" 3; + "~*(?:\b)Woobot(?:\b)" 3; + "~*(?:\b)Wotbox(?:\b)" 3; + "~*(?:\b)Wprecon(?:\b)" 3; + "~*(?:\b)WPScan(?:\b)" 3; + "~*(?:\b)WWW-Collector-E(?:\b)" 3; + "~*(?:\b)WWW-Mechanize(?:\b)" 3; + "~*(?:\b)WWW::Mechanize(?:\b)" 3; + "~*(?:\b)WWWOFFLE(?:\b)" 3; + "~*(?:\b)x09Mozilla(?:\b)" 3; + "~*(?:\b)x22Mozilla(?:\b)" 3; + "~*(?:\b)Xaldon_WebSpider(?:\b)" 3; + "~*(?:\b)Xaldon\ WebSpider(?:\b)" 3; + "~*(?:\b)Xenu(?:\b)" 3; + "~*(?:\b)xpymep1.exe(?:\b)" 3; + "~*(?:\b)YoudaoBot(?:\b)" 3; + "~*(?:\b)Zade(?:\b)" 3; + "~*(?:\b)Zauba(?:\b)" 3; + "~*(?:\b)zauba.io(?:\b)" 3; + "~*(?:\b)Zermelo(?:\b)" 3; + "~*(?:\b)Zeus(?:\b)" 3; + "~*(?:\b)zgrab(?:\b)" 3; + "~*(?:\b)Zitebot(?:\b)" 3; + "~*(?:\b)ZmEu(?:\b)" 3; + "~*(?:\b)ZumBot(?:\b)" 3; + "~*(?:\b)ZyBorg(?:\b)" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------- @@ -717,38 +717,38 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)adidxbot(?:\b|)" 0; - "~*(?:\b)AdsBot-Google(?:\b|)" 0; - "~*(?:\b)aolbuild(?:\b|)" 0; - "~*(?:\b)bingbot(?:\b|)" 0; - "~*(?:\b)bingpreview(?:\b|)" 0; - "~*(?:\b)DoCoMo(?:\b|)" 0; - "~*(?:\b)duckduckgo(?:\b|)" 0; - "~*(?:\b)facebookexternalhit(?:\b|)" 0; - "~*(?:\b)Feedfetcher-Google(?:\b|)" 0; - "~*(?:\b)Googlebot(?:\b|)" 0; - "~*(?:\b)Googlebot-Image(?:\b|)" 0; - "~*(?:\b)Googlebot-Mobile(?:\b|)" 0; - "~*(?:\b)Googlebot-News(?:\b|)" 0; - "~*(?:\b)Googlebot/Test(?:\b|)" 0; - "~*(?:\b)Googlebot-Video(?:\b|)" 0; - "~*(?:\b)Google-HTTP-Java-Client(?:\b|)" 0; - "~*(?:\b)Gravityscan(?:\b|)" 0; - "~*(?:\b)gsa-crawler(?:\b|)" 0; - "~*(?:\b)Jakarta\ Commons(?:\b|)" 0; - "~*(?:\b)Kraken/0.1(?:\b|)" 0; - "~*(?:\b)LinkedInBot(?:\b|)" 0; - "~*(?:\b)Mediapartners-Google(?:\b|)" 0; - "~*(?:\b)msnbot(?:\b|)" 0; - "~*(?:\b)msnbot-media(?:\b|)" 0; - "~*(?:\b)SAMSUNG(?:\b|)" 0; - "~*(?:\b)Slackbot(?:\b|)" 0; - "~*(?:\b)Slackbot-LinkExpanding(?:\b|)" 0; - "~*(?:\b)slurp(?:\b|)" 0; - "~*(?:\b)teoma(?:\b|)" 0; - "~*(?:\b)TwitterBot(?:\b|)" 0; - "~*(?:\b)Wordpress(?:\b|)" 0; - "~*(?:\b)yahoo(?:\b|)" 0; + "~*(?:\b)adidxbot(?:\b)" 0; + "~*(?:\b)AdsBot-Google(?:\b)" 0; + "~*(?:\b)aolbuild(?:\b)" 0; + "~*(?:\b)bingbot(?:\b)" 0; + "~*(?:\b)bingpreview(?:\b)" 0; + "~*(?:\b)DoCoMo(?:\b)" 0; + "~*(?:\b)duckduckgo(?:\b)" 0; + "~*(?:\b)facebookexternalhit(?:\b)" 0; + "~*(?:\b)Feedfetcher-Google(?:\b)" 0; + "~*(?:\b)Googlebot(?:\b)" 0; + "~*(?:\b)Googlebot-Image(?:\b)" 0; + "~*(?:\b)Googlebot-Mobile(?:\b)" 0; + "~*(?:\b)Googlebot-News(?:\b)" 0; + "~*(?:\b)Googlebot/Test(?:\b)" 0; + "~*(?:\b)Googlebot-Video(?:\b)" 0; + "~*(?:\b)Google-HTTP-Java-Client(?:\b)" 0; + "~*(?:\b)Gravityscan(?:\b)" 0; + "~*(?:\b)gsa-crawler(?:\b)" 0; + "~*(?:\b)Jakarta\ Commons(?:\b)" 0; + "~*(?:\b)Kraken/0.1(?:\b)" 0; + "~*(?:\b)LinkedInBot(?:\b)" 0; + "~*(?:\b)Mediapartners-Google(?:\b)" 0; + "~*(?:\b)msnbot(?:\b)" 0; + "~*(?:\b)msnbot-media(?:\b)" 0; + "~*(?:\b)SAMSUNG(?:\b)" 0; + "~*(?:\b)Slackbot(?:\b)" 0; + "~*(?:\b)Slackbot-LinkExpanding(?:\b)" 0; + "~*(?:\b)slurp(?:\b)" 0; + "~*(?:\b)teoma(?:\b)" 0; + "~*(?:\b)TwitterBot(?:\b)" 0; + "~*(?:\b)Wordpress(?:\b)" 0; + "~*(?:\b)yahoo(?:\b)" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # -------------------------------------------------------- @@ -760,12 +760,12 @@ map $http_user_agent $bad_bot { #limit_req zone=bot1_reqlimitip burst=50; # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)jetmon(?:\b|)" 1; - "~*(?:\b)libwww-perl(?:\b|)" 1; - "~*(?:\b)Lynx(?:\b|)" 1; - "~*(?:\b)munin(?:\b|)" 1; - "~*(?:\b)Presto(?:\b|)" 1; - "~*(?:\b)Wget/1.15(?:\b|)" 1; + "~*(?:\b)jetmon(?:\b)" 1; + "~*(?:\b)libwww-perl(?:\b)" 1; + "~*(?:\b)Lynx(?:\b)" 1; + "~*(?:\b)munin(?:\b)" 1; + "~*(?:\b)Presto(?:\b)" 1; + "~*(?:\b)Wget/1.15(?:\b)" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ------------------------------------------------------- @@ -778,23 +778,23 @@ map $http_user_agent $bad_bot { # ----------------------------------------------------------------------- # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*(?:\b)Alexa(?:\b|)" 2; - "~*(?:\b)archive.org(?:\b|)" 2; - "~*(?:\b)Baidu(?:\b|)" 2; - "~*(?:\b)BUbiNG(?:\b|)" 2; - "~*(?:\b)FlipboardProxy(?:\b|)" 2; - "~*(?:\b)ia_archiver(?:\b|)" 2; - "~*(?:\b)MSIE\ 7.0(?:\b|)" 2; - "~*(?:\b)Proximic(?:\b|)" 2; - "~*(?:\b)R6_CommentReader(?:\b|)" 2; - "~*(?:\b)R6_FeedFetcher(?:\b|)" 2; - "~*(?:\b)RED/1(?:\b|)" 2; - "~*(?:\b)RPT-HTTPClient(?:\b|)" 2; - "~*(?:\b)sfFeedReader/0.9(?:\b|)" 2; - "~*(?:\b)Spaidu(?:\b|)" 2; - "~*(?:\b)UptimeRobot/2.0(?:\b|)" 2; - "~*(?:\b)YandexBot(?:\b|)" 2; - "~*(?:\b)YandexImages(?:\b|)" 2; + "~*(?:\b)Alexa(?:\b)" 2; + "~*(?:\b)archive.org(?:\b)" 2; + "~*(?:\b)Baidu(?:\b)" 2; + "~*(?:\b)BUbiNG(?:\b)" 2; + "~*(?:\b)FlipboardProxy(?:\b)" 2; + "~*(?:\b)ia_archiver(?:\b)" 2; + "~*(?:\b)MSIE\ 7.0(?:\b)" 2; + "~*(?:\b)Proximic(?:\b)" 2; + "~*(?:\b)R6_CommentReader(?:\b)" 2; + "~*(?:\b)R6_FeedFetcher(?:\b)" 2; + "~*(?:\b)RED/1(?:\b)" 2; + "~*(?:\b)RPT-HTTPClient(?:\b)" 2; + "~*(?:\b)sfFeedReader/0.9(?:\b)" 2; + "~*(?:\b)Spaidu(?:\b)" 2; + "~*(?:\b)UptimeRobot/2.0(?:\b)" 2; + "~*(?:\b)YandexBot(?:\b)" 2; + "~*(?:\b)YandexImages(?:\b)" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/.dev-tools/test_units/blacklist-ips.conf b/.dev-tools/test_units/blacklist-ips.conf index 6893ce0fc..b9d24324a 100644 --- a/.dev-tools/test_units/blacklist-ips.conf +++ b/.dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -35.192.136.167 1; +34.68.144.114 1; 127.0.0.1 1; diff --git a/.dev-tools/test_units/random-bots-for-test-quick.list b/.dev-tools/test_units/random-bots-for-test-quick.list index 489325858..d91f5f655 100644 --- a/.dev-tools/test_units/random-bots-for-test-quick.list +++ b/.dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Aboundex -BetaBot -ECCP/1.0 -Iria -JetCar -Kozmosbot -MauiBot -Mojeek -OpenLinkProfiler -WebBandit +BackDoorBot +Gigablast +NextGenSearchBot +Nibbler +pcBrowser +Pixray +Reaper +SEOlyticsCrawler +Sitevigil +Sottopop diff --git a/.dev-tools/test_units/random-bots-for-test.list b/.dev-tools/test_units/random-bots-for-test.list index bed60124e..396bccc27 100644 --- a/.dev-tools/test_units/random-bots-for-test.list +++ b/.dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ 404enemy 80legs -Abonti Acunetix ADmantX -AIBOT -AiHitBot -Alexibot -AllSubmitter +Alligator AlphaBot Apexoo +ASPSeek Asterias +Attach +autoemailspider +Backlink-Ceck +backlink-check +BackStreet +BackWeb Badass -Barkrowler -BatchFTP Battleztar Bazinga -Bigfoot -Bitacle +BetaBot Black Hole -BLEXBot -Boardreader -Bolt +BlackWidow +BotALot +Brandprotect Brandwatch BuiltBotTough +BuiltWith BunnySlippers -BuzzSumo -Cegbfeieh +Calculon +CazoodleBot CheeseBot -CherryPicker CheTeam -Chlooe +ChinaClaw Claritybot Cliqzbot coccocbot-web -Cogentbot -cognitiveseo -Collector -com.plumanalytics -CopyRightCheck Copyscape -Cosmos -crawler4j +Craftbot crawler.feedback -CSHttp -demandbase-bot -Devil -DigitalPebble +Custo +DBLBot +Demon +Deusu +Digincore Dirbuster +Disco +Discobot Discoverybot -Drip -DTS Agent +DomainSigmaCrawler +EasyDL ECCP/1.0 -Ecxi EirGrabber -evc-batch -Evil -Exabot +EMail Siphon +EMail Wolf +Express WebPictures ExtLinksBot -EyeNetIE -Firefox/7.0 -Flunky -FyberSpider +Extractor +ExtractorPro +Ezooms +facebookscraper +FDM +FHscan +Fimap +FlashGet +Freeuploader GalaxyBot -GermCrawler GetRight -GetWeb Gigabot G-i-g-a-b-o-t -Go-Ahead-Got-It -Grabber +GoZilla +Go!Zilla +GrabNet GrapeFX +GrapeshotCrawler +GridBot GT::WWW +HaosouSpider Harvest -Havij Heritrix -HTTP::Lite +Hloader +HTMLparser Humanlinks -HybridBot -IDBot +Iblog +Id-search +Image Fetch Image Sucker -IndeedBot -InfoTekies +Indy Library +InfoNaviRobot +Intelliseek InterGET internetVista monitor ips-agent +Iria +IRLbot +IstellaBot +JamesBOT Jbrofuzz +JennyBot JikeSpider -Keyword Density +Joomla +Jorgee +Jyxobot Kozmosbot -LeechGet -Libwhisker -Lightspeedsystems -Likse -LinkextractorPro -LinkWalker +LexiBot +LibWeb +Linkdexbot +LinqiaMetadataDownloaderBot +LinqiaRSSBot Lipperhey -Lipperhey Spider +Litemage_walker Lmspider LNSpiderguy +Ltx71 lwp-request lwp-trivial Magnet -Mag-Net -magpie-crawler +Majestic-SEO MarkMonitor MarkWatch -Mass Downloader -MauiBot -Meanpathbot -MeanPath Bot +Masscan +Mata Hari +meanpathbot +Mediatoolkitbot mediawords -MegaIndex.ru -Microsoft Data Access -Microsoft URL Control +Metauri MIIxpc -MSFrontPage +Mister PiX +MJ12bot +Mojeek MSIECrawler Msrabot -muhstik-scan +Musobot Name Intelligence -Nessus -NetMechanic +Nameprotect +Navroad +Needle +Netcraft +netEstate NE Crawler NetSpider -Nettrack Net Vampire -Nibbler NICErsPRO -Niki-bot +Nikto +NimbleCrawler Nimbostratus Nmap -NPbot -Nutch oBot Offline Explorer +OnCrawl OpenLinkProfiler +Openvas +OpenVAS +OrangeBot OrangeSpider +OutclicksBot OutfoxBot -PageAnalyzer -Page Analyzer -PageGrabber page scorer -Pandalytics -Panscient -pcBrowser -PHPCrawl +PageScorer +Papa Foto +PeoplePal Picscout -Picsearch -Pixray +Pimonster PleaseCrawl plumanalytics -Pockey -POE-Component-Client-HTTP -Probethenet -ProWebWalker -Psbot -Pump +ProPowerBot +PxBroker +PyCurl QueryN Metasearch -Quick-Crawler -RankActive RankActiveLinkBot -RankFlex RankingBot RankingBot2 -RealDownload -Reaper -RepoMonkey +ReGet +Ripper s1z.ru -Scrapy +ScanAlert +scan.lol ScreenerBot -Searchestate +Semrush SemrushBot -SEOkicks -SEOkicks-Robot Seomoz -seoscanners -SeoSiteCheckup SEOstats serpstatbot -Shodan -Siphon SISTRIX -Site Sucker -Snake -Snoopy +SiteExplorer +SiteSucker +SmartDownload +SMTBot +SocialRankIOBot sogouspider +Sogou web spider +Sottopop Spammen -SpankBot -Spanner sp_auditbot +Spinn3r spyfu -Sqlworm -Sqworm -Stripper -Sucker -SuperHTTP +Sqlmap +Steeler +Sucuri +SuperBot +Surfbot +SurveyBot Swiftbot -sysscan -Szukacz -T8Abot -tAkeOut -Teleport -Telesoft -Telesphorep +T0PHackTeam +TheNomad TightTwatBot -Trendiction -Turnitin -Typhoeus -Upflow -VB Project -VCI -Virusdie +Toata +Toweyabot +Trendictionbot +trendiction.com +TwengaBot +Twice +URLy Warning +VidibleScraper Voil -Wallpapers/3.0 WallpapersHD -WBSearchBot +WASALive-Bot +Webalta WebAuto Web Auto +WebBandit WebCollage -WebCopier -WEBDAV +Web Collage WebEnhancer -Web Enhancer +WebFetch Web Fetch +Web Fuck +WebGo IS WebImageCollector -WebmasterWorldForumBot -webmeup-crawler +WebLeacher WebPix -Web Pix -Web Sauger -Web Sucker -WebZIP -WeSEE -Whack -Who.is Bot -Widow +WebReaper +WebSauger +WebsiteQuester +Website Quester +Webster +WebStripper +WebWhacker +Whacker +Whatweb +WiseGuys Robot +WISENutbot Wonderbot -Wprecon +Woobot +Wotbox +WPScan WWW-Mechanize +WWW::Mechanize WWWOFFLE -x09Mozilla Xaldon WebSpider +Xaldon_WebSpider Xenu -xpymep1.exe -YoudaoBot -Zade -Zermelo -Zeus -zgrab -Zitebot +Zauba ZyBorg diff --git a/.dev-tools/test_units/random-bots-for-whitelist-test.list b/.dev-tools/test_units/random-bots-for-whitelist-test.list index 75f38a09b..157672892 100644 --- a/.dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/.dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ +360Spider +404enemy Aboundexbot -Acunetix -ADmantX -AiHitBot +AfD-Verbotsverfahren +AhrefsBot +Aipbot Alexibot -AllSubmitter -Anarchie +Alligator +AlphaBot archive.org_bot -ASPSeek Attach -autoemailspider +BackDoorBot BacklinkCrawler BackStreet -Badass +BackWeb +Bandit Barkrowler -BatchFTP +Battleztar Bazinga BDCbot -BetaBot -Bitacle -Black Hole +Bigfoot BlackWidow -BLEXBot -Blow -BlowFish -Boardreader BotALot -Brandprotect BuiltBotTough -Bullseye -CATExplorador -Cliqzbot -Collector +BuiltWith +Calculon +CazoodleBot +CCBot +Cegbfeieh +CherryPicker +Chlooe +Claritybot +coccocbot-web +Cogentbot +cognitiveseo com.plumanalytics -Copyscape -Cosmos -Craftbot -crawler4j -CrazyWebCrawler -Curious +crawler.feedback +CSHttp Custo +DatabaseDriverMysqli demandbase-bot -Demon Deusu -DigitalPebble -DIIbot -Disco -Dispatch -DittoSpyder +Dirbuster +Discoverybot DomainAppender DomainCrawler -DomainSigmaCrawler -DomainStatsBot Dotbot Download Wonder Dragonfly -eCatch +Ebingbong Ecxi EirGrabber -EMail Siphon -EMail Wolf EroCrawler Evil -Exabot -Express WebPictures ExtLinksBot -EyeNetIE -Fimap +Extractor +Extreme Picture Finder +facebookscraper +FDM +Firefox/7.0 FlashGet Flunky -Foobot -FyberSpider -Fyrebot -GalaxyBot -Getintent Gigabot -G-i-g-a-b-o-t -Go!Zilla +Go-Ahead-Got-It +GoZilla Grabber -GrabNet -GrapeFX -GridBot +GrapeshotCrawler +Haansoft HaosouSpider -Harvest Havij +HEADMasterSEO +Heritrix +Hloader +HTTP::Lite HTTrack Humanlinks -Id-search -Image Fetch -Image Sucker +Iblog +IDBot IndeedBot +InfoTekies InterGET -Internet Ninja internetVista monitor ips-agent -Iskanie +Iria +IRLbot IstellaBot -JamesBOT -Jetty -JOC Web Spider -Keyword Density +JustView +Kenjin Spider Kozmosbot +Larbin +LexiBot Lftp -Libwhisker -Likse Linkdexbot -LinkextractorPro -LinkScan -LinksManager +LinkpadBot +LinkWalker LinqiaMetadataDownloaderBot -LinqiaRSSBot -Lipperhey Spider +Lipperhey Litemage_walker Lmspider -LNSpiderguy -lwp-request -lwp-trivial +Magnet Mag-Net magpie-crawler -Mail.RU_Bot Majestic12 Majestic SEO -MarkMonitor -Masscan +Majestic-SEO Mass Downloader Mata Hari -Meanpathbot MeanPath Bot -Metauri -Microsoft Data Access +mediawords +MFC_Tear_Sample Microsoft URL Control MIDown tool MIIxpc -Mister PiX -MJ12bot -Mojeek +Morfeus Fucking Scanner Mr.4x3 -MSIECrawler +Msrabot Name Intelligence Nameprotect -Navroad -Nessus +NetAnts netEstate NE Crawler NetMechanic -NetSpider +Nettrack +Net Vampire Netvibes -NextGenSearchBot -NimbleCrawler +Nibbler +NICErsPRO +Niki-bot +Nikto +Nimbostratus Nmap -oBot +NPbot +Nutch +Octopus +Offline Navigator Openfind OpenLinkProfiler OpenVAS OrangeBot -OrangeSpider +OutclicksBot OutfoxBot PageGrabber page scorer -PageScorer +Pandalytics Panscient -Papa Foto +Pavuk PECL::HTTP -PeoplePal -PictureFinder -ProWebWalker -PyCurl +PHPCrawl +Picscout +Picsearch +Pi-Monster +PleaseCrawl +plumanalytics +Pockey +POE-Component-Client-HTTP +ProPowerBot RankActive RankFlex RankingBot2 Rankivabot -Reaper +RealDownload +RebelMouse Recorder RedesScrapy -Rogerbot +Ripper ScanAlert -scan.lol -ScoutJet +Scrapy Screaming -Searchestate +SearchmetricsBot +SemrushBot SEOkicks-Robot -Seomoz -SEOprofiler seoscanners +SeoSiteCheckup SEOstats +serpstatbot +sexsearcher +Shodan Siphon SISTRIX -SMTBot +Siteimprove +SiteSucker +SlySearch +SmartDownload Snake +Snoopy +SocialRankIOBot sogouspider Sogou web spider -Spammen +SpaceBison Spanner -SputnikBot +Spinn3r Sqlmap Sqlworm -Sqworm +Steeler Stripper -SuperHTTP -Surfbot +Sucker +Suzuran Swiftbot +sysscan +T0PHackTeam T8Abot tAkeOut -Telesoft +Telesphoreo Telesphorep -Thumbor -TightTwatBot +The Intraformant Titan -Toata -Trendiction Trendictionbot -trendiction.com -Turingos -TurnitinBot -TwengaBot -URLy Warning -URLy.Warning +Twice +Typhoeus +Upflow Vacuum -Vagabondo VCI +VeriCiteCrawler Virusdie -Wallpapers/3.0 -Webalta -WebAuto +Voil +WBSearchBot Web Auto -WebCollage +WebBandit +Web Collage +WebCopier +WEBDAV Web Enhancer -WebFetch WebFuck -Web Fuck WebGo IS +WebImageCollector WebLeacher WebmasterWorldForumBot -Web Pix +WebPix WebReaper WebSauger -Web Sucker -Whack +WebsiteExtractor +WebsiteQuester +Webster +WebSucker +WebZIP +Whacker Widow -WinHTTrack WiseGuys Robot +Wonderbot Woobot +Wotbox WPScan WWW-Collector-E -WWW-Mechanize x09Mozilla -x22Mozilla +Xaldon_WebSpider +Xenu +xpymep1.exe Zade -Zauba -Zermelo +zauba.io +Zeus Zitebot ZmEu diff --git a/.dev-tools/test_units/random-referrers-for-test-quick.list b/.dev-tools/test_units/random-referrers-for-test-quick.list index dca9731b3..08b44b088 100644 --- a/.dev-tools/test_units/random-referrers-for-test-quick.list +++ b/.dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -angigreene.com -animal36.com -gaz-voshod.ru -gorabagrata.ru -kinopolet.net -net-profits.xyz -ru-dety.ru -stackthatbucks.com -ublaze.ru -vkonche.com +citetick.com +distver.ru +elektrozigarette2.wordpressy.pl +fillmewithhappiness.com +freakycheats.com +lotto6888.com +mosrif.ru +safe-app.net +s-iwantyou.com +toon-families.com diff --git a/.dev-tools/test_units/random-referrers-for-test.list b/.dev-tools/test_units/random-referrers-for-test.list index ff43645b9..91782ef3d 100644 --- a/.dev-tools/test_units/random-referrers-for-test.list +++ b/.dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -0lovespells0.blogspot.com -100searchengines.com -101raccoon.ru +000free.us +101flag.ru +11235813.webzdarma.cz 123kuma.com -12bet.com -14b.info -1bet.com -2drittel.de -3dracergames.com -3w24.com -45en.ru -4webmasters.com -55wmz.ru -a14download.com -a1security.com.ua -abiente.ru -academia-nsk.org -acheterviagrafr24.com -adelly.bg -adinterax.com +1kdailyprofit.me +1millionusd.xyz +273-fz.ru +2pxg8bcf.top +4kepics.com +4kpics.rocks +8kisses.eu +8lufu.com +aasoldes.fr +accessoires-mode.in +activecampaign.dreamhosters.com +addbin.men +addblueoff.com.ua +adidas.frwebs.fr +adsloads.com +adsref.men ads-seo.men -agreda.pluto.ro -alfapro.ru -alienwheels.de -allblogroll.com -alldubai.biz -allmarketsnewdayli.gdn -aloofly.com -alpinism.ru -amateurgalls.com -amazingpic.net -anal-acrobats.com -anapa-inns.ru -anlimebel.kiev.ua -apiallgeniusinfo-a.akamaihd.net -apiportalmorecom-a.akamaihd.net -apishops.ru -appsaurus.com -aramaicmedia.org -aruplighting.com -asiengirls.net -asscenihotosidea.blogspot.co.za +adtech.us +adultgalls.com +adultmeetonline.info +adultnet.in +advancedmassagebysara.com +advokateg.xyz +ad-words.ru +aerogo.com +agahinameh.com +agapovdg.ru +ahmedabadwebs.com +air-edem.ru +akusherok.ru +alibestsale.com +allnews24.in +allpdfmags.net +all-streaming-media.com +alphahoverboards.com +amateurlivechat.org +amateurmatch.com +ampower.me +animal-drawings.com +aosheng-tech.com +aquarium-pro.ru +arabsexxxtube.com +arendatora.ru +asacopaco.tk astana.xxxkaz.org -automatic-seo.com -avek.ru +atlasvkusov.ru +atley.eu.pn +auto4style.ru +autotrends.today +auto-zapchasti.org +avitocars.ru +avon-novosib.ru +avtoarenda.by avtochehli.by -avtointeres.ru -azazu.ru -azbaseimages.net -balkanfarma.org -bamo.xsl.pt -beachpics.com -beauby.ru -best-mam.ru -bestpriceninja.com -betterscooter.com -beyan.host.sk -bioca.org -blagovest-med.ru -blog-hits.com -blog.koreadaily.com -bloke.com -bookmark4you.com.biz -boole.onlinewebshop.net -break-the-chains.com -breget74.ru +backgroundpictures.net +bala.getenjoyment.net +banan.tv +bannerads.de +belinka.com.ua +benchmarkcommunications.co.uk +berlininsl.com +bestchoice.cf +bestempresas.es +bestsexyblog.com +beta.hotkeys.com +bezlimitko.xyz +bezsporno.ru +big-boards.info +big-cash.net +biznesluxe.ru +bluerobot.info +bmwhighperformers.com +boeuklad.com +bosman.pluto.ro briomotor.co -bumascloset.com -burn-fat.ga -buron.pw -business-made-fun.com -buypuppies.ca -buzzonclick.com +bristolhostel.com +bryansk.zrus.org +buyclomidonlaine.com +buyk.host.sk bvlgariring.xyz -cacheimages.com -callstevens.com -candytiner.com -capture-room.com -cardsdumps.com -cartiertrinity.xyz -casablancamanor.co.za -cejewelry.xyz -centraletermice.us -centrdebut.ru -centre-indigo.org.ua -ceotrk.com -cfsrating.sonicwall.com -cgi2.nintendo.co.jp -cheapmoncler.win -chiblackhawks-jerseys.com -chidporn.com -chimiver.info +callcafe.info +carsplate.com +cartierlovebraceletreplica.xyz +cartierloveringreplica.xyz +casinorewards.info +caveavins.fr +cbox.ws +cellfun.mobi +cementaresearch.se +cheapjerseysbizwholesale.us +cheapwebsitehoster.com +chelyabinsk.xrus.org +children-learningreading.info chocoslim.pro -christianlouboutinoutlet.win -cicaki.net -ciekawinki.pl -citysecurity.nu -cjs.com.ru -cleanallvirus.com -clickbank.net -clicksor.net -clips.ua.ac.be -club-lukojl.ru +click2pawn.com +clipartnew.com +clksite.com +clodo.ru +companies-catalog.com compliance-brian.top -conciergegroup.org -coslab.club -cowblog.fr -csgo4.win -culturevie.info -custom-electric-guitar.com +compliance-fred.xyz +comsysnet.com +concealthyself.com +confib.ifmo.ru +cosmetic.donna7753191.ru +cosmeticswomens-womensports.rhcloud.com +creams.makeforum.eu +creativehutindia.com +customcollegeessays.net +custom-product-labels.com +cxpromote.com cyse.tk -dailyfinancefix.com -data-mining.tk -dating-time-now.com -dealwifi.com -designdevise.com -dev-seo.blog -digitalfaq.com -dignitasdata.se -dinkolove.ya.ru -djekxa.ru -dlya-android.org -docsportal.net -documentbase.net -dosugrostov.site -dotnetdotcom.org -downloads-whatsapp.com -doyouknowtheword-flummox.ml -dyt.net -ecookna.com.ua -editmedios.com -ekspertmed.com -ekto.ee -electrik-avenue.com -elektronischezigarettekaufen1.myblog.de -elektronischezi.livejournal.com -elvenmachine.com +dandiyabeats.in +deanmoore.ie +decoratinghomes.ga +delta-line.men +den-noch24.ru +design-ideas.info +detalizaciya-tut.biz +detki-opt.ru +dev.citetick.com +devochki.top +de.zapmeta.com +diesel-parts28.ru +dieta-personalna.pl +divatraffic.com +divisioncore.com +dmmspy.com +documentsite.net +dogoimage.com +domaincheck.io +domain-tracker.com +domination.ml +domoysshop.ru +doska-vsem.ru +downloadeer.net +downloadkakaotalk.com +download-of-the-warez.blogspot.com +dpihatinh.gov.vn +dstroy.su +dvd-famille.com +dverimegapolis.ru +e2click.com +easytuningshop.ru +e-biznes.info +e-commerce-seo.com +educontest.net +elektrozigaretten1.postbit.com +elmacho.xyz +empis.magix.net engines-usa.com -escort-russian.com -eshop.md -esnm.ru -executehosting.com -ezigarettenkaufen1.deviantart.com -fermersovet.ru +enternet.ee +eorogo.top +eredijovon.com +ereko.ru +erotikchat-24.com +etur.ru +everflora.ru +exonline.info +expresstoplivo.ru +extrabot.com +fakehandbags.xyz +familyphysician.ru +fashion.stellaconstance.co +femdom.twiclub.in ferrotodo.com -filesdatabase.net -fitfloponline.store -flavors.me -f-loaded.de -floating-share-buttons.com -florida-tourism.net -forums.toucharcade.com -foto-sisek.porngalleries.top -fototravel.eu -frcls.fr -freelifetimefuckbook.com -fullgirl.ru -fun-mobi.pl -fym.com.ru -g33.org -gagrasector.ru -gallerily.com -gallerylisting.com -gamebackyard.com -gamedayhouse.com -gameprimary.com -gateway.zscalerone.net -gdebestkupit.ru -gearcraft.us -getlamborghini.ga -gettpromos.com -ggiaro.com -girlsatgames.ru -glcomputers.ru -glicol.kz -globalscam.ga -gnuetella.com -golfresa.lucania.se -gorabagrata.ru +fickenprivat.info +filerockstar298.weebly.com +financepoints.eu +finemanteam.com +fivedwld.cf +floridahuntingfishingadventures.com +fodelsedagspresenter.nu +forex.osobye.ru +freesitetest.com +freetangodownload.com +freewhatsappload.com +freshwallpapers.info +fuckbuddybestgilf.info +fuckmill.com +fungamelands.com +fungirlsgames.net +gabeshop.ru +gamedayassist.com +games.kolossale.ru +gamingspark.com +gaz-voshod.ru +ge0ip.net +generalporn.org +gilsonchiro.xyz +girlspicsa.com +given2.com +globetrotting-culture.ru +gloverid.site +go2jump.org +goodhousekeeping.com +gopixdatabase.com +graphics8.info +greatdealshop.com +gribkovye-zabolevaniya.com gwhwpxbw.bloger.index.hr -hanwei.us +gz2.bbsoldes.fr +hackers-crackers.tk +hd-film.pl hdhc.ru -healgastro.com -healmytrauma.info -hebr.myddns-flir.com -helvetia.com.ua -hifidesign.ru -holidaypics.org -homeart.space -homedecoguide.info -honyaku.yahoofs.jp +healbio.ru +helpmymacfaster.trade +hem.passagen.se +histats.com +hitcpm.com +hitsbox.info +hmywwogw.bloger.index.hr +homelygarden.com +homemature.net hop.clickbank.net -hopeonthestreet.co.uk -hotxnights.info -housemilan.ru -houseofrose.com -hut1.ru -hyhj.info -i4track.net -idealtits.net -ifmo.ru -ilte.info -img.wallpaperstock.net -iminent.com -imk.com.ua -impisr.ru -infektsii.com -intellektmedia.at -invest-pamm.ru -iqbazar.ru -itsdp3.com -jackwolfskinoutlet.online -japfm.com -jimmychoosale.online -juliadiets.com +horoshieokna.com +hotelcrocenzi.sm +howtoclean.club +https-legalrc.biz +humanelydrew.com +humbmosquina.tk +hundejo.com +idegenvezeto.eu +immobiliaremassaro.com +infogame.name +infospot.pt +insta-add.pro +internet-apteka.ru +iomoio.net +ip-guide.com +irkutsk.zrus.org +istanbulit.com +istizanidineacontrolledsubstance.blogspot.com +izoll.ru +jav-fetish.site +jjbabskoe.ru jus0wil.pp.ua -justucalling32211123456789.tk kaidalibor.de -kashubadesign.ru -keenoutlet.online -keki.info -keyword-suggestions.com -kiinomaniak.pl -kino-fun.ru -kinohall.ru -kinopolet.net -klikbonus.com -konkursov.net -koronirealestate.gr -kosmetyki.tm.pl -kovesszucs.atw.hu -krasivoe-hd.net -krasnodar-avtolombards.ru -krasnodar.ru -kreativperlen.ch -kumuk.info -laexotic.com -lafriore.ru -languagecode.com -larger.io -law-check-nine.xyz -lawyers.cafe -leadwayau.com -lesbian.xyz -lietaer.com -light.ifmo.ru -linkarena.com -linkbux.ru -linkredirect.biz -liran-locks.com -lirunet.ru -loftdigital.eu -lombardfinder.ru -lonerangergames.com -lostfilm-online.ru -ltvperf.com -machicon-akihabara.info -macotool.com -mactechinfo.info -madot.onlinewebshop.net -makemoneyonline.com -manipulyator-peterburg.ru -map028.com -marcogrup.com +kambasoft.com +kam-dom.ru +katadhin.co +kazan.zrus.org +keywordhouse.com +kino-filmi.com +knogg.net +knowsitall.info +kosova.de +kuzinsp.ru +lakomka.com.ua +laulini.soclog.se +lechenie-gemorroya.com +li-er.ru +linksharingt.com +lion.snapmap.us +liveinternet.ru +longlifelomilomi.info +lsp-awak-perikanan.com +lyrics.home-task.com +m0r0zk0-krava.ru +mackeeper-land-672695126.us-east-1.elb.amazonaws.com +mail.allnews24.in +makedo.ru +mall.uk +manifestation.betteroffers.review marcoislandvacations.net -marketingvici.com -marmotstore.online -matematikus.info -matsdale.com -mcar.in.ua -mebel-alait.ru -medicinacom.ru -medispainstitute.com.au -mega-bony2017.pl -metabar.ru -microsearch.ru -misandesign.se +masturbate.co.uk +max-p.men +medanestesia.ru +medosmotr-ufa.ru +meetlocalchicks.com +menetie.ru +migente.com +mir-business-24.ru missvietnam.org -mnogabukaff.net -moneyteam24.com -moscow-clining.ru -moviezbonkerssk.cf -mowser.com -mrvideospornogratis.xxx -muzaporn.com -my-bc.ru +mnogolok.info +mojowhois.com +monclervests.xyz +money-for-placing-articles.com +mouselink.co +moxo.com +mozello.ru +musezone.su +mwtpludn.review +my-cash-bot.co +mydeathspace.com myftpupload.com -mylesosibirsk.ru -myplaycity.com -nac-bearings.ru +myhitmp3.club +mykings.pw +myonigroup.com +najaden.no naperehresti.info -neks.info -netanalytics.xyz -netfacet.net -netoil.no +naphukete.ru +naturalpharm.com.ua new7ob.com -news-speaker.com -nhl09.ru -nofreezingmac.work -northfacestore.online -notasprensa.info -nvssf.com -nzfilecloud.weebly.com -oballergiya.ru -ohmyrings.com -okout.ru +new-post.tk +newsperuse.com +newyorkhotelsmotels.info +nicefloor.co.uk +nightvision746.weebly.com +nikitsyringedrivelg.pen.io +njpalletremoval.com +november-lax.com +novosibirsk.xrus.org +nrjmobile.fr +nuit-artisanale.com +nuker.com +offer.wpsecurity.website okroshki.ru -online247.ml -onload.pw -o-o-8-o-o.com -optikremont.ru -oren-cats.ru -osagonline.ru -otbelivanie-zubov.com -outshop.ru -papasdelivery.ru -paramountmarble.co.uk -partner-pop.men -pc-services.ru -pc-test.net -phobia.us -photochki.com -phpdevops.com -php-market.ru -picture-group.com +ok.ru +ok-ua.info +onetravelguides.com +online-templatestore.com +ontargetseo.us +o-o-6-o-o.ru +openlibrary.org +openmultipleurl.com +open-odyssey.org +oranga.host.sk +orel-reshka.net +orgasmus-virtual.com +orion-v.com +ostroike.org +ownshop.win +pagesense.com +paleohub.info +partners-ship.pro +pb-dv.ru +pcboa.se +pepperstyle.ru +perosan.com +pflexads.com +phormchina.com +picsfair.com +picsforkeywordsuggestion.com +picturesfrom.com +picturesmania.com +pila.pl +pillscheap24h.com pintattoos.com -piratecams.com -pixelrz.com -pixgood.com -pizza-tycoon.com -pochtovyi-index.ru -pons-presse.com -popunder.net -pornoforadult.com -pornoklad.ru -porno-play.net -pornosee.info -porno-video-chati.ru -pornzone.tv -pr0fit-b0x.com -pregnant.guru -pricheskaonline.ru -primedice.com -privhosting.com -prostitutki-astana.org -putitin.me +plastgranar.nu +plastjulgranar.se +pobeiranie.pl +pornoklad.net +portal-eu.ru +portside.xyz +pravoholding.ru +preparevideosafesystem4unow.site +pricheski-video.com +prodess.ru +professionaldieselcare.com +proftests.net +promover.org +propranolol40mg.blogspot.com +psa48.ru +psvita.ru +push-ad.com +pussyspace.net pxhdwsm.com -qpypcx.com -quickcashlimited.com -quit-smoking.ga -rankia.com -rapidgator-porn.ga -realresultslist.com -real-time-analytics.com -rebelmouse.com -reelheroes.net -referencemoi.com -refudiatethissarah.info -remont-comp-pomosh.ru -renecaovillasale.online -renewablewealth.com -rentalcarnavi.info -revistaindustria.com -rewardpoll.com -ring4rhino.com -rootandroid.org -runtnc.net -saletool.ru -sanyuprojects.com -scanmyphones.com -scanner-olga.top -scanner-willy.top -sdrescher.net -se.bnt-team.com -seksotur.ru -semaltmedia.com -service.adtech.fr -service.adtech.us -service-core.ru -serw.clicksor.com -sex-dating.co -sexflirtbook.com +questionmarque.ch +ranksonic.org +rapidsites.pro +remontbiz.ru +remont-fridge-tv.ru +rethinkwasteni.info +rewardit.com +ruclicks.com +rukino.org +ruscopybook.com +rvi.biz +rybalka-opt.ru +sad-torg.com.ua +sampleletters.net +sbt-aqua.ru +scanner-george.top +scansafe.net +searchimage.co +sel-hoz.com +seocdvig.ru +seocheki.net +seomarketings.online +seo-platform.com +sexstream.pl +sex-tracker.com sextracker.de -sex-watch.com -shijian.ac.cn -shillyourcoins.com -sibdevice.ru +sexvideo-sex.com +shakhtar-doneck.ru +shell-pmr.ru +shemalegalls.blogporn.in +shlyahten.ru sibecoprom.ru -sientalyric.co -signx.info +siha.de +sildenafilcitratemed.com +sildenafil-tadalafil.info +simplepooltips.com +sims-sims.ru +sindragosa.comxa.com sinel.info -sinestesia.host.sk -sisiynas.ru -siteaero.com -siteexpress.co.il -sitopreferito.it -skylta.com -sky-mine.ru -slavia.info -slujbauborki.ru -smallseotools.com -sms2x2.ru -snjatie-geroinovoy-lomki.ru -socialmediasuggest.com -softxaker.ru -sonata-arctica.wz.cz -special-porn.com -sports-supplements.us -sprttrack.com -ssconstruction.co -stairliftstrue.com -startufa.ru -statistici.ro -statustroll.com -steamoff.net -stickers-market.ru -stjamesschool.info -strangeduckfilms.com -strigkaomsk.ru -studiofmp.com -superoboi.com.ua -sygraem.com -syndicate.fun -tagil.zrus.org -taqywu51.soup.io -taximytishi.ru -thaismartloan.com -thebestphotos.eu -thecounter.com -the-usa-games.blogspot.com -thin.me.pn -tizanidinerestlesslegsyndrome.blogspot.com -tizanidinesleepaid.blogspot.com -tizanidinestreetvalue.blogspot.com -tizanidinevsvalium.blogspot.com -tmearegion26.com -tobeyouday.win +sitevaluation.com +sitevalued.com +skuteczna-dieta.co.pl +slimcdn.com +slutloadlive.com +smadihome.com +snegozaderzhatel.ru +snow.nvr163.com +snts.shell-pmr.ru +socialmadesimple.com +socialmonkee.com +sortthemesitesby.com +spb.afora.ru +sport-video-obzor.ru +srecorder.com +stairliftsarea.com +stillmiracle.com +strag-invest.ru +streetfire.net +stretchmate.net +superstarfloraluk.com +sweepstakes.rewardit.com +tam-gde-more.ru +teesdaleflyballclub.co.uk +template-kid.com +tengohydar.tk +thaisamkok.com +thefotosgratis.eu +thegolfclub.info +theporndude.com +timhost.ru +tmtrck.com token-lab.org -tongkatmadura.info -toolsky.com -tooplay.com -torrentgamer.net -trackzapper.com -traffique.net -traxdom.ru -truebeauty.cc -tvteleport.ru -uamtrk.com -ukrobstep.com +traderzplanet.in +trahvid.com +tuberkulezanet.ru +tubo360.com +turbabitload.weebly.com +tvnewsclips.info +twincitiescarservice.com +ucsol.ru +uggsale.online +uginekologa.com +uhod-za-sobakoj.ru ultimateclassicrock.com -uni.me -universals.com.ua -us-america.ru -usbggettwku.ga -utiblog.fr -uytmaster.ru -via-energy-acquistare.com -videochat.mx -villacoloniale.com -vitanail.ru -viteonlusarezzo.it -vksaver-all.ru -vut.com.ru -vzlom-na-zakaz.com -webinstantservice.com -webix.biz -webjam.com -webpromotion.ae +underthesite.com +unece.org +uptimebot.net +urldelivery.com +utrolive.ru +valmetrundan.se +venta-prom.ru +ventelnos.com +verymes.xyz +viagengrarx.com +video-chat.in +vip-file.com +vitalads.net +vmnmvzsmn.over-blog.com +voloslove.ru +vtc.pw +vykup-avto-krasnodar.ru +wapsite.me +warezkeeper.com +waterpurifier.club +webscouter.net +wedding-salon.net welcomeauto.ru -weprik.ru -whengirlsgowild.com -whosonmyserver.com -williamrobsonproperty.com -wingsofrefuge.net -wnhjavlhezp.gq +westum.se +wetgames.ru +whatistizanidinehclusedfor.blogspot.com +winterclassichockeyjerseys.com +woman-tampon.ru wonderfulflowers.biz -wosik-dach.service-for-web.de -wpthemedetector.co.uk -ww2awards.info -x-diesel.info -xerox-douglas.cf +www.888.com +xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai +xn------7cdbapdecfd4ak1bn0amjffj7afu3y.xn--p1ai +xn----7sbaaabaei0cc8aj5bj0bncejx.xn--p1ai xn--80ahdheogk5l.xn--p1ai -xn--h1ahbi.com.ua -xn----itbeirbjbi7bc6bh2d.xn--p1ai -xn--lifehacer-1rb.com -xn--oogle-wmc.com -xxxtube69.com -yadro.ru +xolodremont.ru +x-porno.video +x-stars.ru +xxxrus.org +yeartwit.com yebocasino.com +yellowstonevisitortours.com +yginekologa.com +yopoint.in +yottos.com youandcredit.ru -youdao.com -youghbould.wordpress.com -youjizz.vc -youresponsive.com -zakonobosago.ru -zambini.ru -zeroredirect11.com -zeroredirect6.com -zlatnajesen.com -zoogdisany.com +yougetsignal.com +youjizz.com +yoursearch.me +yun56.co +zarabiaj-dzis.pl +zatjmuzu.info +zerocash.msk.ru +zog.link +zootoplist.com diff --git a/.dev-tools/test_units/random-referrers-for-whitelist-test.list b/.dev-tools/test_units/random-referrers-for-whitelist-test.list index b67976ae4..d55b52151 100644 --- a/.dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/.dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -101lesbian.xyz -11pikachu.ru -4kplayer.pl -51.la -6hopping.com -7a2rnuey1tw9ar.ru -8si.ru -aaronabel.com -adanih.com -adultnet.in -afora.ru -aliexpresscashback.club -aloofly.com -aludecor.info -amazon-seo-service.com -animaltoplist.com +01apple.com +178evakuator178.ru +1pamm.ru +4webmasters.org +8kisses.eu +8xv8.com +a1security.com.ua +aa81bf391151884adfa3dd677e41f94be1.com +abcdefh.xyz +abcguru.xyz +accessoires-mode.in +adultactioncam.com +adultmeetonline.info +advancedsoftwaresupport.com +adviceforum.info +advocatemsk.ru +advokateg.xyz +advokat-grodno.by +alarmobninsk.ru +alert.scansafe.net +alldezservice.kz +alldownload.pw +alt-servis.ru apartamentwroclaw.eu -apibatbrowsecom-a.akamaihd.net -apimossnetinfo-a.akamaihd.net -arenda-yeisk.ru -auspolice.com -aversis.net -axbocz.net -babyguns.ru -baersaratov.ru -bag77.ru -bankcrediti.ru -beauty-things.com -bestadbid.com -beta.hotkeys.com -bharatdefencekavach.com -bitcoinremote.com -bitnote.co -blackle.com -boc.kz -bombla.org -brillianty.info -bukleteg.xyz -buytizanidineonlinenoprescription.blogspot.com -callcafe.info -cardsdumps.com -cfcl.co.uk -cfsrating.sonicwall.com -cheapwebsitehoster.com -christianlouboutinreplica.win -chuckguilford.com -ci.ua -compliance-alex.xyz -compliance-elena.top -compliance-elena.xyz -compliance-jess.top -compliance-mark.top -cookielawblog.wordpress.com -cougfan.info -dame-ns.kz -datract.com -dayibiao.com -deplim.com -discovertreasure-a.akamaihd.net -dms-sw.ru -doublepimp.com +apartmentbay.ru +a.pr-cy.ru +artavenuegardenstudios.com +audiobangout.com +ayerbo.xhost.ro +bambi.ck.ua +begalka.xyz +bestcurs.org +bikini-image.com +buron.pw +buyessay3.blogspot.ru +carloans.com +carmuffler.net +cartierlovestore.com +chatseo.com +cheapjerseysa.com +cheapmoncler.xyz +checkm8.com +chee-by.biz +chiptuninger.com +cineacademy.ru +club-samodelkin.ru +cocyq.inwtrade.com +compliance-julia.top +corta.co +cubs-jerseys.us +czeshop.info +day-news.info +directivepub.com +dktr.ru +donvito.unas.cz downloaddy.net -dverimegapolis.ru -earncash.com.ua -earthmagic.info -easync.io -ecommerce-seo.com -ednorxmed.com -eduinfosite.com -eladkarako.github.io -elektrischezigaretten1.blogse.nl -el-nation.com -elstal.com.pl -enge-fotzen.info -ereko.ru -eyemagination.com -ezaz.info -fitfloponline.store -fivedwld.cf -floridamobilebillboards.com -flowersbazar.com -forex-procto.ru -forodvd.com -forsex.info -geilehausfrauen.net -gemara.com -germetiki.com.ua -glasof.es -goldenggames.com -googst2.ru -guge.io -hccoder.info -hol.es -hoste.octopis.com -hulfingtonpost.com -ibb.com.ua -igrovyeavtomaty777.ru -imagine-ex.co -i-midias.net.br -inboxdollars.com -insurple.com -internet-apteka.ru -izhevsk.xrus.org -james13prix.info -javlibrary.cc -kaidalibor.de +driving.kiev.ua +elektrischeziga.livejournal.com +enhand.se +eugenevaultstorage.com +ewebarticle.info +expertnaya-ocenka.ru +ezb.elvenmachine.com +ezigarettenkaufen1.deviantart.com +facecup.top +falopicm.pw +fast-wordpress-start.com +financehint.eu +financepoints.eu +flashlarevista.com +floating-share-buttons.com +footballfarrago.com +forum-engineering.ru +foxjuegos.com +frcls.fr +freejabs.com +fuel-gas.com +fx-brokers-review.com +gamedayhouse.com +gameplexcity.com +gateway.zscalerone.net +gazobeton-p.com.ua +gerhardhealer.com +gettpromos.com +ghazel.ru +glall.ru +globetrotting-culture.ru +gloverid.site +good-mummy.ru +goodprotein.ru +gopixdatabase.com +gsasearchengineranker.site +handicapbathtubarea.com +happy.new.yeartwit.com +hard-porn.mobi +homik.pw +htmlcorner.com +hunterboots.online +ikearugs.xyz +imadedinner.net +infokonkurs.ru +internetproviderstucson.com +investzalog.ru +iomoio.net +islandminingsupply.wordpress.com +jav-fetish.site +jaxcube.info +j-times.ru +juliadiets.com kamen-e.ru -kemerovo.zrus.org -keywordglobal.co.za -kichenaid.ru -kino2018.club -kinofak.net -koleso24.com.ua -koversite.info -kozhniebolezni.com -kuzinsp.ru -languagecode.com -lesbian.xyz -life.biz.ua -liffa.ru -liverpool.gsofootball.com -mackeeper-land-672695126.us-east-1.elb.amazonaws.com -mahnwachen-helfen.info -maslenka.kz -mastershef.club -meds-online24.com -mere.host.sk -mexicosleevegastrectomy.com -millionare.com -minneapoliscopiers.com -mir-limuzinov.ru -mobplayer.net -moscow-region.ru -mostantikor.ru -muizre.ru -mycaf.it -mydirtyhobby.com -mydirtystuff.com -myprintscreen.com -naphukete.ru -nash-krym.info -nationalbreakdown.com +kareliatobacco.ru +keki.info +kenaba.su +ketrzyn.pl +keywordteam.net +kinoplen.ru +konpax.com +koptims.tiu.ru +kosynka-games.ru +kw21.org +la-fa.ru +lampokrat.ws +leboard.ru +levaquin750mg.blogspot.com +loadopia.com +macotool.com +magda-gadalka.ru +magicdiet.gq +mailemedicinals.com +malls.com +marblestyle.ru +maxxtor.eu +medicines-choice.com +medtherapy.ru +merryhouse.co.uk +middlerush-a.akamaihd.net +minecraft-rus.org +mnogolok.info +monetizer.com-01.site +morlat.altervista.org +mundoaberrante.com +musicvidz.ru +mykings.pw +nakozhe.com +naturtreenspicerx.pw netfacet.net -next-dentists.tk -ngps1.ru -njkmznnb.ru -nofreezingmac.click -novatech.vn -nyfinance.ml -oconto.ru -ohmyrings.com -oil-td.ru -ossmalta.com -paphoselectricianandplumber.com -pathwhelp.org -perl.dp.ua +nikitsyringedrivelg.pen.io +njpalletremoval.com +notebook-pro.ru +nullrefer.com +odesproperty.com +omoikiri-japan.ru +omsk.xrus.org +onlinemegax.com +opinionreelle.com +outpersonals.com +paparazzistudios.com.au +parfusale.se +partnerworkroom.men petitions.whitehouse.gov -pharmacyincity.com -picturesmania.com -pistonclasico.com -pokerniydom.ru -polybuild.ru -pornolook.net -powitania.pl -printingpeach.com -profitkode.com -profit-opportunity.com -prokotov.com -promalp-universal.ru -py100.ru -rabotaetvse.ru -rada.ru -rank-checker.online -real-time-analytics.com -remontbiz.ru -riders.ro -rimedia.org -rniaeba.ga -royal-investments.net -sabaapress.com -sad-torg.com.ua -scanner-alexa.top -scanner-nelson.top -seoholding.com -seomarketings.online -sexcamamateurchat.com +pictures-and-images.net +platesauto.com +pogosh.com +pogruztehnik.ru +powenlite24.ru +pozdravleniya-c.ru +pozdrawleniya.ru +praisong.net +pulseonclick.com +randalljhoward.com +relax.ru +remorcicomerciale.ro +rezeptiblud.ru +richinvestmonitor.com +ring4rhino.com +rutor.vip +sabizonline.com +sanatorrii.ru +saugatuck.com +sbt-aqua.ru +scanner-willy.top +scmor.ilxc.cc +selectads.men +seofied.com +seorankinglinks.xyz +seo-tools-optimizing.com +sergiorossistore.online sex-pr.net -sexsearch.com -shariki-zuma-lines.ru -sim-service.net -sitevaluation.com -slonechka.ru -snsdeainavi.info -socialmediasuggest.com -soft-terminal.ru -softtor.com -solnplast.ru -stardevine.com -starwars.wikia.com -stroydetali.ru -taaaak.com -tedxrj.com -thebluffs.com -thetoiletpaper.com -tizanidine4mguses.blogspot.com -tizanidineforopiatewithdrawal.blogspot.com -tizanidinehydrochloride4mgstreetvalue.blogspot.com -tizanidineinfo.blogspot.com -tizanidinemusclerelaxant.blogspot.com -tizanidineusedfor.blogspot.com -tizanidinevsvalium.blogspot.com -toondinsey.com -toon-families.com -torontoplumbinggroup.com -torrent-to-magnet.com -trackmedia101.com -tracksz.co -trafficnetzwerk.de +sharpchallenge.com +shivafurnishings.com +shodanhq.com +shymkent.xkaz.org +sildenafilcitratemed.com +slimcdn.com +slowmac.tech +sluganarodu.ru +smokewithrabbits.com +snegozaderzhatel.ru +social-buttons.xyz +sosdepotdebilan.com +souvenir.cc +spamnuker.com +spb-plitka.ru +spidtest.space +sportbetfair.com +spywarebegone.com +sribno.net +stpolice.com +studiofmp.com +sugarkun.com +superlist.biz +tam-gde-more.ru +tattooha.com +telefonsex-ohne0900.net +thecoral.com.br +thewomenlife.com +tizanidinedrugtest.blogspot.com +tongkatmadura.info +topcar-krasnodar.ru +track112.site +tradedeals.biz +traffic-club.info +traffictrade.life try-rx.com -turizm.bz -twincitiescarservice.com -umekana.ru -unmaroll.ya.ru -uptime-delta.net -veles.shop -videochaty.ru -videogamesecrets.com -visa-china.ru -vzlom-na-zakaz.com -whereverdesperate.gq -wurr.voila.net -x-diesel.biz -xjlottery.com -xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai -xn--80ajjbdhgmudixfjc8c5a9df8b.xn--p1ai -xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks -xnxx-n.com -xportvusbdriver8i.snack.ws -x-rates.ru -xtrafficplus.com -xvideosporn.biz -yellocloud.be -yes-com.com -yougame.biz -yurgorod.ru -zagadki.in.ua -zakon-ob-obrazovanii.ru -zarenica.net -zigzog.ru -zrizvtrnpale.tk +tsstcorpcddvdwshbbdriverfb.aircus.com +tsyndicate.com +tuberkulezanet.ru +ukkelberg.no +ultimatesetnewfreeallsoftupgradesystems.pw +unpredictable.ga +urccvfmc.bloger.index.hr +uytmaster.ru +vacances-voyages.info +valoresito.com +venta-prom.ru +via-energy-cumpara.com +vk-mus.ru +wallpaperstock.net +wechatdownload10.com +wholesalecheapjerseysfree.com +w-journal.ru +wosik-dach.service-for-web.de +xev.ru +xn----7sbaphztdjeboffeiof6c.xn--p1ai +xn--80aodinpgi.xn--p1ai +xxxrus.org +xxx-treker.ru +yellowads.men +zarabotok--doma.ru +zeroredirect11.com diff --git a/.dev-tools/test_units/ratelimittest-beta.txt b/.dev-tools/test_units/ratelimittest-beta.txt index a7215fb74..f371d9e5f 100644 --- a/.dev-tools/test_units/ratelimittest-beta.txt +++ b/.dev-tools/test_units/ratelimittest-beta.txt @@ -1,11 +1,21 @@ - + + +
+This is merely used for testing !!!
+ +This is merely used for testing !!!
- - - -This is merely used for testing !!!
- \ No newline at end of file diff --git a/.dev-tools/test_units/ratelimittest-master.txt b/.dev-tools/test_units/ratelimittest-master.txt index aea69b849..f371d9e5f 100644 --- a/.dev-tools/test_units/ratelimittest-master.txt +++ b/.dev-tools/test_units/ratelimittest-master.txt @@ -126,14 +126,4 @@This is merely used for testing !!!
-