diff --git a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf index 60959195e..fd54c6e82 100644 --- a/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_test1/bots.d/whitelist-ips.conf @@ -33,4 +33,4 @@ # 111.111.111.111 0; -35.192.85.2 0; +35.193.7.13 0; diff --git a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf index 3a4498cec..95677d21d 100644 --- a/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_test1/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1463 -### Updated: Thu Jun 20 22:14:22 SAST 2019 +### Version: V3.2019.06.1465 +### Updated: Fri Jun 21 08:47:44 SAST 2019 ### Bad Referrer Count: 6693 ### Bad Bot Count: 557 ################################################### diff --git a/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf b/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf index 60959195e..fd54c6e82 100644 --- a/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf +++ b/.dev-tools/_conf_files_testing_changes/bots.d/whitelist-ips.conf @@ -33,4 +33,4 @@ # 111.111.111.111 0; -35.192.85.2 0; +35.193.7.13 0; diff --git a/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf b/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf index ab897a1ec..10506cfa0 100644 --- a/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf +++ b/.dev-tools/_conf_files_testing_changes/conf.d/globalblacklist.conf @@ -10,8 +10,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2019.06.1463 -### Updated: Thu Jun 20 22:14:23 SAST 2019 +### Version: V3.2019.06.1465 +### Updated: Fri Jun 21 08:47:44 SAST 2019 ### Bad Referrer Count: 6693 ### Bad Bot Count: 557 ################################################### @@ -130,6 +130,572 @@ map $http_user_agent $bad_bot { # END CUSTOM BLACKLISTED USER AGENTS ### DO NOT EDIT OR REMOVE THIS LINE AT ALL ### # ********************************************************************************* +# ********************************************* +# Bad User-Agent Strings That We Block Outright +# ********************************************* + +# This includes: +# Known Vulnerability Scanners (now merged into one section) + +# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### + "~*360Spider(?:\.|\b|\/)" 3; + "~*404checker(?:\.|\b|\/)" 3; + "~*404enemy(?:\.|\b|\/)" 3; + "~*80legs(?:\.|\b|\/)" 3; + "~*Abonti(?:\.|\b|\/)" 3; + "~*Aboundex(?:\.|\b|\/)" 3; + "~*Aboundexbot(?:\.|\b|\/)" 3; + "~*Acunetix(?:\.|\b|\/)" 3; + "~*ADmantX(?:\.|\b|\/)" 3; + "~*AfD-Verbotsverfahren(?:\.|\b|\/)" 3; + "~*AhrefsBot(?:\.|\b|\/)" 3; + "~*AIBOT(?:\.|\b|\/)" 3; + "~*AiHitBot(?:\.|\b|\/)" 3; + "~*Aipbot(?:\.|\b|\/)" 3; + "~*Alexibot(?:\.|\b|\/)" 3; + "~*Alligator(?:\.|\b|\/)" 3; + "~*AllSubmitter(?:\.|\b|\/)" 3; + "~*AlphaBot(?:\.|\b|\/)" 3; + "~*Anarchie(?:\.|\b|\/)" 3; + "~*Apexoo(?:\.|\b|\/)" 3; + "~*archive.org_bot(?:\.|\b|\/)" 3; + "~*ASPSeek(?:\.|\b|\/)" 3; + "~*Asterias(?:\.|\b|\/)" 3; + "~*Attach(?:\.|\b|\/)" 3; + "~*autoemailspider(?:\.|\b|\/)" 3; + "~*BackDoorBot(?:\.|\b|\/)" 3; + "~*Backlink-Ceck(?:\.|\b|\/)" 3; + "~*backlink-check(?:\.|\b|\/)" 3; + "~*BacklinkCrawler(?:\.|\b|\/)" 3; + "~*BackStreet(?:\.|\b|\/)" 3; + "~*BackWeb(?:\.|\b|\/)" 3; + "~*Badass(?:\.|\b|\/)" 3; + "~*Bandit(?:\.|\b|\/)" 3; + "~*Barkrowler(?:\.|\b|\/)" 3; + "~*BatchFTP(?:\.|\b|\/)" 3; + "~*Battleztar\ Bazinga(?:\.|\b|\/)" 3; + "~*BBBike(?:\.|\b|\/)" 3; + "~*BDCbot(?:\.|\b|\/)" 3; + "~*BDFetch(?:\.|\b|\/)" 3; + "~*BetaBot(?:\.|\b|\/)" 3; + "~*Bigfoot(?:\.|\b|\/)" 3; + "~*Bitacle(?:\.|\b|\/)" 3; + "~*Blackboard(?:\.|\b|\/)" 3; + "~*Black\ Hole(?:\.|\b|\/)" 3; + "~*BlackWidow(?:\.|\b|\/)" 3; + "~*BLEXBot(?:\.|\b|\/)" 3; + "~*Blow(?:\.|\b|\/)" 3; + "~*BlowFish(?:\.|\b|\/)" 3; + "~*Boardreader(?:\.|\b|\/)" 3; + "~*Bolt(?:\.|\b|\/)" 3; + "~*BotALot(?:\.|\b|\/)" 3; + "~*Brandprotect(?:\.|\b|\/)" 3; + "~*Brandwatch(?:\.|\b|\/)" 3; + "~*Bubing(?:\.|\b|\/)" 3; + "~*Buddy(?:\.|\b|\/)" 3; + "~*BuiltBotTough(?:\.|\b|\/)" 3; + "~*BuiltWith(?:\.|\b|\/)" 3; + "~*Bullseye(?:\.|\b|\/)" 3; + "~*BunnySlippers(?:\.|\b|\/)" 3; + "~*BuzzSumo(?:\.|\b|\/)" 3; + "~*Calculon(?:\.|\b|\/)" 3; + "~*CATExplorador(?:\.|\b|\/)" 3; + "~*CazoodleBot(?:\.|\b|\/)" 3; + "~*CCBot(?:\.|\b|\/)" 3; + "~*Cegbfeieh(?:\.|\b|\/)" 3; + "~*CheeseBot(?:\.|\b|\/)" 3; + "~*CherryPicker(?:\.|\b|\/)" 3; + "~*CheTeam(?:\.|\b|\/)" 3; + "~*ChinaClaw(?:\.|\b|\/)" 3; + "~*Chlooe(?:\.|\b|\/)" 3; + "~*Claritybot(?:\.|\b|\/)" 3; + "~*Cliqzbot(?:\.|\b|\/)" 3; + "~*Cloud\ mapping(?:\.|\b|\/)" 3; + "~*coccocbot-web(?:\.|\b|\/)" 3; + "~*Cogentbot(?:\.|\b|\/)" 3; + "~*cognitiveseo(?:\.|\b|\/)" 3; + "~*Collector(?:\.|\b|\/)" 3; + "~*com.plumanalytics(?:\.|\b|\/)" 3; + "~*Copier(?:\.|\b|\/)" 3; + "~*CopyRightCheck(?:\.|\b|\/)" 3; + "~*Copyscape(?:\.|\b|\/)" 3; + "~*Cosmos(?:\.|\b|\/)" 3; + "~*Craftbot(?:\.|\b|\/)" 3; + "~*crawler4j(?:\.|\b|\/)" 3; + "~*crawler.feedback(?:\.|\b|\/)" 3; + "~*crawl.sogou.com(?:\.|\b|\/)" 3; + "~*CrazyWebCrawler(?:\.|\b|\/)" 3; + "~*Crescent(?:\.|\b|\/)" 3; + "~*CrunchBot(?:\.|\b|\/)" 3; + "~*CSHttp(?:\.|\b|\/)" 3; + "~*Curious(?:\.|\b|\/)" 3; + "~*Custo(?:\.|\b|\/)" 3; + "~*DatabaseDriverMysqli(?:\.|\b|\/)" 3; + "~*DataCha0s(?:\.|\b|\/)" 3; + "~*DBLBot(?:\.|\b|\/)" 3; + "~*demandbase-bot(?:\.|\b|\/)" 3; + "~*Demon(?:\.|\b|\/)" 3; + "~*Deusu(?:\.|\b|\/)" 3; + "~*Devil(?:\.|\b|\/)" 3; + "~*Digincore(?:\.|\b|\/)" 3; + "~*DigitalPebble(?:\.|\b|\/)" 3; + "~*DIIbot(?:\.|\b|\/)" 3; + "~*Dirbuster(?:\.|\b|\/)" 3; + "~*Disco(?:\.|\b|\/)" 3; + "~*Discobot(?:\.|\b|\/)" 3; + "~*Discoverybot(?:\.|\b|\/)" 3; + "~*Dispatch(?:\.|\b|\/)" 3; + "~*DittoSpyder(?:\.|\b|\/)" 3; + "~*DnyzBot(?:\.|\b|\/)" 3; + "~*DomainAppender(?:\.|\b|\/)" 3; + "~*DomainCrawler(?:\.|\b|\/)" 3; + "~*DomainSigmaCrawler(?:\.|\b|\/)" 3; + "~*DomainStatsBot(?:\.|\b|\/)" 3; + "~*Dotbot(?:\.|\b|\/)" 3; + "~*Download\ Wonder(?:\.|\b|\/)" 3; + "~*Dragonfly(?:\.|\b|\/)" 3; + "~*Drip(?:\.|\b|\/)" 3; + "~*DSearch(?:\.|\b|\/)" 3; + "~*DTS\ Agent(?:\.|\b|\/)" 3; + "~*EasyDL(?:\.|\b|\/)" 3; + "~*Ebingbong(?:\.|\b|\/)" 3; + "~*eCatch(?:\.|\b|\/)" 3; + "~*ECCP/1.0(?:\.|\b|\/)" 3; + "~*Ecxi(?:\.|\b|\/)" 3; + "~*EirGrabber(?:\.|\b|\/)" 3; + "~*EMail\ Siphon(?:\.|\b|\/)" 3; + "~*EMail\ Wolf(?:\.|\b|\/)" 3; + "~*EroCrawler(?:\.|\b|\/)" 3; + "~*evc-batch(?:\.|\b|\/)" 3; + "~*Evil(?:\.|\b|\/)" 3; + "~*Exabot(?:\.|\b|\/)" 3; + "~*Express\ WebPictures(?:\.|\b|\/)" 3; + "~*ExtLinksBot(?:\.|\b|\/)" 3; + "~*Extractor(?:\.|\b|\/)" 3; + "~*ExtractorPro(?:\.|\b|\/)" 3; + "~*Extreme\ Picture\ Finder(?:\.|\b|\/)" 3; + "~*EyeNetIE(?:\.|\b|\/)" 3; + "~*Ezooms(?:\.|\b|\/)" 3; + "~*facebookscraper(?:\.|\b|\/)" 3; + "~*FDM(?:\.|\b|\/)" 3; + "~*FemtosearchBot(?:\.|\b|\/)" 3; + "~*FHscan(?:\.|\b|\/)" 3; + "~*Fimap(?:\.|\b|\/)" 3; + "~*Firefox/7.0(?:\.|\b|\/)" 3; + "~*FlashGet(?:\.|\b|\/)" 3; + "~*Flunky(?:\.|\b|\/)" 3; + "~*Foobot(?:\.|\b|\/)" 3; + "~*Freeuploader(?:\.|\b|\/)" 3; + "~*FrontPage(?:\.|\b|\/)" 3; + "~*FyberSpider(?:\.|\b|\/)" 3; + "~*Fyrebot(?:\.|\b|\/)" 3; + "~*GalaxyBot(?:\.|\b|\/)" 3; + "~*Genieo(?:\.|\b|\/)" 3; + "~*GermCrawler(?:\.|\b|\/)" 3; + "~*Getintent(?:\.|\b|\/)" 3; + "~*GetRight(?:\.|\b|\/)" 3; + "~*GetWeb(?:\.|\b|\/)" 3; + "~*Gigablast(?:\.|\b|\/)" 3; + "~*Gigabot(?:\.|\b|\/)" 3; + "~*G-i-g-a-b-o-t(?:\.|\b|\/)" 3; + "~*Go-Ahead-Got-It(?:\.|\b|\/)" 3; + "~*Gotit(?:\.|\b|\/)" 3; + "~*GoZilla(?:\.|\b|\/)" 3; + "~*Go!Zilla(?:\.|\b|\/)" 3; + "~*Grabber(?:\.|\b|\/)" 3; + "~*GrabNet(?:\.|\b|\/)" 3; + "~*Grafula(?:\.|\b|\/)" 3; + "~*GrapeFX(?:\.|\b|\/)" 3; + "~*GrapeshotCrawler(?:\.|\b|\/)" 3; + "~*GridBot(?:\.|\b|\/)" 3; + "~*GT::WWW(?:\.|\b|\/)" 3; + "~*Haansoft(?:\.|\b|\/)" 3; + "~*HaosouSpider(?:\.|\b|\/)" 3; + "~*Harvest(?:\.|\b|\/)" 3; + "~*Havij(?:\.|\b|\/)" 3; + "~*HEADMasterSEO(?:\.|\b|\/)" 3; + "~*Heritrix(?:\.|\b|\/)" 3; + "~*Hloader(?:\.|\b|\/)" 3; + "~*HMView(?:\.|\b|\/)" 3; + "~*HTMLparser(?:\.|\b|\/)" 3; + "~*HTTP::Lite(?:\.|\b|\/)" 3; + "~*HTTrack(?:\.|\b|\/)" 3; + "~*Humanlinks(?:\.|\b|\/)" 3; + "~*HybridBot(?:\.|\b|\/)" 3; + "~*Iblog(?:\.|\b|\/)" 3; + "~*IDBot(?:\.|\b|\/)" 3; + "~*Id-search(?:\.|\b|\/)" 3; + "~*IlseBot(?:\.|\b|\/)" 3; + "~*Image\ Fetch(?:\.|\b|\/)" 3; + "~*Image\ Sucker(?:\.|\b|\/)" 3; + "~*IndeedBot(?:\.|\b|\/)" 3; + "~*Indy\ Library(?:\.|\b|\/)" 3; + "~*InfoNaviRobot(?:\.|\b|\/)" 3; + "~*InfoTekies(?:\.|\b|\/)" 3; + "~*instabid(?:\.|\b|\/)" 3; + "~*Intelliseek(?:\.|\b|\/)" 3; + "~*InterGET(?:\.|\b|\/)" 3; + "~*Internet\ Ninja(?:\.|\b|\/)" 3; + "~*InternetSeer(?:\.|\b|\/)" 3; + "~*internetVista\ monitor(?:\.|\b|\/)" 3; + "~*ips-agent(?:\.|\b|\/)" 3; + "~*Iria(?:\.|\b|\/)" 3; + "~*IRLbot(?:\.|\b|\/)" 3; + "~*Iskanie(?:\.|\b|\/)" 3; + "~*IstellaBot(?:\.|\b|\/)" 3; + "~*JamesBOT(?:\.|\b|\/)" 3; + "~*Jbrofuzz(?:\.|\b|\/)" 3; + "~*JennyBot(?:\.|\b|\/)" 3; + "~*JetCar(?:\.|\b|\/)" 3; + "~*Jetty(?:\.|\b|\/)" 3; + "~*JikeSpider(?:\.|\b|\/)" 3; + "~*JOC\ Web\ Spider(?:\.|\b|\/)" 3; + "~*Joomla(?:\.|\b|\/)" 3; + "~*Jorgee(?:\.|\b|\/)" 3; + "~*JustView(?:\.|\b|\/)" 3; + "~*Jyxobot(?:\.|\b|\/)" 3; + "~*Kenjin\ Spider(?:\.|\b|\/)" 3; + "~*Keyword\ Density(?:\.|\b|\/)" 3; + "~*Kozmosbot(?:\.|\b|\/)" 3; + "~*Lanshanbot(?:\.|\b|\/)" 3; + "~*Larbin(?:\.|\b|\/)" 3; + "~*LeechFTP(?:\.|\b|\/)" 3; + "~*LeechGet(?:\.|\b|\/)" 3; + "~*LexiBot(?:\.|\b|\/)" 3; + "~*Lftp(?:\.|\b|\/)" 3; + "~*LibWeb(?:\.|\b|\/)" 3; + "~*Libwhisker(?:\.|\b|\/)" 3; + "~*Lightspeedsystems(?:\.|\b|\/)" 3; + "~*Likse(?:\.|\b|\/)" 3; + "~*Linkdexbot(?:\.|\b|\/)" 3; + "~*LinkextractorPro(?:\.|\b|\/)" 3; + "~*LinkpadBot(?:\.|\b|\/)" 3; + "~*LinkScan(?:\.|\b|\/)" 3; + "~*LinksManager(?:\.|\b|\/)" 3; + "~*LinkWalker(?:\.|\b|\/)" 3; + "~*LinqiaMetadataDownloaderBot(?:\.|\b|\/)" 3; + "~*LinqiaRSSBot(?:\.|\b|\/)" 3; + "~*LinqiaScrapeBot(?:\.|\b|\/)" 3; + "~*Lipperhey(?:\.|\b|\/)" 3; + "~*Lipperhey\ Spider(?:\.|\b|\/)" 3; + "~*Litemage_walker(?:\.|\b|\/)" 3; + "~*Lmspider(?:\.|\b|\/)" 3; + "~*LNSpiderguy(?:\.|\b|\/)" 3; + "~*Ltx71(?:\.|\b|\/)" 3; + "~*lwp-request(?:\.|\b|\/)" 3; + "~*LWP::Simple(?:\.|\b|\/)" 3; + "~*lwp-trivial(?:\.|\b|\/)" 3; + "~*Magnet(?:\.|\b|\/)" 3; + "~*Mag-Net(?:\.|\b|\/)" 3; + "~*magpie-crawler(?:\.|\b|\/)" 3; + "~*Mail.RU_Bot(?:\.|\b|\/)" 3; + "~*Majestic12(?:\.|\b|\/)" 3; + "~*Majestic-SEO(?:\.|\b|\/)" 3; + "~*Majestic\ SEO(?:\.|\b|\/)" 3; + "~*MarkMonitor(?:\.|\b|\/)" 3; + "~*MarkWatch(?:\.|\b|\/)" 3; + "~*Masscan(?:\.|\b|\/)" 3; + "~*Mass\ Downloader(?:\.|\b|\/)" 3; + "~*Mata\ Hari(?:\.|\b|\/)" 3; + "~*MauiBot(?:\.|\b|\/)" 3; + "~*meanpathbot(?:\.|\b|\/)" 3; + "~*Meanpathbot(?:\.|\b|\/)" 3; + "~*MeanPath\ Bot(?:\.|\b|\/)" 3; + "~*Mediatoolkitbot(?:\.|\b|\/)" 3; + "~*mediawords(?:\.|\b|\/)" 3; + "~*MegaIndex.ru(?:\.|\b|\/)" 3; + "~*Metauri(?:\.|\b|\/)" 3; + "~*MFC_Tear_Sample(?:\.|\b|\/)" 3; + "~*Microsoft\ Data\ Access(?:\.|\b|\/)" 3; + "~*Microsoft\ URL\ Control(?:\.|\b|\/)" 3; + "~*MIDown\ tool(?:\.|\b|\/)" 3; + "~*MIIxpc(?:\.|\b|\/)" 3; + "~*Mister\ PiX(?:\.|\b|\/)" 3; + "~*MJ12bot(?:\.|\b|\/)" 3; + "~*Mojeek(?:\.|\b|\/)" 3; + "~*Morfeus\ Fucking\ Scanner(?:\.|\b|\/)" 3; + "~*Mr.4x3(?:\.|\b|\/)" 3; + "~*MSFrontPage(?:\.|\b|\/)" 3; + "~*MSIECrawler(?:\.|\b|\/)" 3; + "~*Msrabot(?:\.|\b|\/)" 3; + "~*MS\ Web\ Services\ Client\ Protocol(?:\.|\b|\/)" 3; + "~*muhstik-scan(?:\.|\b|\/)" 3; + "~*Musobot(?:\.|\b|\/)" 3; + "~*Name\ Intelligence(?:\.|\b|\/)" 3; + "~*Nameprotect(?:\.|\b|\/)" 3; + "~*Navroad(?:\.|\b|\/)" 3; + "~*NearSite(?:\.|\b|\/)" 3; + "~*Needle(?:\.|\b|\/)" 3; + "~*Nessus(?:\.|\b|\/)" 3; + "~*NetAnts(?:\.|\b|\/)" 3; + "~*Netcraft(?:\.|\b|\/)" 3; + "~*netEstate\ NE\ Crawler(?:\.|\b|\/)" 3; + "~*NetLyzer(?:\.|\b|\/)" 3; + "~*NetMechanic(?:\.|\b|\/)" 3; + "~*NetSpider(?:\.|\b|\/)" 3; + "~*Nettrack(?:\.|\b|\/)" 3; + "~*Net\ Vampire(?:\.|\b|\/)" 3; + "~*Netvibes(?:\.|\b|\/)" 3; + "~*NetZIP(?:\.|\b|\/)" 3; + "~*NextGenSearchBot(?:\.|\b|\/)" 3; + "~*Nibbler(?:\.|\b|\/)" 3; + "~*NICErsPRO(?:\.|\b|\/)" 3; + "~*Niki-bot(?:\.|\b|\/)" 3; + "~*Nikto(?:\.|\b|\/)" 3; + "~*NimbleCrawler(?:\.|\b|\/)" 3; + "~*Nimbostratus(?:\.|\b|\/)" 3; + "~*Ninja(?:\.|\b|\/)" 3; + "~*Nmap(?:\.|\b|\/)" 3; + "~*NPbot(?:\.|\b|\/)" 3; + "~*Nutch(?:\.|\b|\/)" 3; + "~*oBot(?:\.|\b|\/)" 3; + "~*Octopus(?:\.|\b|\/)" 3; + "~*Offline\ Explorer(?:\.|\b|\/)" 3; + "~*Offline\ Navigator(?:\.|\b|\/)" 3; + "~*OnCrawl(?:\.|\b|\/)" 3; + "~*Openfind(?:\.|\b|\/)" 3; + "~*OpenLinkProfiler(?:\.|\b|\/)" 3; + "~*Openvas(?:\.|\b|\/)" 3; + "~*OpenVAS(?:\.|\b|\/)" 3; + "~*OrangeBot(?:\.|\b|\/)" 3; + "~*OrangeSpider(?:\.|\b|\/)" 3; + "~*OutclicksBot(?:\.|\b|\/)" 3; + "~*OutfoxBot(?:\.|\b|\/)" 3; + "~*PageAnalyzer(?:\.|\b|\/)" 3; + "~*Page\ Analyzer(?:\.|\b|\/)" 3; + "~*PageGrabber(?:\.|\b|\/)" 3; + "~*page\ scorer(?:\.|\b|\/)" 3; + "~*PageScorer(?:\.|\b|\/)" 3; + "~*Pandalytics(?:\.|\b|\/)" 3; + "~*Panscient(?:\.|\b|\/)" 3; + "~*Papa\ Foto(?:\.|\b|\/)" 3; + "~*Pavuk(?:\.|\b|\/)" 3; + "~*pcBrowser(?:\.|\b|\/)" 3; + "~*PECL::HTTP(?:\.|\b|\/)" 3; + "~*PeoplePal(?:\.|\b|\/)" 3; + "~*PHPCrawl(?:\.|\b|\/)" 3; + "~*Picscout(?:\.|\b|\/)" 3; + "~*Picsearch(?:\.|\b|\/)" 3; + "~*PictureFinder(?:\.|\b|\/)" 3; + "~*Pimonster(?:\.|\b|\/)" 3; + "~*Pi-Monster(?:\.|\b|\/)" 3; + "~*Pixray(?:\.|\b|\/)" 3; + "~*PleaseCrawl(?:\.|\b|\/)" 3; + "~*plumanalytics(?:\.|\b|\/)" 3; + "~*Pockey(?:\.|\b|\/)" 3; + "~*POE-Component-Client-HTTP(?:\.|\b|\/)" 3; + "~*Probethenet(?:\.|\b|\/)" 3; + "~*ProPowerBot(?:\.|\b|\/)" 3; + "~*ProWebWalker(?:\.|\b|\/)" 3; + "~*Psbot(?:\.|\b|\/)" 3; + "~*Pump(?:\.|\b|\/)" 3; + "~*PxBroker(?:\.|\b|\/)" 3; + "~*PyCurl(?:\.|\b|\/)" 3; + "~*QueryN\ Metasearch(?:\.|\b|\/)" 3; + "~*Quick-Crawler(?:\.|\b|\/)" 3; + "~*RankActive(?:\.|\b|\/)" 3; + "~*RankActiveLinkBot(?:\.|\b|\/)" 3; + "~*RankFlex(?:\.|\b|\/)" 3; + "~*RankingBot(?:\.|\b|\/)" 3; + "~*RankingBot2(?:\.|\b|\/)" 3; + "~*Rankivabot(?:\.|\b|\/)" 3; + "~*RankurBot(?:\.|\b|\/)" 3; + "~*RealDownload(?:\.|\b|\/)" 3; + "~*Reaper(?:\.|\b|\/)" 3; + "~*RebelMouse(?:\.|\b|\/)" 3; + "~*Recorder(?:\.|\b|\/)" 3; + "~*RedesScrapy(?:\.|\b|\/)" 3; + "~*ReGet(?:\.|\b|\/)" 3; + "~*RepoMonkey(?:\.|\b|\/)" 3; + "~*Ripper(?:\.|\b|\/)" 3; + "~*RocketCrawler(?:\.|\b|\/)" 3; + "~*Rogerbot(?:\.|\b|\/)" 3; + "~*s1z.ru(?:\.|\b|\/)" 3; + "~*SalesIntelligent(?:\.|\b|\/)" 3; + "~*SBIder(?:\.|\b|\/)" 3; + "~*ScanAlert(?:\.|\b|\/)" 3; + "~*Scanbot(?:\.|\b|\/)" 3; + "~*scan.lol(?:\.|\b|\/)" 3; + "~*ScoutJet(?:\.|\b|\/)" 3; + "~*Scrapy(?:\.|\b|\/)" 3; + "~*Screaming(?:\.|\b|\/)" 3; + "~*ScreenerBot(?:\.|\b|\/)" 3; + "~*Searchestate(?:\.|\b|\/)" 3; + "~*SearchmetricsBot(?:\.|\b|\/)" 3; + "~*Semrush(?:\.|\b|\/)" 3; + "~*SemrushBot(?:\.|\b|\/)" 3; + "~*SEOkicks(?:\.|\b|\/)" 3; + "~*SEOkicks-Robot(?:\.|\b|\/)" 3; + "~*SEOlyticsCrawler(?:\.|\b|\/)" 3; + "~*Seomoz(?:\.|\b|\/)" 3; + "~*SEOprofiler(?:\.|\b|\/)" 3; + "~*seoscanners(?:\.|\b|\/)" 3; + "~*SeoSiteCheckup(?:\.|\b|\/)" 3; + "~*SEOstats(?:\.|\b|\/)" 3; + "~*serpstatbot(?:\.|\b|\/)" 3; + "~*sexsearcher(?:\.|\b|\/)" 3; + "~*Shodan(?:\.|\b|\/)" 3; + "~*Siphon(?:\.|\b|\/)" 3; + "~*SISTRIX(?:\.|\b|\/)" 3; + "~*Sitebeam(?:\.|\b|\/)" 3; + "~*SiteExplorer(?:\.|\b|\/)" 3; + "~*Siteimprove(?:\.|\b|\/)" 3; + "~*SiteLockSpider(?:\.|\b|\/)" 3; + "~*SiteSnagger(?:\.|\b|\/)" 3; + "~*SiteSucker(?:\.|\b|\/)" 3; + "~*Site\ Sucker(?:\.|\b|\/)" 3; + "~*Sitevigil(?:\.|\b|\/)" 3; + "~*Slackbot-LinkExpanding(?:\.|\b|\/)" 3; + "~*SlySearch(?:\.|\b|\/)" 3; + "~*SmartDownload(?:\.|\b|\/)" 3; + "~*SMTBot(?:\.|\b|\/)" 3; + "~*Snake(?:\.|\b|\/)" 3; + "~*Snapbot(?:\.|\b|\/)" 3; + "~*Snoopy(?:\.|\b|\/)" 3; + "~*SocialRankIOBot(?:\.|\b|\/)" 3; + "~*Sociscraper(?:\.|\b|\/)" 3; + "~*sogouspider(?:\.|\b|\/)" 3; + "~*Sogou\ web\ spider(?:\.|\b|\/)" 3; + "~*Sosospider(?:\.|\b|\/)" 3; + "~*Sottopop(?:\.|\b|\/)" 3; + "~*SpaceBison(?:\.|\b|\/)" 3; + "~*Spammen(?:\.|\b|\/)" 3; + "~*SpankBot(?:\.|\b|\/)" 3; + "~*Spanner(?:\.|\b|\/)" 3; + "~*sp_auditbot(?:\.|\b|\/)" 3; + "~*Spbot(?:\.|\b|\/)" 3; + "~*Spinn3r(?:\.|\b|\/)" 3; + "~*SputnikBot(?:\.|\b|\/)" 3; + "~*spyfu(?:\.|\b|\/)" 3; + "~*Sqlmap(?:\.|\b|\/)" 3; + "~*Sqlworm(?:\.|\b|\/)" 3; + "~*Sqworm(?:\.|\b|\/)" 3; + "~*Steeler(?:\.|\b|\/)" 3; + "~*Stripper(?:\.|\b|\/)" 3; + "~*Sucker(?:\.|\b|\/)" 3; + "~*Sucuri(?:\.|\b|\/)" 3; + "~*SuperBot(?:\.|\b|\/)" 3; + "~*SuperHTTP(?:\.|\b|\/)" 3; + "~*Surfbot(?:\.|\b|\/)" 3; + "~*SurveyBot(?:\.|\b|\/)" 3; + "~*Suzuran(?:\.|\b|\/)" 3; + "~*Swiftbot(?:\.|\b|\/)" 3; + "~*sysscan(?:\.|\b|\/)" 3; + "~*Szukacz(?:\.|\b|\/)" 3; + "~*T0PHackTeam(?:\.|\b|\/)" 3; + "~*T8Abot(?:\.|\b|\/)" 3; + "~*tAkeOut(?:\.|\b|\/)" 3; + "~*Teleport(?:\.|\b|\/)" 3; + "~*TeleportPro(?:\.|\b|\/)" 3; + "~*Telesoft(?:\.|\b|\/)" 3; + "~*Telesphoreo(?:\.|\b|\/)" 3; + "~*Telesphorep(?:\.|\b|\/)" 3; + "~*The\ Intraformant(?:\.|\b|\/)" 3; + "~*TheNomad(?:\.|\b|\/)" 3; + "~*TightTwatBot(?:\.|\b|\/)" 3; + "~*Titan(?:\.|\b|\/)" 3; + "~*Toata(?:\.|\b|\/)" 3; + "~*Toweyabot(?:\.|\b|\/)" 3; + "~*Tracemyfile(?:\.|\b|\/)" 3; + "~*Trendiction(?:\.|\b|\/)" 3; + "~*Trendictionbot(?:\.|\b|\/)" 3; + "~*trendiction.com(?:\.|\b|\/)" 3; + "~*trendiction.de(?:\.|\b|\/)" 3; + "~*True_Robot(?:\.|\b|\/)" 3; + "~*Turingos(?:\.|\b|\/)" 3; + "~*Turnitin(?:\.|\b|\/)" 3; + "~*TurnitinBot(?:\.|\b|\/)" 3; + "~*TwengaBot(?:\.|\b|\/)" 3; + "~*Twice(?:\.|\b|\/)" 3; + "~*Typhoeus(?:\.|\b|\/)" 3; + "~*UnisterBot(?:\.|\b|\/)" 3; + "~*Upflow(?:\.|\b|\/)" 3; + "~*URLy.Warning(?:\.|\b|\/)" 3; + "~*URLy\ Warning(?:\.|\b|\/)" 3; + "~*Vacuum(?:\.|\b|\/)" 3; + "~*Vagabondo(?:\.|\b|\/)" 3; + "~*VB\ Project(?:\.|\b|\/)" 3; + "~*VCI(?:\.|\b|\/)" 3; + "~*VeriCiteCrawler(?:\.|\b|\/)" 3; + "~*VidibleScraper(?:\.|\b|\/)" 3; + "~*Virusdie(?:\.|\b|\/)" 3; + "~*VoidEYE(?:\.|\b|\/)" 3; + "~*Voil(?:\.|\b|\/)" 3; + "~*Voltron(?:\.|\b|\/)" 3; + "~*Wallpapers/3.0(?:\.|\b|\/)" 3; + "~*WallpapersHD(?:\.|\b|\/)" 3; + "~*WASALive-Bot(?:\.|\b|\/)" 3; + "~*WBSearchBot(?:\.|\b|\/)" 3; + "~*Webalta(?:\.|\b|\/)" 3; + "~*WebAuto(?:\.|\b|\/)" 3; + "~*Web\ Auto(?:\.|\b|\/)" 3; + "~*WebBandit(?:\.|\b|\/)" 3; + "~*WebCollage(?:\.|\b|\/)" 3; + "~*Web\ Collage(?:\.|\b|\/)" 3; + "~*WebCopier(?:\.|\b|\/)" 3; + "~*WEBDAV(?:\.|\b|\/)" 3; + "~*WebEnhancer(?:\.|\b|\/)" 3; + "~*Web\ Enhancer(?:\.|\b|\/)" 3; + "~*WebFetch(?:\.|\b|\/)" 3; + "~*Web\ Fetch(?:\.|\b|\/)" 3; + "~*WebFuck(?:\.|\b|\/)" 3; + "~*Web\ Fuck(?:\.|\b|\/)" 3; + "~*WebGo\ IS(?:\.|\b|\/)" 3; + "~*WebImageCollector(?:\.|\b|\/)" 3; + "~*WebLeacher(?:\.|\b|\/)" 3; + "~*WebmasterWorldForumBot(?:\.|\b|\/)" 3; + "~*webmeup-crawler(?:\.|\b|\/)" 3; + "~*WebPix(?:\.|\b|\/)" 3; + "~*Web\ Pix(?:\.|\b|\/)" 3; + "~*WebReaper(?:\.|\b|\/)" 3; + "~*WebSauger(?:\.|\b|\/)" 3; + "~*Web\ Sauger(?:\.|\b|\/)" 3; + "~*Webshag(?:\.|\b|\/)" 3; + "~*WebsiteExtractor(?:\.|\b|\/)" 3; + "~*WebsiteQuester(?:\.|\b|\/)" 3; + "~*Website\ Quester(?:\.|\b|\/)" 3; + "~*Webster(?:\.|\b|\/)" 3; + "~*WebStripper(?:\.|\b|\/)" 3; + "~*WebSucker(?:\.|\b|\/)" 3; + "~*Web\ Sucker(?:\.|\b|\/)" 3; + "~*WebWhacker(?:\.|\b|\/)" 3; + "~*WebZIP(?:\.|\b|\/)" 3; + "~*WeSEE(?:\.|\b|\/)" 3; + "~*Whack(?:\.|\b|\/)" 3; + "~*Whacker(?:\.|\b|\/)" 3; + "~*Whatweb(?:\.|\b|\/)" 3; + "~*Who.is\ Bot(?:\.|\b|\/)" 3; + "~*Widow(?:\.|\b|\/)" 3; + "~*WinHTTrack(?:\.|\b|\/)" 3; + "~*WiseGuys\ Robot(?:\.|\b|\/)" 3; + "~*WISENutbot(?:\.|\b|\/)" 3; + "~*Wonderbot(?:\.|\b|\/)" 3; + "~*Woobot(?:\.|\b|\/)" 3; + "~*Wotbox(?:\.|\b|\/)" 3; + "~*Wprecon(?:\.|\b|\/)" 3; + "~*WPScan(?:\.|\b|\/)" 3; + "~*WWW-Collector-E(?:\.|\b|\/)" 3; + "~*WWW-Mechanize(?:\.|\b|\/)" 3; + "~*WWW::Mechanize(?:\.|\b|\/)" 3; + "~*WWWOFFLE(?:\.|\b|\/)" 3; + "~*x09Mozilla(?:\.|\b|\/)" 3; + "~*x22Mozilla(?:\.|\b|\/)" 3; + "~*Xaldon_WebSpider(?:\.|\b|\/)" 3; + "~*Xaldon\ WebSpider(?:\.|\b|\/)" 3; + "~*Xenu(?:\.|\b|\/)" 3; + "~*xpymep1.exe(?:\.|\b|\/)" 3; + "~*YoudaoBot(?:\.|\b|\/)" 3; + "~*Zade(?:\.|\b|\/)" 3; + "~*Zauba(?:\.|\b|\/)" 3; + "~*zauba.io(?:\.|\b|\/)" 3; + "~*Zermelo(?:\.|\b|\/)" 3; + "~*Zeus(?:\.|\b|\/)" 3; + "~*zgrab(?:\.|\b|\/)" 3; + "~*Zitebot(?:\.|\b|\/)" 3; + "~*ZmEu(?:\.|\b|\/)" 3; + "~*ZumBot(?:\.|\b|\/)" 3; + "~*ZyBorg(?:\.|\b|\/)" 3; +# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # *********************************************** # Allow Good User-Agent Strings We Know and Trust @@ -209,573 +775,11 @@ map $http_user_agent $bad_bot { "~*\bYandexBot\b" 2; "~*\bYandexImages\b" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - -# ********************************************* -# Bad User-Agent Strings That We Block Outright -# ********************************************* -# This includes: -# Known Vulnerability Scanners (now merged into one section) -# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~*/360Spider?(\b)/" 3; - "~*/404checker?(\b)/" 3; - "~*/404enemy?(\b)/" 3; - "~*/80legs?(\b)/" 3; - "~*/Abonti?(\b)/" 3; - "~*/Aboundex?(\b)/" 3; - "~*/Aboundexbot?(\b)/" 3; - "~*/Acunetix?(\b)/" 3; - "~*/ADmantX?(\b)/" 3; - "~*/AfD-Verbotsverfahren?(\b)/" 3; - "~*/AhrefsBot?(\b)/" 3; - "~*/AIBOT?(\b)/" 3; - "~*/AiHitBot?(\b)/" 3; - "~*/Aipbot?(\b)/" 3; - "~*/Alexibot?(\b)/" 3; - "~*/Alligator?(\b)/" 3; - "~*/AllSubmitter?(\b)/" 3; - "~*/AlphaBot?(\b)/" 3; - "~*/Anarchie?(\b)/" 3; - "~*/Apexoo?(\b)/" 3; - "~*/archive.org_bot?(\b)/" 3; - "~*/ASPSeek?(\b)/" 3; - "~*/Asterias?(\b)/" 3; - "~*/Attach?(\b)/" 3; - "~*/autoemailspider?(\b)/" 3; - "~*/BackDoorBot?(\b)/" 3; - "~*/Backlink-Ceck?(\b)/" 3; - "~*/backlink-check?(\b)/" 3; - "~*/BacklinkCrawler?(\b)/" 3; - "~*/BackStreet?(\b)/" 3; - "~*/BackWeb?(\b)/" 3; - "~*/Badass?(\b)/" 3; - "~*/Bandit?(\b)/" 3; - "~*/Barkrowler?(\b)/" 3; - "~*/BatchFTP?(\b)/" 3; - "~*/Battleztar\ Bazinga?(\b)/" 3; - "~*/BBBike?(\b)/" 3; - "~*/BDCbot?(\b)/" 3; - "~*/BDFetch?(\b)/" 3; - "~*/BetaBot?(\b)/" 3; - "~*/Bigfoot?(\b)/" 3; - "~*/Bitacle?(\b)/" 3; - "~*/Blackboard?(\b)/" 3; - "~*/Black\ Hole?(\b)/" 3; - "~*/BlackWidow?(\b)/" 3; - "~*/BLEXBot?(\b)/" 3; - "~*/Blow?(\b)/" 3; - "~*/BlowFish?(\b)/" 3; - "~*/Boardreader?(\b)/" 3; - "~*/Bolt?(\b)/" 3; - "~*/BotALot?(\b)/" 3; - "~*/Brandprotect?(\b)/" 3; - "~*/Brandwatch?(\b)/" 3; - "~*/Bubing?(\b)/" 3; - "~*/Buddy?(\b)/" 3; - "~*/BuiltBotTough?(\b)/" 3; - "~*/BuiltWith?(\b)/" 3; - "~*/Bullseye?(\b)/" 3; - "~*/BunnySlippers?(\b)/" 3; - "~*/BuzzSumo?(\b)/" 3; - "~*/Calculon?(\b)/" 3; - "~*/CATExplorador?(\b)/" 3; - "~*/CazoodleBot?(\b)/" 3; - "~*/CCBot?(\b)/" 3; - "~*/Cegbfeieh?(\b)/" 3; - "~*/CheeseBot?(\b)/" 3; - "~*/CherryPicker?(\b)/" 3; - "~*/CheTeam?(\b)/" 3; - "~*/ChinaClaw?(\b)/" 3; - "~*/Chlooe?(\b)/" 3; - "~*/Claritybot?(\b)/" 3; - "~*/Cliqzbot?(\b)/" 3; - "~*/Cloud\ mapping?(\b)/" 3; - "~*/coccocbot-web?(\b)/" 3; - "~*/Cogentbot?(\b)/" 3; - "~*/cognitiveseo?(\b)/" 3; - "~*/Collector?(\b)/" 3; - "~*/com.plumanalytics?(\b)/" 3; - "~*/Copier?(\b)/" 3; - "~*/CopyRightCheck?(\b)/" 3; - "~*/Copyscape?(\b)/" 3; - "~*/Cosmos?(\b)/" 3; - "~*/Craftbot?(\b)/" 3; - "~*/crawler4j?(\b)/" 3; - "~*/crawler.feedback?(\b)/" 3; - "~*/crawl.sogou.com?(\b)/" 3; - "~*/CrazyWebCrawler?(\b)/" 3; - "~*/Crescent?(\b)/" 3; - "~*/CrunchBot?(\b)/" 3; - "~*/CSHttp?(\b)/" 3; - "~*/Curious?(\b)/" 3; - "~*/Custo?(\b)/" 3; - "~*/DatabaseDriverMysqli?(\b)/" 3; - "~*/DataCha0s?(\b)/" 3; - "~*/DBLBot?(\b)/" 3; - "~*/demandbase-bot?(\b)/" 3; - "~*/Demon?(\b)/" 3; - "~*/Deusu?(\b)/" 3; - "~*/Devil?(\b)/" 3; - "~*/Digincore?(\b)/" 3; - "~*/DigitalPebble?(\b)/" 3; - "~*/DIIbot?(\b)/" 3; - "~*/Dirbuster?(\b)/" 3; - "~*/Disco?(\b)/" 3; - "~*/Discobot?(\b)/" 3; - "~*/Discoverybot?(\b)/" 3; - "~*/Dispatch?(\b)/" 3; - "~*/DittoSpyder?(\b)/" 3; - "~*/DnyzBot?(\b)/" 3; - "~*/DomainAppender?(\b)/" 3; - "~*/DomainCrawler?(\b)/" 3; - "~*/DomainSigmaCrawler?(\b)/" 3; - "~*/DomainStatsBot?(\b)/" 3; - "~*/Dotbot?(\b)/" 3; - "~*/Download\ Wonder?(\b)/" 3; - "~*/Dragonfly?(\b)/" 3; - "~*/Drip?(\b)/" 3; - "~*/DSearch?(\b)/" 3; - "~*/DTS\ Agent?(\b)/" 3; - "~*/EasyDL?(\b)/" 3; - "~*/Ebingbong?(\b)/" 3; - "~*/eCatch?(\b)/" 3; - "~*/ECCP/1.0?(\b)/" 3; - "~*/Ecxi?(\b)/" 3; - "~*/EirGrabber?(\b)/" 3; - "~*/EMail\ Siphon?(\b)/" 3; - "~*/EMail\ Wolf?(\b)/" 3; - "~*/EroCrawler?(\b)/" 3; - "~*/evc-batch?(\b)/" 3; - "~*/Evil?(\b)/" 3; - "~*/Exabot?(\b)/" 3; - "~*/Express\ WebPictures?(\b)/" 3; - "~*/ExtLinksBot?(\b)/" 3; - "~*/Extractor?(\b)/" 3; - "~*/ExtractorPro?(\b)/" 3; - "~*/Extreme\ Picture\ Finder?(\b)/" 3; - "~*/EyeNetIE?(\b)/" 3; - "~*/Ezooms?(\b)/" 3; - "~*/facebookscraper?(\b)/" 3; - "~*/FDM?(\b)/" 3; - "~*/FemtosearchBot?(\b)/" 3; - "~*/FHscan?(\b)/" 3; - "~*/Fimap?(\b)/" 3; - "~*/Firefox/7.0?(\b)/" 3; - "~*/FlashGet?(\b)/" 3; - "~*/Flunky?(\b)/" 3; - "~*/Foobot?(\b)/" 3; - "~*/Freeuploader?(\b)/" 3; - "~*/FrontPage?(\b)/" 3; - "~*/FyberSpider?(\b)/" 3; - "~*/Fyrebot?(\b)/" 3; - "~*/GalaxyBot?(\b)/" 3; - "~*/Genieo?(\b)/" 3; - "~*/GermCrawler?(\b)/" 3; - "~*/Getintent?(\b)/" 3; - "~*/GetRight?(\b)/" 3; - "~*/GetWeb?(\b)/" 3; - "~*/Gigablast?(\b)/" 3; - "~*/Gigabot?(\b)/" 3; - "~*/G-i-g-a-b-o-t?(\b)/" 3; - "~*/Go-Ahead-Got-It?(\b)/" 3; - "~*/Gotit?(\b)/" 3; - "~*/GoZilla?(\b)/" 3; - "~*/Go!Zilla?(\b)/" 3; - "~*/Grabber?(\b)/" 3; - "~*/GrabNet?(\b)/" 3; - "~*/Grafula?(\b)/" 3; - "~*/GrapeFX?(\b)/" 3; - "~*/GrapeshotCrawler?(\b)/" 3; - "~*/GridBot?(\b)/" 3; - "~*/GT::WWW?(\b)/" 3; - "~*/Haansoft?(\b)/" 3; - "~*/HaosouSpider?(\b)/" 3; - "~*/Harvest?(\b)/" 3; - "~*/Havij?(\b)/" 3; - "~*/HEADMasterSEO?(\b)/" 3; - "~*/Heritrix?(\b)/" 3; - "~*/Hloader?(\b)/" 3; - "~*/HMView?(\b)/" 3; - "~*/HTMLparser?(\b)/" 3; - "~*/HTTP::Lite?(\b)/" 3; - "~*/HTTrack?(\b)/" 3; - "~*/Humanlinks?(\b)/" 3; - "~*/HybridBot?(\b)/" 3; - "~*/Iblog?(\b)/" 3; - "~*/IDBot?(\b)/" 3; - "~*/Id-search?(\b)/" 3; - "~*/IlseBot?(\b)/" 3; - "~*/Image\ Fetch?(\b)/" 3; - "~*/Image\ Sucker?(\b)/" 3; - "~*/IndeedBot?(\b)/" 3; - "~*/Indy\ Library?(\b)/" 3; - "~*/InfoNaviRobot?(\b)/" 3; - "~*/InfoTekies?(\b)/" 3; - "~*/instabid?(\b)/" 3; - "~*/Intelliseek?(\b)/" 3; - "~*/InterGET?(\b)/" 3; - "~*/Internet\ Ninja?(\b)/" 3; - "~*/InternetSeer?(\b)/" 3; - "~*/internetVista\ monitor?(\b)/" 3; - "~*/ips-agent?(\b)/" 3; - "~*/Iria?(\b)/" 3; - "~*/IRLbot?(\b)/" 3; - "~*/Iskanie?(\b)/" 3; - "~*/IstellaBot?(\b)/" 3; - "~*/JamesBOT?(\b)/" 3; - "~*/Jbrofuzz?(\b)/" 3; - "~*/JennyBot?(\b)/" 3; - "~*/JetCar?(\b)/" 3; - "~*/Jetty?(\b)/" 3; - "~*/JikeSpider?(\b)/" 3; - "~*/JOC\ Web\ Spider?(\b)/" 3; - "~*/Joomla?(\b)/" 3; - "~*/Jorgee?(\b)/" 3; - "~*/JustView?(\b)/" 3; - "~*/Jyxobot?(\b)/" 3; - "~*/Kenjin\ Spider?(\b)/" 3; - "~*/Keyword\ Density?(\b)/" 3; - "~*/Kozmosbot?(\b)/" 3; - "~*/Lanshanbot?(\b)/" 3; - "~*/Larbin?(\b)/" 3; - "~*/LeechFTP?(\b)/" 3; - "~*/LeechGet?(\b)/" 3; - "~*/LexiBot?(\b)/" 3; - "~*/Lftp?(\b)/" 3; - "~*/LibWeb?(\b)/" 3; - "~*/Libwhisker?(\b)/" 3; - "~*/Lightspeedsystems?(\b)/" 3; - "~*/Likse?(\b)/" 3; - "~*/Linkdexbot?(\b)/" 3; - "~*/LinkextractorPro?(\b)/" 3; - "~*/LinkpadBot?(\b)/" 3; - "~*/LinkScan?(\b)/" 3; - "~*/LinksManager?(\b)/" 3; - "~*/LinkWalker?(\b)/" 3; - "~*/LinqiaMetadataDownloaderBot?(\b)/" 3; - "~*/LinqiaRSSBot?(\b)/" 3; - "~*/LinqiaScrapeBot?(\b)/" 3; - "~*/Lipperhey?(\b)/" 3; - "~*/Lipperhey\ Spider?(\b)/" 3; - "~*/Litemage_walker?(\b)/" 3; - "~*/Lmspider?(\b)/" 3; - "~*/LNSpiderguy?(\b)/" 3; - "~*/Ltx71?(\b)/" 3; - "~*/lwp-request?(\b)/" 3; - "~*/LWP::Simple?(\b)/" 3; - "~*/lwp-trivial?(\b)/" 3; - "~*/Magnet?(\b)/" 3; - "~*/Mag-Net?(\b)/" 3; - "~*/magpie-crawler?(\b)/" 3; - "~*/Mail.RU_Bot?(\b)/" 3; - "~*/Majestic12?(\b)/" 3; - "~*/Majestic-SEO?(\b)/" 3; - "~*/Majestic\ SEO?(\b)/" 3; - "~*/MarkMonitor?(\b)/" 3; - "~*/MarkWatch?(\b)/" 3; - "~*/Masscan?(\b)/" 3; - "~*/Mass\ Downloader?(\b)/" 3; - "~*/Mata\ Hari?(\b)/" 3; - "~*/MauiBot?(\b)/" 3; - "~*/meanpathbot?(\b)/" 3; - "~*/Meanpathbot?(\b)/" 3; - "~*/MeanPath\ Bot?(\b)/" 3; - "~*/Mediatoolkitbot?(\b)/" 3; - "~*/mediawords?(\b)/" 3; - "~*/MegaIndex.ru?(\b)/" 3; - "~*/Metauri?(\b)/" 3; - "~*/MFC_Tear_Sample?(\b)/" 3; - "~*/Microsoft\ Data\ Access?(\b)/" 3; - "~*/Microsoft\ URL\ Control?(\b)/" 3; - "~*/MIDown\ tool?(\b)/" 3; - "~*/MIIxpc?(\b)/" 3; - "~*/Mister\ PiX?(\b)/" 3; - "~*/MJ12bot?(\b)/" 3; - "~*/Mojeek?(\b)/" 3; - "~*/Morfeus\ Fucking\ Scanner?(\b)/" 3; - "~*/Mr.4x3?(\b)/" 3; - "~*/MSFrontPage?(\b)/" 3; - "~*/MSIECrawler?(\b)/" 3; - "~*/Msrabot?(\b)/" 3; - "~*/MS\ Web\ Services\ Client\ Protocol?(\b)/" 3; - "~*/muhstik-scan?(\b)/" 3; - "~*/Musobot?(\b)/" 3; - "~*/Name\ Intelligence?(\b)/" 3; - "~*/Nameprotect?(\b)/" 3; - "~*/Navroad?(\b)/" 3; - "~*/NearSite?(\b)/" 3; - "~*/Needle?(\b)/" 3; - "~*/Nessus?(\b)/" 3; - "~*/NetAnts?(\b)/" 3; - "~*/Netcraft?(\b)/" 3; - "~*/netEstate\ NE\ Crawler?(\b)/" 3; - "~*/NetLyzer?(\b)/" 3; - "~*/NetMechanic?(\b)/" 3; - "~*/NetSpider?(\b)/" 3; - "~*/Nettrack?(\b)/" 3; - "~*/Net\ Vampire?(\b)/" 3; - "~*/Netvibes?(\b)/" 3; - "~*/NetZIP?(\b)/" 3; - "~*/NextGenSearchBot?(\b)/" 3; - "~*/Nibbler?(\b)/" 3; - "~*/NICErsPRO?(\b)/" 3; - "~*/Niki-bot?(\b)/" 3; - "~*/Nikto?(\b)/" 3; - "~*/NimbleCrawler?(\b)/" 3; - "~*/Nimbostratus?(\b)/" 3; - "~*/Ninja?(\b)/" 3; - "~*/Nmap?(\b)/" 3; - "~*/NPbot?(\b)/" 3; - "~*/Nutch?(\b)/" 3; - "~*/oBot?(\b)/" 3; - "~*/Octopus?(\b)/" 3; - "~*/Offline\ Explorer?(\b)/" 3; - "~*/Offline\ Navigator?(\b)/" 3; - "~*/OnCrawl?(\b)/" 3; - "~*/Openfind?(\b)/" 3; - "~*/OpenLinkProfiler?(\b)/" 3; - "~*/Openvas?(\b)/" 3; - "~*/OpenVAS?(\b)/" 3; - "~*/OrangeBot?(\b)/" 3; - "~*/OrangeSpider?(\b)/" 3; - "~*/OutclicksBot?(\b)/" 3; - "~*/OutfoxBot?(\b)/" 3; - "~*/PageAnalyzer?(\b)/" 3; - "~*/Page\ Analyzer?(\b)/" 3; - "~*/PageGrabber?(\b)/" 3; - "~*/page\ scorer?(\b)/" 3; - "~*/PageScorer?(\b)/" 3; - "~*/Pandalytics?(\b)/" 3; - "~*/Panscient?(\b)/" 3; - "~*/Papa\ Foto?(\b)/" 3; - "~*/Pavuk?(\b)/" 3; - "~*/pcBrowser?(\b)/" 3; - "~*/PECL::HTTP?(\b)/" 3; - "~*/PeoplePal?(\b)/" 3; - "~*/PHPCrawl?(\b)/" 3; - "~*/Picscout?(\b)/" 3; - "~*/Picsearch?(\b)/" 3; - "~*/PictureFinder?(\b)/" 3; - "~*/Pimonster?(\b)/" 3; - "~*/Pi-Monster?(\b)/" 3; - "~*/Pixray?(\b)/" 3; - "~*/PleaseCrawl?(\b)/" 3; - "~*/plumanalytics?(\b)/" 3; - "~*/Pockey?(\b)/" 3; - "~*/POE-Component-Client-HTTP?(\b)/" 3; - "~*/Probethenet?(\b)/" 3; - "~*/ProPowerBot?(\b)/" 3; - "~*/ProWebWalker?(\b)/" 3; - "~*/Psbot?(\b)/" 3; - "~*/Pump?(\b)/" 3; - "~*/PxBroker?(\b)/" 3; - "~*/PyCurl?(\b)/" 3; - "~*/QueryN\ Metasearch?(\b)/" 3; - "~*/Quick-Crawler?(\b)/" 3; - "~*/RankActive?(\b)/" 3; - "~*/RankActiveLinkBot?(\b)/" 3; - "~*/RankFlex?(\b)/" 3; - "~*/RankingBot?(\b)/" 3; - "~*/RankingBot2?(\b)/" 3; - "~*/Rankivabot?(\b)/" 3; - "~*/RankurBot?(\b)/" 3; - "~*/RealDownload?(\b)/" 3; - "~*/Reaper?(\b)/" 3; - "~*/RebelMouse?(\b)/" 3; - "~*/Recorder?(\b)/" 3; - "~*/RedesScrapy?(\b)/" 3; - "~*/ReGet?(\b)/" 3; - "~*/RepoMonkey?(\b)/" 3; - "~*/Ripper?(\b)/" 3; - "~*/RocketCrawler?(\b)/" 3; - "~*/Rogerbot?(\b)/" 3; - "~*/s1z.ru?(\b)/" 3; - "~*/SalesIntelligent?(\b)/" 3; - "~*/SBIder?(\b)/" 3; - "~*/ScanAlert?(\b)/" 3; - "~*/Scanbot?(\b)/" 3; - "~*/scan.lol?(\b)/" 3; - "~*/ScoutJet?(\b)/" 3; - "~*/Scrapy?(\b)/" 3; - "~*/Screaming?(\b)/" 3; - "~*/ScreenerBot?(\b)/" 3; - "~*/Searchestate?(\b)/" 3; - "~*/SearchmetricsBot?(\b)/" 3; - "~*/Semrush?(\b)/" 3; - "~*/SemrushBot?(\b)/" 3; - "~*/SEOkicks?(\b)/" 3; - "~*/SEOkicks-Robot?(\b)/" 3; - "~*/SEOlyticsCrawler?(\b)/" 3; - "~*/Seomoz?(\b)/" 3; - "~*/SEOprofiler?(\b)/" 3; - "~*/seoscanners?(\b)/" 3; - "~*/SeoSiteCheckup?(\b)/" 3; - "~*/SEOstats?(\b)/" 3; - "~*/serpstatbot?(\b)/" 3; - "~*/sexsearcher?(\b)/" 3; - "~*/Shodan?(\b)/" 3; - "~*/Siphon?(\b)/" 3; - "~*/SISTRIX?(\b)/" 3; - "~*/Sitebeam?(\b)/" 3; - "~*/SiteExplorer?(\b)/" 3; - "~*/Siteimprove?(\b)/" 3; - "~*/SiteLockSpider?(\b)/" 3; - "~*/SiteSnagger?(\b)/" 3; - "~*/SiteSucker?(\b)/" 3; - "~*/Site\ Sucker?(\b)/" 3; - "~*/Sitevigil?(\b)/" 3; - "~*/Slackbot-LinkExpanding?(\b)/" 3; - "~*/SlySearch?(\b)/" 3; - "~*/SmartDownload?(\b)/" 3; - "~*/SMTBot?(\b)/" 3; - "~*/Snake?(\b)/" 3; - "~*/Snapbot?(\b)/" 3; - "~*/Snoopy?(\b)/" 3; - "~*/SocialRankIOBot?(\b)/" 3; - "~*/Sociscraper?(\b)/" 3; - "~*/sogouspider?(\b)/" 3; - "~*/Sogou\ web\ spider?(\b)/" 3; - "~*/Sosospider?(\b)/" 3; - "~*/Sottopop?(\b)/" 3; - "~*/SpaceBison?(\b)/" 3; - "~*/Spammen?(\b)/" 3; - "~*/SpankBot?(\b)/" 3; - "~*/Spanner?(\b)/" 3; - "~*/sp_auditbot?(\b)/" 3; - "~*/Spbot?(\b)/" 3; - "~*/Spinn3r?(\b)/" 3; - "~*/SputnikBot?(\b)/" 3; - "~*/spyfu?(\b)/" 3; - "~*/Sqlmap?(\b)/" 3; - "~*/Sqlworm?(\b)/" 3; - "~*/Sqworm?(\b)/" 3; - "~*/Steeler?(\b)/" 3; - "~*/Stripper?(\b)/" 3; - "~*/Sucker?(\b)/" 3; - "~*/Sucuri?(\b)/" 3; - "~*/SuperBot?(\b)/" 3; - "~*/SuperHTTP?(\b)/" 3; - "~*/Surfbot?(\b)/" 3; - "~*/SurveyBot?(\b)/" 3; - "~*/Suzuran?(\b)/" 3; - "~*/Swiftbot?(\b)/" 3; - "~*/sysscan?(\b)/" 3; - "~*/Szukacz?(\b)/" 3; - "~*/T0PHackTeam?(\b)/" 3; - "~*/T8Abot?(\b)/" 3; - "~*/tAkeOut?(\b)/" 3; - "~*/Teleport?(\b)/" 3; - "~*/TeleportPro?(\b)/" 3; - "~*/Telesoft?(\b)/" 3; - "~*/Telesphoreo?(\b)/" 3; - "~*/Telesphorep?(\b)/" 3; - "~*/The\ Intraformant?(\b)/" 3; - "~*/TheNomad?(\b)/" 3; - "~*/TightTwatBot?(\b)/" 3; - "~*/Titan?(\b)/" 3; - "~*/Toata?(\b)/" 3; - "~*/Toweyabot?(\b)/" 3; - "~*/Tracemyfile?(\b)/" 3; - "~*/Trendiction?(\b)/" 3; - "~*/Trendictionbot?(\b)/" 3; - "~*/trendiction.com?(\b)/" 3; - "~*/trendiction.de?(\b)/" 3; - "~*/True_Robot?(\b)/" 3; - "~*/Turingos?(\b)/" 3; - "~*/Turnitin?(\b)/" 3; - "~*/TurnitinBot?(\b)/" 3; - "~*/TwengaBot?(\b)/" 3; - "~*/Twice?(\b)/" 3; - "~*/Typhoeus?(\b)/" 3; - "~*/UnisterBot?(\b)/" 3; - "~*/Upflow?(\b)/" 3; - "~*/URLy.Warning?(\b)/" 3; - "~*/URLy\ Warning?(\b)/" 3; - "~*/Vacuum?(\b)/" 3; - "~*/Vagabondo?(\b)/" 3; - "~*/VB\ Project?(\b)/" 3; - "~*/VCI?(\b)/" 3; - "~*/VeriCiteCrawler?(\b)/" 3; - "~*/VidibleScraper?(\b)/" 3; - "~*/Virusdie?(\b)/" 3; - "~*/VoidEYE?(\b)/" 3; - "~*/Voil?(\b)/" 3; - "~*/Voltron?(\b)/" 3; - "~*/Wallpapers/3.0?(\b)/" 3; - "~*/WallpapersHD?(\b)/" 3; - "~*/WASALive-Bot?(\b)/" 3; - "~*/WBSearchBot?(\b)/" 3; - "~*/Webalta?(\b)/" 3; - "~*/WebAuto?(\b)/" 3; - "~*/Web\ Auto?(\b)/" 3; - "~*/WebBandit?(\b)/" 3; - "~*/WebCollage?(\b)/" 3; - "~*/Web\ Collage?(\b)/" 3; - "~*/WebCopier?(\b)/" 3; - "~*/WEBDAV?(\b)/" 3; - "~*/WebEnhancer?(\b)/" 3; - "~*/Web\ Enhancer?(\b)/" 3; - "~*/WebFetch?(\b)/" 3; - "~*/Web\ Fetch?(\b)/" 3; - "~*/WebFuck?(\b)/" 3; - "~*/Web\ Fuck?(\b)/" 3; - "~*/WebGo\ IS?(\b)/" 3; - "~*/WebImageCollector?(\b)/" 3; - "~*/WebLeacher?(\b)/" 3; - "~*/WebmasterWorldForumBot?(\b)/" 3; - "~*/webmeup-crawler?(\b)/" 3; - "~*/WebPix?(\b)/" 3; - "~*/Web\ Pix?(\b)/" 3; - "~*/WebReaper?(\b)/" 3; - "~*/WebSauger?(\b)/" 3; - "~*/Web\ Sauger?(\b)/" 3; - "~*/Webshag?(\b)/" 3; - "~*/WebsiteExtractor?(\b)/" 3; - "~*/WebsiteQuester?(\b)/" 3; - "~*/Website\ Quester?(\b)/" 3; - "~*/Webster?(\b)/" 3; - "~*/WebStripper?(\b)/" 3; - "~*/WebSucker?(\b)/" 3; - "~*/Web\ Sucker?(\b)/" 3; - "~*/WebWhacker?(\b)/" 3; - "~*/WebZIP?(\b)/" 3; - "~*/WeSEE?(\b)/" 3; - "~*/Whack?(\b)/" 3; - "~*/Whacker?(\b)/" 3; - "~*/Whatweb?(\b)/" 3; - "~*/Who.is\ Bot?(\b)/" 3; - "~*/Widow?(\b)/" 3; - "~*/WinHTTrack?(\b)/" 3; - "~*/WiseGuys\ Robot?(\b)/" 3; - "~*/WISENutbot?(\b)/" 3; - "~*/Wonderbot?(\b)/" 3; - "~*/Woobot?(\b)/" 3; - "~*/Wotbox?(\b)/" 3; - "~*/Wprecon?(\b)/" 3; - "~*/WPScan?(\b)/" 3; - "~*/WWW-Collector-E?(\b)/" 3; - "~*/WWW-Mechanize?(\b)/" 3; - "~*/WWW::Mechanize?(\b)/" 3; - "~*/WWWOFFLE?(\b)/" 3; - "~*/x09Mozilla?(\b)/" 3; - "~*/x22Mozilla?(\b)/" 3; - "~*/Xaldon_WebSpider?(\b)/" 3; - "~*/Xaldon\ WebSpider?(\b)/" 3; - "~*/Xenu?(\b)/" 3; - "~*/xpymep1.exe?(\b)/" 3; - "~*/YoudaoBot?(\b)/" 3; - "~*/Zade?(\b)/" 3; - "~*/Zauba?(\b)/" 3; - "~*/zauba.io?(\b)/" 3; - "~*/Zermelo?(\b)/" 3; - "~*/Zeus?(\b)/" 3; - "~*/zgrab?(\b)/" 3; - "~*/Zitebot?(\b)/" 3; - "~*/ZmEu?(\b)/" 3; - "~*/ZumBot?(\b)/" 3; - "~*/ZyBorg?(\b)/" 3; -# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### + + + } diff --git a/.dev-tools/_curl_tests/curltest1.txt b/.dev-tools/_curl_tests/curltest1.txt index 41c27b363..40f3c1399 100755 --- a/.dev-tools/_curl_tests/curltest1.txt +++ b/.dev-tools/_curl_tests/curltest1.txt @@ -1,4 +1,4 @@ -Last Tested: Thu Jun 20 22:14:36 SAST 2019 +Last Tested: Fri Jun 21 08:47:58 SAST 2019 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest2.txt b/.dev-tools/_curl_tests/curltest2.txt index 41c27b363..40f3c1399 100755 --- a/.dev-tools/_curl_tests/curltest2.txt +++ b/.dev-tools/_curl_tests/curltest2.txt @@ -1,4 +1,4 @@ -Last Tested: Thu Jun 20 22:14:36 SAST 2019 +Last Tested: Fri Jun 21 08:47:58 SAST 2019 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest3.txt b/.dev-tools/_curl_tests/curltest3.txt index 41c27b363..40f3c1399 100755 --- a/.dev-tools/_curl_tests/curltest3.txt +++ b/.dev-tools/_curl_tests/curltest3.txt @@ -1,4 +1,4 @@ -Last Tested: Thu Jun 20 22:14:36 SAST 2019 +Last Tested: Fri Jun 21 08:47:58 SAST 2019 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest4.txt b/.dev-tools/_curl_tests/curltest4.txt index 41c27b363..40f3c1399 100755 --- a/.dev-tools/_curl_tests/curltest4.txt +++ b/.dev-tools/_curl_tests/curltest4.txt @@ -1,4 +1,4 @@ -Last Tested: Thu Jun 20 22:14:36 SAST 2019 +Last Tested: Fri Jun 21 08:47:58 SAST 2019 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed diff --git a/.dev-tools/_curl_tests/curltest5.txt b/.dev-tools/_curl_tests/curltest5.txt index 53d104a36..9f0a87796 100755 --- a/.dev-tools/_curl_tests/curltest5.txt +++ b/.dev-tools/_curl_tests/curltest5.txt @@ -1,4 +1,4 @@ -Last Tested: Thu Jun 20 22:14:36 SAST 2019 +Last Tested: Fri Jun 21 08:47:58 SAST 2019