vikarti.anatra/nginx-ultimate-bad-bot-blocker
1
0
Fork 0
mirror of https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git synced 2025-09-02 18:50:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

New Referers / Bots Added

Browse source
This commit is contained in:
Mitchell Krog UB1 2017-01-29 12:03:51 +02:00
parent 0ba196ed29
commit f87b5576f2
2 changed files with 434 additions and 305 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

303
conf.d/globalblacklist.conf
View file

@ -17,11 +17,11 @@
### - https://github.com/oohnoitz/nginx-blacklist ### - https://github.com/oohnoitz/nginx-blacklist
### Last Updated ### Last Updated
### Sun Jan 29 11:24:21 SAST 2017 ### Sun Jan 29 12:03:50 SAST 2017
### End Last Updated ### End Last Updated
### Generated in ### Generated in
### 0.29238986969 seconds ### 0.272795915604 seconds
### End Generated in ### End Generated in
### Tested on: nginx/1.10.0 (Ubuntu 16.04) ### Tested on: nginx/1.10.0 (Ubuntu 16.04)
@ -200,23 +200,23 @@ map $http_user_agent $bad_bot {
# Known Vulnerability Scanners (now merged into one section) # Known Vulnerability Scanners (now merged into one section)
# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###
"~*80legs" 3;
"~*360Spider" 3; "~*360Spider" 3;
"~*Aboundex" 3; "~*80legs" 3;
"~*Abonti" 3; "~*Abonti" 3;
"~*Aboundex" 3;
"~*Acunetix" 3; "~*Acunetix" 3;
"~*AhrefsBot" 3; "~*AhrefsBot" 3;
"~*AIBOT" 3; "~*AIBOT" 3;
"~*aipbot" 3; "~*AiHitBot" 3;
"~*aiHitBot" 3; "~*Aipbot" 3;
"~*Alexibot" 3; "~*Alexibot" 3;
"~*Alligator" 3; "~*Alligator" 3;
"~*AllSubmitter" 3; "~*AllSubmitter" 3;
"~*Anarchie" 3; "~*Anarchie" 3;
"~*Apexoo" 3; "~*Apexoo" 3;
"~*asterias" 3;
"~*ASPSeek" 3; "~*ASPSeek" 3;
"~*attach" 3; "~*Asterias" 3;
"~*Attach" 3;
"~*autoemailspider" 3; "~*autoemailspider" 3;
"~*BackDoorBot" 3; "~*BackDoorBot" 3;
"~*BackStreet" 3; "~*BackStreet" 3;
@ -227,30 +227,34 @@ map $http_user_agent $bad_bot {
"~*Battleztar\ Bazinga" 3; "~*Battleztar\ Bazinga" 3;
"~*BBBike" 3; "~*BBBike" 3;
"~*BDFetch" 3; "~*BDFetch" 3;
"~*BetaBot" 3;
"~*Bigfoot" 3; "~*Bigfoot" 3;
"~*Bitacle" 3; "~*Bitacle" 3;
"~*Black\ Hole" 3; "~*Black\ Hole" 3;
"~*BlackWidow" 3; "~*BlackWidow" 3;
"~*BLEXBot" 3; "~*BLEXBot" 3;
"~*BlowFish" 3;
"~*Blow" 3; "~*Blow" 3;
"~*boardreader" 3; "~*BlowFish" 3;
"~*Boardreader" 3;
"~*Bolt" 3;
"~*BotALot" 3; "~*BotALot" 3;
"~*Brandprotect" 3; "~*Brandprotect" 3;
"~*Brandwatch" 3; "~*Brandwatch" 3;
"~*Bubing" 3;
"~*Buddy" 3; "~*Buddy" 3;
"~*BuiltBotTough" 3; "~*BuiltBotTough" 3;
"~*BuiltWith" 3; "~*BuiltWith" 3;
"~*Bullseye" 3; "~*Bullseye" 3;
"~*BunnySlippers" 3; "~*BunnySlippers" 3;
"~*BuzzSumo" 3; "~*BuzzSumo" 3;
"~*calculon\ spider" 3; "~*Calculon" 3;
"~*CazoodleBot" 3;
"~*CCBot" 3; "~*CCBot" 3;
"~*Cegbfeieh" 3; "~*Cegbfeieh" 3;
"~*CheeseBot" 3; "~*CheeseBot" 3;
"~*CherryPicker" 3; "~*CherryPicker" 3;
"~*ChinaClaw" 3; "~*ChinaClaw" 3;
"~*chlooe" 3; "~*Chlooe" 3;
"~*Claritybot" 3; "~*Claritybot" 3;
"~*Cliqzbot" 3; "~*Cliqzbot" 3;
"~*Cogentbot" 3; "~*Cogentbot" 3;
@ -259,42 +263,41 @@ map $http_user_agent $bad_bot {
"~*CopyRightCheck" 3; "~*CopyRightCheck" 3;
"~*Copyscape" 3; "~*Copyscape" 3;
"~*Cosmos" 3; "~*Cosmos" 3;
"~*craftbot@yahoo.com" 3; "~*Craftbot" 3;
"~*CrazyWebCrawler" 3; "~*CrazyWebCrawler" 3;
"~*Crescent" 3; "~*Crescent" 3;
"~*CSHttp" 3; "~*CSHttp" 3;
"~*Curious" 3;
"~*Custo" 3; "~*Custo" 3;
"~*DBLBot" 3; "~*DBLBot" 3;
"~*Demon" 3; "~*Demon" 3;
"~*deusu" 3; "~*Deusu" 3;
"~*Devil" 3; "~*Devil" 3;
"~*Disco" 3;
"~*DIIbot" 3; "~*DIIbot" 3;
"~*dirbuster" 3; "~*Dirbuster" 3;
"~*discobot" 3; "~*Disco" 3;
"~*discoverybot" 3; "~*Discobot" 3;
"~*Discoverybot" 3;
"~*DittoSpyder" 3; "~*DittoSpyder" 3;
"~*DomainAppender" 3; "~*DomainAppender" 3;
"~*DomainSigmaCrawler" 3; "~*DomainSigmaCrawler" 3;
"~*dotbot" 3; "~*Dotbot" 3;
"~*Download\ Demon" 3; "~*Download\ Demon" 3;
"~*Download\ Devil" 3; "~*Download\ Devil" 3;
"~*Download\ Wonder" 3; "~*Download\ Wonder" 3;
"~*dragonfly" 3; "~*Dragonfly" 3;
"~*Drip" 3; "~*Drip" 3;
"~*DTS\ Agent" 3; "~*DTS\ Agent" 3;
"~*EasyDL" 3; "~*EasyDL" 3;
"~*ebingbong" 3; "~*Ebingbong" 3;
"~*eCatch" 3; "~*eCatch" 3;
"~*ECCP/1.0" 3; "~*ECCP/1.0" 3;
"~*Ecxi" 3;
"~*EirGrabber" 3; "~*EirGrabber" 3;
"~*EMail\ Collector" 3; "~*EMail\ Collector" 3;
"~*EmailCollector" 3;
"~*EMail\ Extractor" 3; "~*EMail\ Extractor" 3;
"~*EMail\ Siphon" 3; "~*EMail\ Siphon" 3;
"~*EmailSiphon" 3;
"~*EMail\ Wolf" 3; "~*EMail\ Wolf" 3;
"~*EmailWolf" 3;
"~*EroCrawler" 3; "~*EroCrawler" 3;
"~*Evil" 3; "~*Evil" 3;
"~*Exabot" 3; "~*Exabot" 3;
@ -302,43 +305,48 @@ map $http_user_agent $bad_bot {
"~*Extractor" 3; "~*Extractor" 3;
"~*ExtractorPro" 3; "~*ExtractorPro" 3;
"~*EyeNetIE" 3; "~*EyeNetIE" 3;
"~*ezooms" 3; "~*Ezooms" 3;
"~*FHscan" 3; "~*FHscan" 3;
"~*fimap" 3; "~*Fimap" 3;
"~*Findxbot" 3;
"~*FlashGet" 3; "~*FlashGet" 3;
"~*flunky" 3; "~*Flunky" 3;
"~*Foobot" 3; "~*Foobot" 3;
"~*freeuploader" 3; "~*Freeuploader" 3;
"~*FrontPage" 3; "~*FrontPage" 3;
"~*GalaxyBot" 3; "~*GalaxyBot" 3;
"~*Genieo" 3; "~*Genieo" 3;
"~*getintent" 3; "~*Getintent" 3;
"~*GetRight" 3; "~*GetRight" 3;
"~*GetWeb" 3; "~*GetWeb" 3;
"~*gigablast" 3; "~*Gigablast" 3;
"~*gigabot" 3; "~*Gigabot" 3;
"~*Go-Ahead-Got-It" 3; "~*Go-Ahead-Got-It" 3;
"~*gotit" 3;
"~*Go!Zilla" 3; "~*Go!Zilla" 3;
"~*Gotit" 3;
"~*GoZilla" 3;
"~*Grabber" 3; "~*Grabber" 3;
"~*GrabNet" 3; "~*GrabNet" 3;
"~*Grafula" 3; "~*Grafula" 3;
"~*grapeFX" 3; "~*GrapeFX" 3;
"~*GrapeshotCrawler" 3; "~*GrapeshotCrawler" 3;
"~*GridBot" 3; "~*GridBot" 3;
"~*GT::WWW" 3;
"~*GuzzleHttp" 3;
"~*HaosouSpider" 3; "~*HaosouSpider" 3;
"~*Harvest" 3; "~*Harvest" 3;
"~*Havij" 3; "~*Havij" 3;
"~*Heritrix" 3; "~*Heritrix" 3;
"~*hloader" 3; "~*Hloader" 3;
"~*HMView" 3; "~*HMView" 3;
"~*HTTrack" 3;
"~*HTMLparser" 3; "~*HTMLparser" 3;
"~*Huaweisymantecspider" 3; "~*HTTP::Lite" 3;
"~*humanlinks" 3; "~*HTTrack" 3;
"~*iblog" 3; "~*Humanlinks" 3;
"~*Iblog" 3;
"~*Id-search" 3;
"~*IDBot" 3;
"~*IlseBot" 3; "~*IlseBot" 3;
"~*imagefetch" 3;
"~*Image\ Fetch" 3; "~*Image\ Fetch" 3;
"~*Image\ Stripper" 3; "~*Image\ Stripper" 3;
"~*Image\ Sucker" 3; "~*Image\ Sucker" 3;
@ -346,91 +354,100 @@ map $http_user_agent $bad_bot {
"~*InfoNaviRobot" 3; "~*InfoNaviRobot" 3;
"~*InfoTekies" 3; "~*InfoTekies" 3;
"~*Intelliseek" 3; "~*Intelliseek" 3;
"~*Internet\ Ninja" 3;
"~*internetVista\ monitor" 3;
"~*InterGET" 3; "~*InterGET" 3;
"~*Internet\ Ninja" 3;
"~*InternetSeer" 3;
"~*internetVista\ monitor" 3;
"~*Iria" 3; "~*Iria" 3;
"~*iskanie" 3; "~*IRLbot" 3;
"~*Iskanie" 3;
"~*JamesBOT" 3; "~*JamesBOT" 3;
"~*Jbrofuzz" 3;
"~*JennyBot" 3; "~*JennyBot" 3;
"~*JetCar" 3; "~*JetCar" 3;
"~*jbrofuzz" 3;
"~*JikeSpider" 3; "~*JikeSpider" 3;
"~*JOC" 3;
"~*JOC\ Web\ Spider" 3; "~*JOC\ Web\ Spider" 3;
"~*Joomla" 3; "~*Joomla" 3;
"~*JustView" 3; "~*JustView" 3;
"~*Jyxobot" 3; "~*Jyxobot" 3;
"~*Kenjin\ Spider" 3; "~*Kenjin\ Spider" 3;
"~*Keyword\ Density" 3; "~*Keyword\ Density" 3;
"~*kraken" 3; "~*Kraken" 3;
"~*lanshanbot" 3; "~*Lanshanbot" 3;
"~*larbin" 3; "~*Larbin" 3;
"~*LeechFTP" 3; "~*LeechFTP" 3;
"~*LeechGet" 3; "~*LeechGet" 3;
"~*LexiBot" 3; "~*LexiBot" 3;
"~*lftp" 3; "~*Lftp" 3;
"~*libcurl" 3; "~*LibWeb" 3;
"~*libWeb" 3; "~*Libwhisker" 3;
"~*libwhisker" 3; "~*Lightspeedsystems" 3;
"~*likse" 3; "~*Likse" 3;
"~*linkdexbot" 3; "~*Linkdexbot" 3;
"~*LinkpadBot" 3;
"~*LinkextractorPro" 3; "~*LinkextractorPro" 3;
"~*LinkpadBot" 3;
"~*LinkScan" 3; "~*LinkScan" 3;
"~*LinqiaMetadataDownloaderBot" 3; "~*LinksManager" 3;
"~*LinqiaScrapeBot" 3;
"~*LinqiaRSSBot" 3;
"~*lmspider" 3;
"~*LNSpiderguy" 3;
"~*LinkWalker" 3; "~*LinkWalker" 3;
"~*ltx71" 3; "~*LinqiaMetadataDownloaderBot" 3;
"~*LinqiaRSSBot" 3;
"~*LinqiaScrapeBot" 3;
"~*Lipperhey" 3;
"~*Litemage_walker" 3;
"~*Lmspider" 3;
"~*LNSpiderguy" 3;
"~*Ltx71" 3;
"~*lwp-request" 3; "~*lwp-request" 3;
"~*lwp-trivial" 3;
"~*LWP::Simple" 3; "~*LWP::Simple" 3;
"~*magpie-crawler" 3;
"~*Magnet" 3;
"~*Mag-Net" 3; "~*Mag-Net" 3;
"~*mail.ru" 3; "~*Magnet" 3;
"~*Mail.RU_Bot" 3; "~*magpie-crawler" 3;
"~*majestic12" 3; "~*Mail.ru" 3;
"~*Majestic12" 3;
"~*MarkMonitor" 3; "~*MarkMonitor" 3;
"~*MarkWatch" 3; "~*MarkWatch" 3;
"~*masscan" 3;
"~*Mass\ Downloader" 3; "~*Mass\ Downloader" 3;
"~*Masscan" 3;
"~*Mata\ Hari" 3; "~*Mata\ Hari" 3;
"~*Maxthon" 3;
"~*Meanpathbot" 3;
"~*MegaIndex.ru" 3; "~*MegaIndex.ru" 3;
"~*Metauri" 3; "~*Metauri" 3;
"~*MFC_Tear_Sample" 3;
"~*Microsoft\ Data\ Access" 3; "~*Microsoft\ Data\ Access" 3;
"~*Microsoft\ URL\ Control" 3; "~*Microsoft\ URL\ Control" 3;
"~*MIDown\ tool" 3; "~*MIDown\ tool" 3;
"~*MIIxpc" 3; "~*MIIxpc" 3;
"~*Mister\ PiX" 3; "~*Mister\ PiX" 3;
"~*MJ12bot" 3; "~*MJ12bot" 3;
"~*mojeek" 3; "~*Mojeek" 3;
"~*MSIECrawler" 3;
"~*MSIE\ 6.0" 3;
"~*MS\ Web\ Services\ Client\ Protocol" 3; "~*MS\ Web\ Services\ Client\ Protocol" 3;
"~*msrabot" 3; "~*MSFrontPage" 3;
"~*musobot" 3; "~*MSIE\ 6.0" 3;
"~*MSIECrawler" 3;
"~*Msrabot" 3;
"~*Musobot" 3;
"~*Name\ Intelligence" 3; "~*Name\ Intelligence" 3;
"~*Nameprotect" 3; "~*Nameprotect" 3;
"~*Navroad" 3; "~*Navroad" 3;
"~*NearSite" 3; "~*NearSite" 3;
"~*needle" 3; "~*Needle" 3;
"~*nessus" 3; "~*Nessus" 3;
"~*Net\ Vampire" 3;
"~*NetAnts" 3; "~*NetAnts" 3;
"~*Netcraft" 3; "~*Netcraft" 3;
"~*netEstate\ NE\ Crawler" 3; "~*netEstate\ NE\ Crawler" 3;
"~*NetLyzer" 3;
"~*NetMechanic" 3; "~*NetMechanic" 3;
"~*NetSpider" 3; "~*NetSpider" 3;
"~*nettrack" 3; "~*Nettrack" 3;
"~*Net\ Vampire" 3;
"~*NetZIP" 3; "~*NetZIP" 3;
"~*NextGenSearchBot" 3; "~*NextGenSearchBot" 3;
"~*nibbler" 3; "~*Nibbler" 3;
"~*NICErsPRO" 3; "~*NICErsPRO" 3;
"~*niki-bot" 3; "~*Niki-bot" 3;
"~*nikto" 3; "~*Nikto" 3;
"~*NimbleCrawler" 3; "~*NimbleCrawler" 3;
"~*Ninja" 3; "~*Ninja" 3;
"~*Nmap" 3; "~*Nmap" 3;
@ -439,70 +456,83 @@ map $http_user_agent $bad_bot {
"~*Octopus" 3; "~*Octopus" 3;
"~*Offline\ Explorer" 3; "~*Offline\ Explorer" 3;
"~*Offline\ Navigator" 3; "~*Offline\ Navigator" 3;
"~*omgilibot" 3; "~*Omgilibot" 3;
"~*OpenLinkProfiler" 3;
"~*Openfind" 3; "~*Openfind" 3;
"~*OpenLinkProfiler" 3;
"~*Openvas" 3; "~*Openvas" 3;
"~*OrangeBot" 3; "~*OrangeBot" 3;
"~*OrangeSpider" 3; "~*OrangeSpider" 3;
"~*OutfoxBot" 3; "~*OutfoxBot" 3;
"~*PageAnalyzer" 3;
"~*Page\ Analyzer" 3; "~*Page\ Analyzer" 3;
"~*PageGrabber" 3;
"~*Page\ Grabber" 3; "~*Page\ Grabber" 3;
"~*page\ scorer" 3; "~*page\ scorer" 3;
"~*PageAnalyzer" 3;
"~*PageGrabber" 3;
"~*PageScorer" 3;
"~*Panscient" 3;
"~*Papa\ Foto" 3; "~*Papa\ Foto" 3;
"~*pavuk" 3; "~*Pavuk" 3;
"~*pcBrowser" 3; "~*pcBrowser" 3;
"~*PECL::HTTP" 3; "~*PECL::HTTP" 3;
"~*PeoplePal" 3;
"~*PHPCrawl" 3;
"~*Pi-Monster" 3;
"~*Picscout" 3; "~*Picscout" 3;
"~*Picsearch" 3; "~*Picsearch" 3;
"~*pimonster" 3; "~*Pimonster" 3;
"~*Pi-Monster" 3;
"~*Pixray" 3; "~*Pixray" 3;
"~*PleaseCrawl" 3;
"~*Pockey" 3; "~*Pockey" 3;
"~*POE-Component-Client-HTTP" 3; "~*POE-Component-Client-HTTP" 3;
"~*probethenet" 3; "~*Probethenet" 3;
"~*ProPowerBot" 3; "~*ProPowerBot" 3;
"~*ProWebWalker" 3; "~*ProWebWalker" 3;
"~*proximic" 3; "~*Proximic" 3;
"~*psbot" 3; "~*Psbot" 3;
"~*Pump" 3; "~*Pump" 3;
"~*PyCurl" 3; "~*PyCurl" 3;
"~*QueryN\ Metasearch" 3; "~*QueryN\ Metasearch" 3;
"~*Qwantify" 3;
"~*RankActiveLinkBot" 3; "~*RankActiveLinkBot" 3;
"~*Rankivabot" 3; "~*Rankivabot" 3;
"~*RealDownload" 3; "~*RealDownload" 3;
"~*Reaper" 3; "~*Reaper" 3;
"~*Ripper" 3;
"~*Recorder" 3; "~*Recorder" 3;
"~*RedesScrapy" 3;
"~*ReGet" 3; "~*ReGet" 3;
"~*RepoMonkey" 3; "~*RepoMonkey" 3;
"~*Ripper" 3;
"~*RocketCrawler" 3;
"~*Rogerbot" 3; "~*Rogerbot" 3;
"~*SBIder" 3;
"~*ScanAlert" 3; "~*ScanAlert" 3;
"~*scanbot" 3; "~*Scanbot" 3;
"~*Scrapy" 3; "~*Scrapy" 3;
"~*Screaming\ Frog\ SEO\ Spider" 3; "~*Screaming\ Frog\ SEO\ Spider" 3;
"~*ScreenerBot" 3; "~*ScreenerBot" 3;
"~*searchestate" 3; "~*SeaMonkey" 3;
"~*Searchestate" 3;
"~*SearchmetricsBot" 3; "~*SearchmetricsBot" 3;
"~*Semrush" 3; "~*Semrush" 3;
"~*SemrushBot" 3; "~*SemrushBot" 3;
"~*SEOkicks" 3; "~*SEOkicks" 3;
"~*SEOkicks-Robot" 3; "~*SEOkicks-Robot" 3;
"~*SEOlyticsCrawler" 3; "~*SEOlyticsCrawler" 3;
"~*seomoz" 3; "~*Seomoz" 3;
"~*SEOprofiler" 3;
"~*SEOstats" 3; "~*SEOstats" 3;
"~*SISTRIX" 3; "~*SeznamBot" 3;
"~*Siphon" 3; "~*Siphon" 3;
"~*sitebeam" 3; "~*SISTRIX" 3;
"~*SISTRIX\ Crawler" 3;
"~*Site\ Sucker" 3;
"~*Sitebeam" 3;
"~*SiteExplorer" 3; "~*SiteExplorer" 3;
"~*Siteimprove" 3; "~*Siteimprove" 3;
"~*SiteLockSpider" 3; "~*SiteLockSpider" 3;
"~*SiteSucker" 3;
"~*Site\ Sucker" 3;
"~*SiteSnagger" 3; "~*SiteSnagger" 3;
"~*sitevigil" 3; "~*SiteSucker" 3;
"~*Sitevigil" 3;
"~*Slackbot-LinkExpanding" 3; "~*Slackbot-LinkExpanding" 3;
"~*SlySearch" 3; "~*SlySearch" 3;
"~*SmartDownload" 3; "~*SmartDownload" 3;
@ -513,14 +543,16 @@ map $http_user_agent $bad_bot {
"~*Sogou\ web\ spider" 3; "~*Sogou\ web\ spider" 3;
"~*Sosospider" 3; "~*Sosospider" 3;
"~*SpaceBison" 3; "~*SpaceBison" 3;
"~*spammen" 3; "~*Spammen" 3;
"~*spanner" 3;
"~*SpankBot" 3; "~*SpankBot" 3;
"~*Spanner" 3;
"~*Spbot" 3;
"~*Spinn3r" 3; "~*Spinn3r" 3;
"~*spbot" 3; "~*SputnikBot" 3;
"~*sqlmap" 3; "~*Sqlmap" 3;
"~*Sqlworm" 3; "~*Sqlworm" 3;
"~*Sqworm" 3; "~*Sqworm" 3;
"~*Steeler" 3;
"~*Stripper" 3; "~*Stripper" 3;
"~*Sucker" 3; "~*Sucker" 3;
"~*Sucuri" 3; "~*Sucuri" 3;
@ -528,10 +560,10 @@ map $http_user_agent $bad_bot {
"~*SuperHTTP" 3; "~*SuperHTTP" 3;
"~*Surfbot" 3; "~*Surfbot" 3;
"~*SurveyBot" 3; "~*SurveyBot" 3;
"~*suzuran" 3; "~*Suzuran" 3;
"~*Swiftbot" 3; "~*Swiftbot" 3;
"~*SISTRIX\ Crawler" 3;
"~*Szukacz" 3; "~*Szukacz" 3;
"~*T0PHackTeam" 3;
"~*T8Abot" 3; "~*T8Abot" 3;
"~*tAkeOut" 3; "~*tAkeOut" 3;
"~*Teleport" 3; "~*Teleport" 3;
@ -543,91 +575,102 @@ map $http_user_agent $bad_bot {
"~*TheNomad" 3; "~*TheNomad" 3;
"~*TightTwatBot" 3; "~*TightTwatBot" 3;
"~*Titan" 3; "~*Titan" 3;
"~*T0PHackTeam" 3; "~*Toata" 3;
"~*Toweyabot" 3; "~*Toweyabot" 3;
"~*Trendictionbot" 3; "~*Trendictionbot" 3;
"~*True_Robot" 3; "~*True_Robot" 3;
"~*Turingos" 3; "~*Turingos" 3;
"~*TurnitinBot" 3;
"~*Turnitin\ Robot" 3;
"~*Turnitin\ Bot" 3; "~*Turnitin\ Bot" 3;
"~*Turnitin\ Robot" 3;
"~*TurnitinBot" 3;
"~*TwengaBot" 3;
"~*Twice" 3; "~*Twice" 3;
"~*Typhoeus" 3;
"~*UptimeRobot/2.0" 3; "~*UptimeRobot/2.0" 3;
"~*URLy.Warning" 3; "~*URLy.Warning" 3;
"~*URLy\ Warning" 3; "~*URLy\ Warning" 3;
"~*Vacuum" 3; "~*Vacuum" 3;
"~*Vagabondo" 3;
"~*VB\ Project" 3; "~*VB\ Project" 3;
"~*VCI" 3; "~*VCI" 3;
"~*VeriCiteCrawler" 3; "~*VeriCiteCrawler" 3;
"~*VidibleScraper" 3; "~*VidibleScraper" 3;
"~*VoidEYE" 3; "~*VoidEYE" 3;
"~*Voil" 3; "~*Voil" 3;
"~*Voltron" 3;
"~*WallpapersHD" 3; "~*WallpapersHD" 3;
"~*WASALive-Bot" 3; "~*WASALive-Bot" 3;
"~*WBSearchBot" 3; "~*WBSearchBot" 3;
"~*Web\ Auto" 3;
"~*Web\ Bandit" 3;
"~*Web\ Collage" 3;
"~*Web\ Copier" 3;
"~*Web\ Enhancer" 3;
"~*Web\ Fetch" 3;
"~*Web\ Fuck" 3;
"~*Web\ Image\ Collector" 3;
"~*Web\ Pix" 3;
"~*Web\ Reaper" 3;
"~*Web\ Sauger" 3;
"~*Web\ Stripper" 3;
"~*Web\ Sucker" 3;
"~*Web\ Whacker" 3;
"~*Webalta" 3;
"~*WebAuto" 3; "~*WebAuto" 3;
"~*WebBandit" 3; "~*WebBandit" 3;
"~*Web\ Bandit" 3; "~*WebCollage" 3;
"~*WebCopier" 3; "~*WebCopier" 3;
"~*Web\ Copier" 3;
"~*WEBDAV" 3; "~*WEBDAV" 3;
"~*WEBDAV\ Client" 3; "~*WEBDAV\ Client" 3;
"~*WebEnhancer" 3; "~*WebEnhancer" 3;
"~*Web\ Enhancer" 3;
"~*WebFetch" 3; "~*WebFetch" 3;
"~*Web\ Fetch" 3;
"~*WebFuck" 3; "~*WebFuck" 3;
"~*Web\ Fuck" 3; "~*WebImageCollector" 3;
"~*WebGo\ IS" 3; "~*WebGo\ IS" 3;
"~*Web\ Image\ Collector" 3;
"~*WebLeacher" 3; "~*WebLeacher" 3;
"~*WebmasterWorldForumBot" 3; "~*WebmasterWorldForumBot" 3;
"~*WebPix" 3; "~*WebPix" 3;
"~*Web\ Pix" 3;
"~*WebReaper" 3; "~*WebReaper" 3;
"~*Web\ Reaper" 3;
"~*WebSauger" 3; "~*WebSauger" 3;
"~*Web\ Sauger" 3; "~*Webshag" 3;
"~*webshag" 3;
"~*WebsiteExtractor" 3;
"~*Website\ Extractor" 3; "~*Website\ Extractor" 3;
"~*WebsiteQuester" 3;
"~*Website\ Quester" 3; "~*Website\ Quester" 3;
"~*WebsiteExtractor" 3;
"~*WebsiteQuester" 3;
"~*Webster" 3; "~*Webster" 3;
"~*WebStripper" 3; "~*WebStripper" 3;
"~*Web\ Stripper" 3;
"~*WebSucker" 3; "~*WebSucker" 3;
"~*Web\ Sucker" 3;
"~*WebWhacker" 3; "~*WebWhacker" 3;
"~*Web\ Whacker" 3;
"~*WebZIP" 3; "~*WebZIP" 3;
"~*WeSEE" 3; "~*WeSEE" 3;
"~*Whack" 3; "~*Whack" 3;
"~*Whacker" 3; "~*Whacker" 3;
"~*whatweb" 3; "~*Whatweb" 3;
"~*Widow" 3; "~*Widow" 3;
"~*WinHTTrack" 3; "~*WinHTTrack" 3;
"~*WiseGuys\ Robot" 3; "~*WiseGuys\ Robot" 3;
"~*WISENutbot" 3; "~*WISENutbot" 3;
"~*wonderbot" 3; "~*Wonderbot" 3;
"~*woobot" 3; "~*Woobot" 3;
"~*Wotbox" 3; "~*Wotbox" 3;
"~*wprecon" 3; "~*Wprecon" 3;
"~*WWWOFFLE" 3;
"~*WWW::Mechanize" 3;
"~*WWW-Collector-E" 3; "~*WWW-Collector-E" 3;
"~*WWW-Mechanize" 3;
"~*WWW::Mechanize" 3;
"~*WWWOFFLE" 3;
"~*Xaldon_WebSpider" 3; "~*Xaldon_WebSpider" 3;
"~*Xaldon\ WebSpider" 3; "~*Xaldon\ WebSpider" 3;
"~*Xenu" 3; "~*Xenu" 3;
"~*Y!J-ASR" 3; "~*Y!J-ASR" 3;
"~*YoudaoBot" 3; "~*YoudaoBot" 3;
"~*Zade" 3; "~*Zade" 3;
"~*Zermelo" 3;
"~*Zeus" 3; "~*Zeus" 3;
"~*zgrab" 3; "~*Zgrab" 3;
"~*zitebot" 3; "~*Zitebot" 3;
"~*ZmEu" 3; "~*ZmEu" 3;
"~*ZumBot" 3; "~*ZumBot" 3;
"~*Zyborg" 3; "~*ZyBorg" 3;
# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###
} }

426
robots.txt/robots.txt
View file

@ -1,28 +1,20 @@
User-agent: Acunetix User-agent: 360Spider
Disallow:/
User-agent: FHscan
Disallow:/
User-agent: masscan
Disallow:/
User-agent: probethenet
Disallow:/
User-agent: scanbot
Disallow:/ Disallow:/
User-agent: 80legs User-agent: 80legs
Disallow:/ Disallow:/
User-agent: 360Spider User-agent: Abonti
Disallow:/ Disallow:/
User-agent: Aboundex User-agent: Aboundex
Disallow:/ Disallow:/
User-agent: Abonti User-agent: Acunetix
Disallow:/ Disallow:/
User-agent: AhrefsBot User-agent: AhrefsBot
Disallow:/ Disallow:/
User-agent: AIBOT User-agent: AIBOT
Disallow:/ Disallow:/
User-agent: aipbot User-agent: AiHitBot
Disallow:/ Disallow:/
User-agent: aiHitBot User-agent: Aipbot
Disallow:/ Disallow:/
User-agent: Alexibot User-agent: Alexibot
Disallow:/ Disallow:/
@ -34,11 +26,11 @@ User-agent: Anarchie
Disallow:/ Disallow:/
User-agent: Apexoo User-agent: Apexoo
Disallow:/ Disallow:/
User-agent: asterias
Disallow:/
User-agent: ASPSeek User-agent: ASPSeek
Disallow:/ Disallow:/
User-agent: attach User-agent: Asterias
Disallow:/
User-agent: Attach
Disallow:/ Disallow:/
User-agent: autoemailspider User-agent: autoemailspider
Disallow:/ Disallow:/
@ -60,6 +52,8 @@ User-agent: BBBike
Disallow:/ Disallow:/
User-agent: BDFetch User-agent: BDFetch
Disallow:/ Disallow:/
User-agent: BetaBot
Disallow:/
User-agent: Bigfoot User-agent: Bigfoot
Disallow:/ Disallow:/
User-agent: Bitacle User-agent: Bitacle
@ -70,11 +64,13 @@ User-agent: BlackWidow
Disallow:/ Disallow:/
User-agent: BLEXBot User-agent: BLEXBot
Disallow:/ Disallow:/
User-agent: BlowFish
Disallow:/
User-agent: Blow User-agent: Blow
Disallow:/ Disallow:/
User-agent: boardreader User-agent: BlowFish
Disallow:/
User-agent: Boardreader
Disallow:/
User-agent: Bolt
Disallow:/ Disallow:/
User-agent: BotALot User-agent: BotALot
Disallow:/ Disallow:/
@ -82,6 +78,8 @@ User-agent: Brandprotect
Disallow:/ Disallow:/
User-agent: Brandwatch User-agent: Brandwatch
Disallow:/ Disallow:/
User-agent: Bubing
Disallow:/
User-agent: Buddy User-agent: Buddy
Disallow:/ Disallow:/
User-agent: BuiltBotTough User-agent: BuiltBotTough
@ -94,7 +92,9 @@ User-agent: BunnySlippers
Disallow:/ Disallow:/
User-agent: BuzzSumo User-agent: BuzzSumo
Disallow:/ Disallow:/
User-agent: calculon spider User-agent: Calculon
Disallow:/
User-agent: CazoodleBot
Disallow:/ Disallow:/
User-agent: CCBot User-agent: CCBot
Disallow:/ Disallow:/
@ -106,7 +106,7 @@ User-agent: CherryPicker
Disallow:/ Disallow:/
User-agent: ChinaClaw User-agent: ChinaClaw
Disallow:/ Disallow:/
User-agent: chlooe User-agent: Chlooe
Disallow:/ Disallow:/
User-agent: Claritybot User-agent: Claritybot
Disallow:/ Disallow:/
@ -124,7 +124,7 @@ User-agent: Copyscape
Disallow:/ Disallow:/
User-agent: Cosmos User-agent: Cosmos
Disallow:/ Disallow:/
User-agent: craftbot@yahoo.com User-agent: Craftbot
Disallow:/ Disallow:/
User-agent: CrazyWebCrawler User-agent: CrazyWebCrawler
Disallow:/ Disallow:/
@ -132,25 +132,27 @@ User-agent: Crescent
Disallow:/ Disallow:/
User-agent: CSHttp User-agent: CSHttp
Disallow:/ Disallow:/
User-agent: Curious
Disallow:/
User-agent: Custo User-agent: Custo
Disallow:/ Disallow:/
User-agent: DBLBot User-agent: DBLBot
Disallow:/ Disallow:/
User-agent: Demon User-agent: Demon
Disallow:/ Disallow:/
User-agent: deusu User-agent: Deusu
Disallow:/ Disallow:/
User-agent: Devil User-agent: Devil
Disallow:/ Disallow:/
User-agent: Disco
Disallow:/
User-agent: DIIbot User-agent: DIIbot
Disallow:/ Disallow:/
User-agent: dirbuster User-agent: Dirbuster
Disallow:/ Disallow:/
User-agent: discobot User-agent: Disco
Disallow:/ Disallow:/
User-agent: discoverybot User-agent: Discobot
Disallow:/
User-agent: Discoverybot
Disallow:/ Disallow:/
User-agent: DittoSpyder User-agent: DittoSpyder
Disallow:/ Disallow:/
@ -158,7 +160,7 @@ User-agent: DomainAppender
Disallow:/ Disallow:/
User-agent: DomainSigmaCrawler User-agent: DomainSigmaCrawler
Disallow:/ Disallow:/
User-agent: dotbot User-agent: Dotbot
Disallow:/ Disallow:/
User-agent: Download Demon User-agent: Download Demon
Disallow:/ Disallow:/
@ -166,7 +168,7 @@ User-agent: Download Devil
Disallow:/ Disallow:/
User-agent: Download Wonder User-agent: Download Wonder
Disallow:/ Disallow:/
User-agent: dragonfly User-agent: Dragonfly
Disallow:/ Disallow:/
User-agent: Drip User-agent: Drip
Disallow:/ Disallow:/
@ -174,28 +176,24 @@ User-agent: DTS Agent
Disallow:/ Disallow:/
User-agent: EasyDL User-agent: EasyDL
Disallow:/ Disallow:/
User-agent: ebingbong User-agent: Ebingbong
Disallow:/ Disallow:/
User-agent: eCatch User-agent: eCatch
Disallow:/ Disallow:/
User-agent: ECCP/1.0 User-agent: ECCP/1.0
Disallow:/ Disallow:/
User-agent: Ecxi
Disallow:/
User-agent: EirGrabber User-agent: EirGrabber
Disallow:/ Disallow:/
User-agent: EMail Collector User-agent: EMail Collector
Disallow:/ Disallow:/
User-agent: EmailCollector
Disallow:/
User-agent: EMail Extractor User-agent: EMail Extractor
Disallow:/ Disallow:/
User-agent: EMail Siphon User-agent: EMail Siphon
Disallow:/ Disallow:/
User-agent: EmailSiphon
Disallow:/
User-agent: EMail Wolf User-agent: EMail Wolf
Disallow:/ Disallow:/
User-agent: EmailWolf
Disallow:/
User-agent: EroCrawler User-agent: EroCrawler
Disallow:/ Disallow:/
User-agent: Evil User-agent: Evil
@ -210,17 +208,21 @@ User-agent: ExtractorPro
Disallow:/ Disallow:/
User-agent: EyeNetIE User-agent: EyeNetIE
Disallow:/ Disallow:/
User-agent: ezooms User-agent: Ezooms
Disallow:/ Disallow:/
User-agent: fimap User-agent: FHscan
Disallow:/
User-agent: Fimap
Disallow:/
User-agent: Findxbot
Disallow:/ Disallow:/
User-agent: FlashGet User-agent: FlashGet
Disallow:/ Disallow:/
User-agent: flunky User-agent: Flunky
Disallow:/ Disallow:/
User-agent: Foobot User-agent: Foobot
Disallow:/ Disallow:/
User-agent: freeuploader User-agent: Freeuploader
Disallow:/ Disallow:/
User-agent: FrontPage User-agent: FrontPage
Disallow:/ Disallow:/
@ -228,34 +230,40 @@ User-agent: GalaxyBot
Disallow:/ Disallow:/
User-agent: Genieo User-agent: Genieo
Disallow:/ Disallow:/
User-agent: getintent User-agent: Getintent
Disallow:/ Disallow:/
User-agent: GetRight User-agent: GetRight
Disallow:/ Disallow:/
User-agent: GetWeb User-agent: GetWeb
Disallow:/ Disallow:/
User-agent: gigablast User-agent: Gigablast
Disallow:/ Disallow:/
User-agent: gigabot User-agent: Gigabot
Disallow:/ Disallow:/
User-agent: Go-Ahead-Got-It User-agent: Go-Ahead-Got-It
Disallow:/ Disallow:/
User-agent: gotit
Disallow:/
User-agent: Go!Zilla User-agent: Go!Zilla
Disallow:/ Disallow:/
User-agent: Gotit
Disallow:/
User-agent: GoZilla
Disallow:/
User-agent: Grabber User-agent: Grabber
Disallow:/ Disallow:/
User-agent: GrabNet User-agent: GrabNet
Disallow:/ Disallow:/
User-agent: Grafula User-agent: Grafula
Disallow:/ Disallow:/
User-agent: grapeFX User-agent: GrapeFX
Disallow:/ Disallow:/
User-agent: GrapeshotCrawler User-agent: GrapeshotCrawler
Disallow:/ Disallow:/
User-agent: GridBot User-agent: GridBot
Disallow:/ Disallow:/
User-agent: GT::WWW
Disallow:/
User-agent: GuzzleHttp
Disallow:/
User-agent: HaosouSpider User-agent: HaosouSpider
Disallow:/ Disallow:/
User-agent: Harvest User-agent: Harvest
@ -264,24 +272,26 @@ User-agent: Havij
Disallow:/ Disallow:/
User-agent: Heritrix User-agent: Heritrix
Disallow:/ Disallow:/
User-agent: hloader User-agent: Hloader
Disallow:/ Disallow:/
User-agent: HMView User-agent: HMView
Disallow:/ Disallow:/
User-agent: HTTrack
Disallow:/
User-agent: HTMLparser User-agent: HTMLparser
Disallow:/ Disallow:/
User-agent: Huaweisymantecspider User-agent: HTTP::Lite
Disallow:/ Disallow:/
User-agent: humanlinks User-agent: HTTrack
Disallow:/ Disallow:/
User-agent: iblog User-agent: Humanlinks
Disallow:/
User-agent: Iblog
Disallow:/
User-agent: Id-search
Disallow:/
User-agent: IDBot
Disallow:/ Disallow:/
User-agent: IlseBot User-agent: IlseBot
Disallow:/ Disallow:/
User-agent: imagefetch
Disallow:/
User-agent: Image Fetch User-agent: Image Fetch
Disallow:/ Disallow:/
User-agent: Image Stripper User-agent: Image Stripper
@ -296,28 +306,30 @@ User-agent: InfoTekies
Disallow:/ Disallow:/
User-agent: Intelliseek User-agent: Intelliseek
Disallow:/ Disallow:/
User-agent: InterGET
Disallow:/
User-agent: Internet Ninja User-agent: Internet Ninja
Disallow:/ Disallow:/
User-agent: InternetSeer
Disallow:/
User-agent: internetVista monitor User-agent: internetVista monitor
Disallow:/ Disallow:/
User-agent: InterGET
Disallow:/
User-agent: Iria User-agent: Iria
Disallow:/ Disallow:/
User-agent: iskanie User-agent: IRLbot
Disallow:/
User-agent: Iskanie
Disallow:/ Disallow:/
User-agent: JamesBOT User-agent: JamesBOT
Disallow:/ Disallow:/
User-agent: Jbrofuzz
Disallow:/
User-agent: JennyBot User-agent: JennyBot
Disallow:/ Disallow:/
User-agent: JetCar User-agent: JetCar
Disallow:/ Disallow:/
User-agent: jbrofuzz
Disallow:/
User-agent: JikeSpider User-agent: JikeSpider
Disallow:/ Disallow:/
User-agent: JOC
Disallow:/
User-agent: JOC Web Spider User-agent: JOC Web Spider
Disallow:/ Disallow:/
User-agent: Joomla User-agent: Joomla
@ -330,11 +342,11 @@ User-agent: Kenjin Spider
Disallow:/ Disallow:/
User-agent: Keyword Density User-agent: Keyword Density
Disallow:/ Disallow:/
User-agent: kraken User-agent: Kraken
Disallow:/ Disallow:/
User-agent: lanshanbot User-agent: Lanshanbot
Disallow:/ Disallow:/
User-agent: larbin User-agent: Larbin
Disallow:/ Disallow:/
User-agent: LeechFTP User-agent: LeechFTP
Disallow:/ Disallow:/
@ -342,53 +354,59 @@ User-agent: LeechGet
Disallow:/ Disallow:/
User-agent: LexiBot User-agent: LexiBot
Disallow:/ Disallow:/
User-agent: lftp User-agent: Lftp
Disallow:/ Disallow:/
User-agent: libcurl User-agent: LibWeb
Disallow:/ Disallow:/
User-agent: libWeb User-agent: Libwhisker
Disallow:/ Disallow:/
User-agent: libwhisker User-agent: Lightspeedsystems
Disallow:/ Disallow:/
User-agent: likse User-agent: Likse
Disallow:/ Disallow:/
User-agent: linkdexbot User-agent: Linkdexbot
Disallow:/
User-agent: LinkpadBot
Disallow:/ Disallow:/
User-agent: LinkextractorPro User-agent: LinkextractorPro
Disallow:/ Disallow:/
User-agent: LinkpadBot
Disallow:/
User-agent: LinkScan User-agent: LinkScan
Disallow:/ Disallow:/
User-agent: LinqiaMetadataDownloaderBot User-agent: LinksManager
Disallow:/
User-agent: LinqiaScrapeBot
Disallow:/
User-agent: LinqiaRSSBot
Disallow:/
User-agent: lmspider
Disallow:/
User-agent: LNSpiderguy
Disallow:/ Disallow:/
User-agent: LinkWalker User-agent: LinkWalker
Disallow:/ Disallow:/
User-agent: ltx71 User-agent: LinqiaMetadataDownloaderBot
Disallow:/
User-agent: LinqiaRSSBot
Disallow:/
User-agent: LinqiaScrapeBot
Disallow:/
User-agent: Lipperhey
Disallow:/
User-agent: Litemage_walker
Disallow:/
User-agent: Lmspider
Disallow:/
User-agent: LNSpiderguy
Disallow:/
User-agent: Ltx71
Disallow:/ Disallow:/
User-agent: lwp-request User-agent: lwp-request
Disallow:/ Disallow:/
User-agent: lwp-trivial
Disallow:/
User-agent: LWP::Simple User-agent: LWP::Simple
Disallow:/ Disallow:/
User-agent: magpie-crawler
Disallow:/
User-agent: Magnet
Disallow:/
User-agent: Mag-Net User-agent: Mag-Net
Disallow:/ Disallow:/
User-agent: mail.ru User-agent: Magnet
Disallow:/ Disallow:/
User-agent: Mail.RU_Bot User-agent: magpie-crawler
Disallow:/ Disallow:/
User-agent: majestic12 User-agent: Mail.ru
Disallow:/
User-agent: Majestic12
Disallow:/ Disallow:/
User-agent: MarkMonitor User-agent: MarkMonitor
Disallow:/ Disallow:/
@ -396,12 +414,20 @@ User-agent: MarkWatch
Disallow:/ Disallow:/
User-agent: Mass Downloader User-agent: Mass Downloader
Disallow:/ Disallow:/
User-agent: Masscan
Disallow:/
User-agent: Mata Hari User-agent: Mata Hari
Disallow:/ Disallow:/
User-agent: Maxthon
Disallow:/
User-agent: Meanpathbot
Disallow:/
User-agent: MegaIndex.ru User-agent: MegaIndex.ru
Disallow:/ Disallow:/
User-agent: Metauri User-agent: Metauri
Disallow:/ Disallow:/
User-agent: MFC_Tear_Sample
Disallow:/
User-agent: Microsoft Data Access User-agent: Microsoft Data Access
Disallow:/ Disallow:/
User-agent: Microsoft URL Control User-agent: Microsoft URL Control
@ -414,17 +440,19 @@ User-agent: Mister PiX
Disallow:/ Disallow:/
User-agent: MJ12bot User-agent: MJ12bot
Disallow:/ Disallow:/
User-agent: mojeek User-agent: Mojeek
Disallow:/
User-agent: MSIECrawler
Disallow:/
User-agent: MSIE 6.0
Disallow:/ Disallow:/
User-agent: MS Web Services Client Protocol User-agent: MS Web Services Client Protocol
Disallow:/ Disallow:/
User-agent: msrabot User-agent: MSFrontPage
Disallow:/ Disallow:/
User-agent: musobot User-agent: MSIE 6.0
Disallow:/
User-agent: MSIECrawler
Disallow:/
User-agent: Msrabot
Disallow:/
User-agent: Musobot
Disallow:/ Disallow:/
User-agent: Name Intelligence User-agent: Name Intelligence
Disallow:/ Disallow:/
@ -434,9 +462,11 @@ User-agent: Navroad
Disallow:/ Disallow:/
User-agent: NearSite User-agent: NearSite
Disallow:/ Disallow:/
User-agent: needle User-agent: Needle
Disallow:/ Disallow:/
User-agent: nessus User-agent: Nessus
Disallow:/
User-agent: Net Vampire
Disallow:/ Disallow:/
User-agent: NetAnts User-agent: NetAnts
Disallow:/ Disallow:/
@ -444,25 +474,25 @@ User-agent: Netcraft
Disallow:/ Disallow:/
User-agent: netEstate NE Crawler User-agent: netEstate NE Crawler
Disallow:/ Disallow:/
User-agent: NetLyzer
Disallow:/
User-agent: NetMechanic User-agent: NetMechanic
Disallow:/ Disallow:/
User-agent: NetSpider User-agent: NetSpider
Disallow:/ Disallow:/
User-agent: nettrack User-agent: Nettrack
Disallow:/
User-agent: Net Vampire
Disallow:/ Disallow:/
User-agent: NetZIP User-agent: NetZIP
Disallow:/ Disallow:/
User-agent: NextGenSearchBot User-agent: NextGenSearchBot
Disallow:/ Disallow:/
User-agent: nibbler User-agent: Nibbler
Disallow:/ Disallow:/
User-agent: NICErsPRO User-agent: NICErsPRO
Disallow:/ Disallow:/
User-agent: niki-bot User-agent: Niki-bot
Disallow:/ Disallow:/
User-agent: nikto User-agent: Nikto
Disallow:/ Disallow:/
User-agent: NimbleCrawler User-agent: NimbleCrawler
Disallow:/ Disallow:/
@ -480,12 +510,12 @@ User-agent: Offline Explorer
Disallow:/ Disallow:/
User-agent: Offline Navigator User-agent: Offline Navigator
Disallow:/ Disallow:/
User-agent: omgilibot User-agent: Omgilibot
Disallow:/
User-agent: OpenLinkProfiler
Disallow:/ Disallow:/
User-agent: Openfind User-agent: Openfind
Disallow:/ Disallow:/
User-agent: OpenLinkProfiler
Disallow:/
User-agent: Openvas User-agent: Openvas
Disallow:/ Disallow:/
User-agent: OrangeBot User-agent: OrangeBot
@ -496,43 +526,55 @@ User-agent: OutfoxBot
Disallow:/ Disallow:/
User-agent: Page Analyzer User-agent: Page Analyzer
Disallow:/ Disallow:/
User-agent: Page Analyzer
Disallow:/
User-agent: PageGrabber
Disallow:/
User-agent: Page Grabber User-agent: Page Grabber
Disallow:/ Disallow:/
User-agent: page scorer User-agent: page scorer
Disallow:/ Disallow:/
User-agent: PageAnalyzer
Disallow:/
User-agent: PageGrabber
Disallow:/
User-agent: PageScorer
Disallow:/
User-agent: Panscient
Disallow:/
User-agent: Papa Foto User-agent: Papa Foto
Disallow:/ Disallow:/
User-agent: pavuk User-agent: Pavuk
Disallow:/ Disallow:/
User-agent: pcBrowser User-agent: pcBrowser
Disallow:/ Disallow:/
User-agent: PECL::HTTP User-agent: PECL::HTTP
Disallow:/ Disallow:/
User-agent: PeoplePal
Disallow:/
User-agent: PHPCrawl
Disallow:/
User-agent: Pi-Monster
Disallow:/
User-agent: Picscout User-agent: Picscout
Disallow:/ Disallow:/
User-agent: Picsearch User-agent: Picsearch
Disallow:/ Disallow:/
User-agent: pimonster User-agent: Pimonster
Disallow:/
User-agent: Pi-Monster
Disallow:/ Disallow:/
User-agent: Pixray User-agent: Pixray
Disallow:/ Disallow:/
User-agent: PleaseCrawl
Disallow:/
User-agent: Pockey User-agent: Pockey
Disallow:/ Disallow:/
User-agent: POE-Component-Client-HTTP User-agent: POE-Component-Client-HTTP
Disallow:/ Disallow:/
User-agent: Probethenet
Disallow:/
User-agent: ProPowerBot User-agent: ProPowerBot
Disallow:/ Disallow:/
User-agent: ProWebWalker User-agent: ProWebWalker
Disallow:/ Disallow:/
User-agent: proximic User-agent: Proximic
Disallow:/ Disallow:/
User-agent: psbot User-agent: Psbot
Disallow:/ Disallow:/
User-agent: Pump User-agent: Pump
Disallow:/ Disallow:/
@ -540,6 +582,8 @@ User-agent: PyCurl
Disallow:/ Disallow:/
User-agent: QueryN Metasearch User-agent: QueryN Metasearch
Disallow:/ Disallow:/
User-agent: Qwantify
Disallow:/
User-agent: RankActiveLinkBot User-agent: RankActiveLinkBot
Disallow:/ Disallow:/
User-agent: Rankivabot User-agent: Rankivabot
@ -548,25 +592,35 @@ User-agent: RealDownload
Disallow:/ Disallow:/
User-agent: Reaper User-agent: Reaper
Disallow:/ Disallow:/
User-agent: Ripper
Disallow:/
User-agent: Recorder User-agent: Recorder
Disallow:/ Disallow:/
User-agent: RedesScrapy
Disallow:/
User-agent: ReGet User-agent: ReGet
Disallow:/ Disallow:/
User-agent: RepoMonkey User-agent: RepoMonkey
Disallow:/ Disallow:/
User-agent: Ripper
Disallow:/
User-agent: RocketCrawler
Disallow:/
User-agent: Rogerbot User-agent: Rogerbot
Disallow:/ Disallow:/
User-agent: SBIder
Disallow:/
User-agent: ScanAlert User-agent: ScanAlert
Disallow:/ Disallow:/
User-agent: Scanbot
Disallow:/
User-agent: Scrapy User-agent: Scrapy
Disallow:/ Disallow:/
User-agent: Screaming Frog SEO Spider User-agent: Screaming Frog SEO Spider
Disallow:/ Disallow:/
User-agent: ScreenerBot User-agent: ScreenerBot
Disallow:/ Disallow:/
User-agent: searchestate User-agent: SeaMonkey
Disallow:/
User-agent: Searchestate
Disallow:/ Disallow:/
User-agent: SearchmetricsBot User-agent: SearchmetricsBot
Disallow:/ Disallow:/
@ -580,15 +634,23 @@ User-agent: SEOkicks-Robot
Disallow:/ Disallow:/
User-agent: SEOlyticsCrawler User-agent: SEOlyticsCrawler
Disallow:/ Disallow:/
User-agent: seomoz User-agent: Seomoz
Disallow:/
User-agent: SEOprofiler
Disallow:/ Disallow:/
User-agent: SEOstats User-agent: SEOstats
Disallow:/ Disallow:/
User-agent: SISTRIX User-agent: SeznamBot
Disallow:/ Disallow:/
User-agent: Siphon User-agent: Siphon
Disallow:/ Disallow:/
User-agent: sitebeam User-agent: SISTRIX
Disallow:/
User-agent: SISTRIX Crawler
Disallow:/
User-agent: Site Sucker
Disallow:/
User-agent: Sitebeam
Disallow:/ Disallow:/
User-agent: SiteExplorer User-agent: SiteExplorer
Disallow:/ Disallow:/
@ -596,13 +658,11 @@ User-agent: Siteimprove
Disallow:/ Disallow:/
User-agent: SiteLockSpider User-agent: SiteLockSpider
Disallow:/ Disallow:/
User-agent: SiteSucker
Disallow:/
User-agent: Site Sucker
Disallow:/
User-agent: SiteSnagger User-agent: SiteSnagger
Disallow:/ Disallow:/
User-agent: sitevigil User-agent: SiteSucker
Disallow:/
User-agent: Sitevigil
Disallow:/ Disallow:/
User-agent: Slackbot-LinkExpanding User-agent: Slackbot-LinkExpanding
Disallow:/ Disallow:/
@ -624,22 +684,26 @@ User-agent: Sosospider
Disallow:/ Disallow:/
User-agent: SpaceBison User-agent: SpaceBison
Disallow:/ Disallow:/
User-agent: spammen User-agent: Spammen
Disallow:/
User-agent: spanner
Disallow:/ Disallow:/
User-agent: SpankBot User-agent: SpankBot
Disallow:/ Disallow:/
User-agent: Spanner
Disallow:/
User-agent: Spbot
Disallow:/
User-agent: Spinn3r User-agent: Spinn3r
Disallow:/ Disallow:/
User-agent: spbot User-agent: SputnikBot
Disallow:/ Disallow:/
User-agent: sqlmap User-agent: Sqlmap
Disallow:/ Disallow:/
User-agent: Sqlworm User-agent: Sqlworm
Disallow:/ Disallow:/
User-agent: Sqworm User-agent: Sqworm
Disallow:/ Disallow:/
User-agent: Steeler
Disallow:/
User-agent: Stripper User-agent: Stripper
Disallow:/ Disallow:/
User-agent: Sucker User-agent: Sucker
@ -654,14 +718,14 @@ User-agent: Surfbot
Disallow:/ Disallow:/
User-agent: SurveyBot User-agent: SurveyBot
Disallow:/ Disallow:/
User-agent: suzuran User-agent: Suzuran
Disallow:/ Disallow:/
User-agent: Swiftbot User-agent: Swiftbot
Disallow:/ Disallow:/
User-agent: SISTRIX Crawler
Disallow:/
User-agent: Szukacz User-agent: Szukacz
Disallow:/ Disallow:/
User-agent: T0PHackTeam
Disallow:/
User-agent: T8Abot User-agent: T8Abot
Disallow:/ Disallow:/
User-agent: tAkeOut User-agent: tAkeOut
@ -684,7 +748,7 @@ User-agent: TightTwatBot
Disallow:/ Disallow:/
User-agent: Titan User-agent: Titan
Disallow:/ Disallow:/
User-agent: T0PHackTeam User-agent: Toata
Disallow:/ Disallow:/
User-agent: Toweyabot User-agent: Toweyabot
Disallow:/ Disallow:/
@ -700,8 +764,12 @@ User-agent: Turnitin Robot
Disallow:/ Disallow:/
User-agent: TurnitinBot User-agent: TurnitinBot
Disallow:/ Disallow:/
User-agent: TwengaBot
Disallow:/
User-agent: Twice User-agent: Twice
Disallow:/ Disallow:/
User-agent: Typhoeus
Disallow:/
User-agent: UptimeRobot/2.0 User-agent: UptimeRobot/2.0
Disallow:/ Disallow:/
User-agent: URLy.Warning User-agent: URLy.Warning
@ -710,6 +778,8 @@ User-agent: URLy Warning
Disallow:/ Disallow:/
User-agent: Vacuum User-agent: Vacuum
Disallow:/ Disallow:/
User-agent: Vagabondo
Disallow:/
User-agent: VB Project User-agent: VB Project
Disallow:/ Disallow:/
User-agent: VCI User-agent: VCI
@ -722,6 +792,8 @@ User-agent: VoidEYE
Disallow:/ Disallow:/
User-agent: Voil User-agent: Voil
Disallow:/ Disallow:/
User-agent: Voltron
Disallow:/
User-agent: WallpapersHD User-agent: WallpapersHD
Disallow:/ Disallow:/
User-agent: WASALive-Bot User-agent: WASALive-Bot
@ -732,10 +804,38 @@ User-agent: WebAuto
Disallow:/ Disallow:/
User-agent: Web Bandit User-agent: Web Bandit
Disallow:/ Disallow:/
User-agent: Web Bandit User-agent: Web Collage
Disallow:/ Disallow:/
User-agent: Web Copier User-agent: Web Copier
Disallow:/ Disallow:/
User-agent: Web Enhancer
Disallow:/
User-agent: Web Fetch
Disallow:/
User-agent: Web Fuck
Disallow:/
User-agent: Web Image Collector
Disallow:/
User-agent: Web Pix
Disallow:/
User-agent: Web Reaper
Disallow:/
User-agent: Web Sauger
Disallow:/
User-agent: Web Stripper
Disallow:/
User-agent: Web Sucker
Disallow:/
User-agent: Web Whacker
Disallow:/
User-agent: Webalta
Disallow:/
User-agent: WebAuto
Disallow:/
User-agent: WebBandit
Disallow:/
User-agent: WebCollage
Disallow:/
User-agent: WebCopier User-agent: WebCopier
Disallow:/ Disallow:/
User-agent: WEBDAV User-agent: WEBDAV
@ -744,58 +844,40 @@ User-agent: WEBDAV Client
Disallow:/ Disallow:/
User-agent: WebEnhancer User-agent: WebEnhancer
Disallow:/ Disallow:/
User-agent: Web Enhancer
Disallow:/
User-agent: WebFetch
Disallow:/
User-agent: WebFetch User-agent: WebFetch
Disallow:/ Disallow:/
User-agent: WebFuck User-agent: WebFuck
Disallow:/ Disallow:/
User-agent: Web Fuck
Disallow:/
User-agent: WebGo IS
Disallow:/
User-agent: WebImageCollector User-agent: WebImageCollector
Disallow:/ Disallow:/
User-agent: WebGo IS
Disallow:/
User-agent: WebLeacher User-agent: WebLeacher
Disallow:/ Disallow:/
User-agent: WebmasterWorldForumBot User-agent: WebmasterWorldForumBot
Disallow:/ Disallow:/
User-agent: WebPix User-agent: WebPix
Disallow:/ Disallow:/
User-agent: Web Pix
Disallow:/
User-agent: WebReaper
Disallow:/
User-agent: WebReaper User-agent: WebReaper
Disallow:/ Disallow:/
User-agent: WebSauger User-agent: WebSauger
Disallow:/ Disallow:/
User-agent: Web Sauger User-agent: Webshag
Disallow:/
User-agent: webshag
Disallow:/
User-agent: WebsiteExtractor
Disallow:/ Disallow:/
User-agent: Website Extractor User-agent: Website Extractor
Disallow:/ Disallow:/
User-agent: Website Quester User-agent: Website Quester
Disallow:/ Disallow:/
User-agent: WebsiteExtractor
Disallow:/
User-agent: WebsiteQuester User-agent: WebsiteQuester
Disallow:/ Disallow:/
User-agent: Webster User-agent: Webster
Disallow:/ Disallow:/
User-agent: WebStripper User-agent: WebStripper
Disallow:/ Disallow:/
User-agent: Web Stripper
Disallow:/
User-agent: WebSucker User-agent: WebSucker
Disallow:/ Disallow:/
User-agent: Web Sucker
Disallow:/
User-agent: WebWhacker
Disallow:/
User-agent: WebWhacker User-agent: WebWhacker
Disallow:/ Disallow:/
User-agent: WebZIP User-agent: WebZIP
@ -806,7 +888,7 @@ User-agent: Whack
Disallow:/ Disallow:/
User-agent: Whacker User-agent: Whacker
Disallow:/ Disallow:/
User-agent: whatweb User-agent: Whatweb
Disallow:/ Disallow:/
User-agent: Widow User-agent: Widow
Disallow:/ Disallow:/
@ -816,19 +898,21 @@ User-agent: WiseGuys Robot
Disallow:/ Disallow:/
User-agent: WISENutbot User-agent: WISENutbot
Disallow:/ Disallow:/
User-agent: wonderbot User-agent: Wonderbot
Disallow:/ Disallow:/
User-agent: woobot User-agent: Woobot
Disallow:/ Disallow:/
User-agent: Wotbox User-agent: Wotbox
Disallow:/ Disallow:/
User-agent: wprecon User-agent: Wprecon
Disallow:/ Disallow:/
User-agent: WWWOFFLE User-agent: WWW-Collector-E
Disallow:/
User-agent: WWW-Mechanize
Disallow:/ Disallow:/
User-agent: WWW::Mechanize User-agent: WWW::Mechanize
Disallow:/ Disallow:/
User-agent: WWW-Collector-E User-agent: WWWOFFLE
Disallow:/ Disallow:/
User-agent: Xaldon_WebSpider User-agent: Xaldon_WebSpider
Disallow:/ Disallow:/
@ -842,11 +926,13 @@ User-agent: YoudaoBot
Disallow:/ Disallow:/
User-agent: Zade User-agent: Zade
Disallow:/ Disallow:/
User-agent: Zermelo
Disallow:/
User-agent: Zeus User-agent: Zeus
Disallow:/ Disallow:/
User-agent: zgrab User-agent: Zgrab
Disallow:/ Disallow:/
User-agent: zitebot User-agent: Zitebot
Disallow:/ Disallow:/
User-agent: ZmEu User-agent: ZmEu
Disallow:/ Disallow:/