mirror of
https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker.git
synced 2025-09-02 18:50:13 +00:00
New Referers / Bots Added
This commit is contained in:
Mitchell Krog UB1
parent
0ba196ed29
commit
f87b5576f2
2 changed files with 434 additions and 305 deletions
|
|
@ -17,11 +17,11 @@
|
|||
### - https://github.com/oohnoitz/nginx-blacklist
|
||||
|
||||
### Last Updated
|
||||
### Sun Jan 29 11:24:21 SAST 2017
|
||||
### Sun Jan 29 12:03:50 SAST 2017
|
||||
### End Last Updated
|
||||
|
||||
### Generated in
|
||||
### 0.29238986969 seconds
|
||||
### 0.272795915604 seconds
|
||||
### End Generated in
|
||||
|
||||
### Tested on: nginx/1.10.0 (Ubuntu 16.04)
|
||||
|
|
@ -200,23 +200,23 @@ map $http_user_agent $bad_bot {
|
|||
# Known Vulnerability Scanners (now merged into one section)
|
||||
|
||||
# START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###
|
||||
"~*80legs" 3;
|
||||
"~*360Spider" 3;
|
||||
"~*Aboundex" 3;
|
||||
"~*80legs" 3;
|
||||
"~*Abonti" 3;
|
||||
"~*Aboundex" 3;
|
||||
"~*Acunetix" 3;
|
||||
"~*AhrefsBot" 3;
|
||||
"~*AIBOT" 3;
|
||||
"~*aipbot" 3;
|
||||
"~*aiHitBot" 3;
|
||||
"~*AiHitBot" 3;
|
||||
"~*Aipbot" 3;
|
||||
"~*Alexibot" 3;
|
||||
"~*Alligator" 3;
|
||||
"~*AllSubmitter" 3;
|
||||
"~*Anarchie" 3;
|
||||
"~*Apexoo" 3;
|
||||
"~*asterias" 3;
|
||||
"~*ASPSeek" 3;
|
||||
"~*attach" 3;
|
||||
"~*Asterias" 3;
|
||||
"~*Attach" 3;
|
||||
"~*autoemailspider" 3;
|
||||
"~*BackDoorBot" 3;
|
||||
"~*BackStreet" 3;
|
||||
|
|
@ -227,30 +227,34 @@ map $http_user_agent $bad_bot {
|
|||
"~*Battleztar\ Bazinga" 3;
|
||||
"~*BBBike" 3;
|
||||
"~*BDFetch" 3;
|
||||
"~*BetaBot" 3;
|
||||
"~*Bigfoot" 3;
|
||||
"~*Bitacle" 3;
|
||||
"~*Black\ Hole" 3;
|
||||
"~*BlackWidow" 3;
|
||||
"~*BLEXBot" 3;
|
||||
"~*BlowFish" 3;
|
||||
"~*Blow" 3;
|
||||
"~*boardreader" 3;
|
||||
"~*BlowFish" 3;
|
||||
"~*Boardreader" 3;
|
||||
"~*Bolt" 3;
|
||||
"~*BotALot" 3;
|
||||
"~*Brandprotect" 3;
|
||||
"~*Brandwatch" 3;
|
||||
"~*Bubing" 3;
|
||||
"~*Buddy" 3;
|
||||
"~*BuiltBotTough" 3;
|
||||
"~*BuiltWith" 3;
|
||||
"~*Bullseye" 3;
|
||||
"~*BunnySlippers" 3;
|
||||
"~*BuzzSumo" 3;
|
||||
"~*calculon\ spider" 3;
|
||||
"~*Calculon" 3;
|
||||
"~*CazoodleBot" 3;
|
||||
"~*CCBot" 3;
|
||||
"~*Cegbfeieh" 3;
|
||||
"~*CheeseBot" 3;
|
||||
"~*CherryPicker" 3;
|
||||
"~*ChinaClaw" 3;
|
||||
"~*chlooe" 3;
|
||||
"~*Chlooe" 3;
|
||||
"~*Claritybot" 3;
|
||||
"~*Cliqzbot" 3;
|
||||
"~*Cogentbot" 3;
|
||||
|
|
@ -259,42 +263,41 @@ map $http_user_agent $bad_bot {
|
|||
"~*CopyRightCheck" 3;
|
||||
"~*Copyscape" 3;
|
||||
"~*Cosmos" 3;
|
||||
"~*craftbot@yahoo.com" 3;
|
||||
"~*Craftbot" 3;
|
||||
"~*CrazyWebCrawler" 3;
|
||||
"~*Crescent" 3;
|
||||
"~*CSHttp" 3;
|
||||
"~*Curious" 3;
|
||||
"~*Custo" 3;
|
||||
"~*DBLBot" 3;
|
||||
"~*Demon" 3;
|
||||
"~*deusu" 3;
|
||||
"~*Deusu" 3;
|
||||
"~*Devil" 3;
|
||||
"~*Disco" 3;
|
||||
"~*DIIbot" 3;
|
||||
"~*dirbuster" 3;
|
||||
"~*discobot" 3;
|
||||
"~*discoverybot" 3;
|
||||
"~*Dirbuster" 3;
|
||||
"~*Disco" 3;
|
||||
"~*Discobot" 3;
|
||||
"~*Discoverybot" 3;
|
||||
"~*DittoSpyder" 3;
|
||||
"~*DomainAppender" 3;
|
||||
"~*DomainSigmaCrawler" 3;
|
||||
"~*dotbot" 3;
|
||||
"~*Dotbot" 3;
|
||||
"~*Download\ Demon" 3;
|
||||
"~*Download\ Devil" 3;
|
||||
"~*Download\ Wonder" 3;
|
||||
"~*dragonfly" 3;
|
||||
"~*Dragonfly" 3;
|
||||
"~*Drip" 3;
|
||||
"~*DTS\ Agent" 3;
|
||||
"~*EasyDL" 3;
|
||||
"~*ebingbong" 3;
|
||||
"~*Ebingbong" 3;
|
||||
"~*eCatch" 3;
|
||||
"~*ECCP/1.0" 3;
|
||||
"~*Ecxi" 3;
|
||||
"~*EirGrabber" 3;
|
||||
"~*EMail\ Collector" 3;
|
||||
"~*EmailCollector" 3;
|
||||
"~*EMail\ Extractor" 3;
|
||||
"~*EMail\ Siphon" 3;
|
||||
"~*EmailSiphon" 3;
|
||||
"~*EMail\ Wolf" 3;
|
||||
"~*EmailWolf" 3;
|
||||
"~*EroCrawler" 3;
|
||||
"~*Evil" 3;
|
||||
"~*Exabot" 3;
|
||||
|
|
@ -302,43 +305,48 @@ map $http_user_agent $bad_bot {
|
|||
"~*Extractor" 3;
|
||||
"~*ExtractorPro" 3;
|
||||
"~*EyeNetIE" 3;
|
||||
"~*ezooms" 3;
|
||||
"~*Ezooms" 3;
|
||||
"~*FHscan" 3;
|
||||
"~*fimap" 3;
|
||||
"~*Fimap" 3;
|
||||
"~*Findxbot" 3;
|
||||
"~*FlashGet" 3;
|
||||
"~*flunky" 3;
|
||||
"~*Flunky" 3;
|
||||
"~*Foobot" 3;
|
||||
"~*freeuploader" 3;
|
||||
"~*Freeuploader" 3;
|
||||
"~*FrontPage" 3;
|
||||
"~*GalaxyBot" 3;
|
||||
"~*Genieo" 3;
|
||||
"~*getintent" 3;
|
||||
"~*Getintent" 3;
|
||||
"~*GetRight" 3;
|
||||
"~*GetWeb" 3;
|
||||
"~*gigablast" 3;
|
||||
"~*gigabot" 3;
|
||||
"~*Gigablast" 3;
|
||||
"~*Gigabot" 3;
|
||||
"~*Go-Ahead-Got-It" 3;
|
||||
"~*gotit" 3;
|
||||
"~*Go!Zilla" 3;
|
||||
"~*Gotit" 3;
|
||||
"~*GoZilla" 3;
|
||||
"~*Grabber" 3;
|
||||
"~*GrabNet" 3;
|
||||
"~*Grafula" 3;
|
||||
"~*grapeFX" 3;
|
||||
"~*GrapeFX" 3;
|
||||
"~*GrapeshotCrawler" 3;
|
||||
"~*GridBot" 3;
|
||||
"~*GT::WWW" 3;
|
||||
"~*GuzzleHttp" 3;
|
||||
"~*HaosouSpider" 3;
|
||||
"~*Harvest" 3;
|
||||
"~*Havij" 3;
|
||||
"~*Heritrix" 3;
|
||||
"~*hloader" 3;
|
||||
"~*Hloader" 3;
|
||||
"~*HMView" 3;
|
||||
"~*HTTrack" 3;
|
||||
"~*HTMLparser" 3;
|
||||
"~*Huaweisymantecspider" 3;
|
||||
"~*humanlinks" 3;
|
||||
"~*iblog" 3;
|
||||
"~*HTTP::Lite" 3;
|
||||
"~*HTTrack" 3;
|
||||
"~*Humanlinks" 3;
|
||||
"~*Iblog" 3;
|
||||
"~*Id-search" 3;
|
||||
"~*IDBot" 3;
|
||||
"~*IlseBot" 3;
|
||||
"~*imagefetch" 3;
|
||||
"~*Image\ Fetch" 3;
|
||||
"~*Image\ Stripper" 3;
|
||||
"~*Image\ Sucker" 3;
|
||||
|
|
@ -346,91 +354,100 @@ map $http_user_agent $bad_bot {
|
|||
"~*InfoNaviRobot" 3;
|
||||
"~*InfoTekies" 3;
|
||||
"~*Intelliseek" 3;
|
||||
"~*Internet\ Ninja" 3;
|
||||
"~*internetVista\ monitor" 3;
|
||||
"~*InterGET" 3;
|
||||
"~*Internet\ Ninja" 3;
|
||||
"~*InternetSeer" 3;
|
||||
"~*internetVista\ monitor" 3;
|
||||
"~*Iria" 3;
|
||||
"~*iskanie" 3;
|
||||
"~*IRLbot" 3;
|
||||
"~*Iskanie" 3;
|
||||
"~*JamesBOT" 3;
|
||||
"~*Jbrofuzz" 3;
|
||||
"~*JennyBot" 3;
|
||||
"~*JetCar" 3;
|
||||
"~*jbrofuzz" 3;
|
||||
"~*JikeSpider" 3;
|
||||
"~*JOC" 3;
|
||||
"~*JOC\ Web\ Spider" 3;
|
||||
"~*Joomla" 3;
|
||||
"~*JustView" 3;
|
||||
"~*Jyxobot" 3;
|
||||
"~*Kenjin\ Spider" 3;
|
||||
"~*Keyword\ Density" 3;
|
||||
"~*kraken" 3;
|
||||
"~*lanshanbot" 3;
|
||||
"~*larbin" 3;
|
||||
"~*Kraken" 3;
|
||||
"~*Lanshanbot" 3;
|
||||
"~*Larbin" 3;
|
||||
"~*LeechFTP" 3;
|
||||
"~*LeechGet" 3;
|
||||
"~*LexiBot" 3;
|
||||
"~*lftp" 3;
|
||||
"~*libcurl" 3;
|
||||
"~*libWeb" 3;
|
||||
"~*libwhisker" 3;
|
||||
"~*likse" 3;
|
||||
"~*linkdexbot" 3;
|
||||
"~*LinkpadBot" 3;
|
||||
"~*Lftp" 3;
|
||||
"~*LibWeb" 3;
|
||||
"~*Libwhisker" 3;
|
||||
"~*Lightspeedsystems" 3;
|
||||
"~*Likse" 3;
|
||||
"~*Linkdexbot" 3;
|
||||
"~*LinkextractorPro" 3;
|
||||
"~*LinkpadBot" 3;
|
||||
"~*LinkScan" 3;
|
||||
"~*LinqiaMetadataDownloaderBot" 3;
|
||||
"~*LinqiaScrapeBot" 3;
|
||||
"~*LinqiaRSSBot" 3;
|
||||
"~*lmspider" 3;
|
||||
"~*LNSpiderguy" 3;
|
||||
"~*LinksManager" 3;
|
||||
"~*LinkWalker" 3;
|
||||
"~*ltx71" 3;
|
||||
"~*LinqiaMetadataDownloaderBot" 3;
|
||||
"~*LinqiaRSSBot" 3;
|
||||
"~*LinqiaScrapeBot" 3;
|
||||
"~*Lipperhey" 3;
|
||||
"~*Litemage_walker" 3;
|
||||
"~*Lmspider" 3;
|
||||
"~*LNSpiderguy" 3;
|
||||
"~*Ltx71" 3;
|
||||
"~*lwp-request" 3;
|
||||
"~*lwp-trivial" 3;
|
||||
"~*LWP::Simple" 3;
|
||||
"~*magpie-crawler" 3;
|
||||
"~*Magnet" 3;
|
||||
"~*Mag-Net" 3;
|
||||
"~*mail.ru" 3;
|
||||
"~*Mail.RU_Bot" 3;
|
||||
"~*majestic12" 3;
|
||||
"~*Magnet" 3;
|
||||
"~*magpie-crawler" 3;
|
||||
"~*Mail.ru" 3;
|
||||
"~*Majestic12" 3;
|
||||
"~*MarkMonitor" 3;
|
||||
"~*MarkWatch" 3;
|
||||
"~*masscan" 3;
|
||||
"~*Mass\ Downloader" 3;
|
||||
"~*Masscan" 3;
|
||||
"~*Mata\ Hari" 3;
|
||||
"~*Maxthon" 3;
|
||||
"~*Meanpathbot" 3;
|
||||
"~*MegaIndex.ru" 3;
|
||||
"~*Metauri" 3;
|
||||
"~*MFC_Tear_Sample" 3;
|
||||
"~*Microsoft\ Data\ Access" 3;
|
||||
"~*Microsoft\ URL\ Control" 3;
|
||||
"~*MIDown\ tool" 3;
|
||||
"~*MIIxpc" 3;
|
||||
"~*Mister\ PiX" 3;
|
||||
"~*MJ12bot" 3;
|
||||
"~*mojeek" 3;
|
||||
"~*MSIECrawler" 3;
|
||||
"~*MSIE\ 6.0" 3;
|
||||
"~*Mojeek" 3;
|
||||
"~*MS\ Web\ Services\ Client\ Protocol" 3;
|
||||
"~*msrabot" 3;
|
||||
"~*musobot" 3;
|
||||
"~*MSFrontPage" 3;
|
||||
"~*MSIE\ 6.0" 3;
|
||||
"~*MSIECrawler" 3;
|
||||
"~*Msrabot" 3;
|
||||
"~*Musobot" 3;
|
||||
"~*Name\ Intelligence" 3;
|
||||
"~*Nameprotect" 3;
|
||||
"~*Navroad" 3;
|
||||
"~*NearSite" 3;
|
||||
"~*needle" 3;
|
||||
"~*nessus" 3;
|
||||
"~*Needle" 3;
|
||||
"~*Nessus" 3;
|
||||
"~*Net\ Vampire" 3;
|
||||
"~*NetAnts" 3;
|
||||
"~*Netcraft" 3;
|
||||
"~*netEstate\ NE\ Crawler" 3;
|
||||
"~*NetLyzer" 3;
|
||||
"~*NetMechanic" 3;
|
||||
"~*NetSpider" 3;
|
||||
"~*nettrack" 3;
|
||||
"~*Net\ Vampire" 3;
|
||||
"~*Nettrack" 3;
|
||||
"~*NetZIP" 3;
|
||||
"~*NextGenSearchBot" 3;
|
||||
"~*nibbler" 3;
|
||||
"~*Nibbler" 3;
|
||||
"~*NICErsPRO" 3;
|
||||
"~*niki-bot" 3;
|
||||
"~*nikto" 3;
|
||||
"~*Niki-bot" 3;
|
||||
"~*Nikto" 3;
|
||||
"~*NimbleCrawler" 3;
|
||||
"~*Ninja" 3;
|
||||
"~*Nmap" 3;
|
||||
|
|
@ -439,70 +456,83 @@ map $http_user_agent $bad_bot {
|
|||
"~*Octopus" 3;
|
||||
"~*Offline\ Explorer" 3;
|
||||
"~*Offline\ Navigator" 3;
|
||||
"~*omgilibot" 3;
|
||||
"~*OpenLinkProfiler" 3;
|
||||
"~*Omgilibot" 3;
|
||||
"~*Openfind" 3;
|
||||
"~*OpenLinkProfiler" 3;
|
||||
"~*Openvas" 3;
|
||||
"~*OrangeBot" 3;
|
||||
"~*OrangeSpider" 3;
|
||||
"~*OutfoxBot" 3;
|
||||
"~*PageAnalyzer" 3;
|
||||
"~*Page\ Analyzer" 3;
|
||||
"~*PageGrabber" 3;
|
||||
"~*Page\ Grabber" 3;
|
||||
"~*page\ scorer" 3;
|
||||
"~*PageAnalyzer" 3;
|
||||
"~*PageGrabber" 3;
|
||||
"~*PageScorer" 3;
|
||||
"~*Panscient" 3;
|
||||
"~*Papa\ Foto" 3;
|
||||
"~*pavuk" 3;
|
||||
"~*Pavuk" 3;
|
||||
"~*pcBrowser" 3;
|
||||
"~*PECL::HTTP" 3;
|
||||
"~*PeoplePal" 3;
|
||||
"~*PHPCrawl" 3;
|
||||
"~*Pi-Monster" 3;
|
||||
"~*Picscout" 3;
|
||||
"~*Picsearch" 3;
|
||||
"~*pimonster" 3;
|
||||
"~*Pi-Monster" 3;
|
||||
"~*Pimonster" 3;
|
||||
"~*Pixray" 3;
|
||||
"~*PleaseCrawl" 3;
|
||||
"~*Pockey" 3;
|
||||
"~*POE-Component-Client-HTTP" 3;
|
||||
"~*probethenet" 3;
|
||||
"~*Probethenet" 3;
|
||||
"~*ProPowerBot" 3;
|
||||
"~*ProWebWalker" 3;
|
||||
"~*proximic" 3;
|
||||
"~*psbot" 3;
|
||||
"~*Proximic" 3;
|
||||
"~*Psbot" 3;
|
||||
"~*Pump" 3;
|
||||
"~*PyCurl" 3;
|
||||
"~*QueryN\ Metasearch" 3;
|
||||
"~*Qwantify" 3;
|
||||
"~*RankActiveLinkBot" 3;
|
||||
"~*Rankivabot" 3;
|
||||
"~*RealDownload" 3;
|
||||
"~*Reaper" 3;
|
||||
"~*Ripper" 3;
|
||||
"~*Recorder" 3;
|
||||
"~*RedesScrapy" 3;
|
||||
"~*ReGet" 3;
|
||||
"~*RepoMonkey" 3;
|
||||
"~*Ripper" 3;
|
||||
"~*RocketCrawler" 3;
|
||||
"~*Rogerbot" 3;
|
||||
"~*SBIder" 3;
|
||||
"~*ScanAlert" 3;
|
||||
"~*scanbot" 3;
|
||||
"~*Scanbot" 3;
|
||||
"~*Scrapy" 3;
|
||||
"~*Screaming\ Frog\ SEO\ Spider" 3;
|
||||
"~*ScreenerBot" 3;
|
||||
"~*searchestate" 3;
|
||||
"~*SeaMonkey" 3;
|
||||
"~*Searchestate" 3;
|
||||
"~*SearchmetricsBot" 3;
|
||||
"~*Semrush" 3;
|
||||
"~*SemrushBot" 3;
|
||||
"~*SEOkicks" 3;
|
||||
"~*SEOkicks-Robot" 3;
|
||||
"~*SEOlyticsCrawler" 3;
|
||||
"~*seomoz" 3;
|
||||
"~*Seomoz" 3;
|
||||
"~*SEOprofiler" 3;
|
||||
"~*SEOstats" 3;
|
||||
"~*SISTRIX" 3;
|
||||
"~*SeznamBot" 3;
|
||||
"~*Siphon" 3;
|
||||
"~*sitebeam" 3;
|
||||
"~*SISTRIX" 3;
|
||||
"~*SISTRIX\ Crawler" 3;
|
||||
"~*Site\ Sucker" 3;
|
||||
"~*Sitebeam" 3;
|
||||
"~*SiteExplorer" 3;
|
||||
"~*Siteimprove" 3;
|
||||
"~*SiteLockSpider" 3;
|
||||
"~*SiteSucker" 3;
|
||||
"~*Site\ Sucker" 3;
|
||||
"~*SiteSnagger" 3;
|
||||
"~*sitevigil" 3;
|
||||
"~*SiteSucker" 3;
|
||||
"~*Sitevigil" 3;
|
||||
"~*Slackbot-LinkExpanding" 3;
|
||||
"~*SlySearch" 3;
|
||||
"~*SmartDownload" 3;
|
||||
|
|
@ -513,14 +543,16 @@ map $http_user_agent $bad_bot {
|
|||
"~*Sogou\ web\ spider" 3;
|
||||
"~*Sosospider" 3;
|
||||
"~*SpaceBison" 3;
|
||||
"~*spammen" 3;
|
||||
"~*spanner" 3;
|
||||
"~*Spammen" 3;
|
||||
"~*SpankBot" 3;
|
||||
"~*Spanner" 3;
|
||||
"~*Spbot" 3;
|
||||
"~*Spinn3r" 3;
|
||||
"~*spbot" 3;
|
||||
"~*sqlmap" 3;
|
||||
"~*SputnikBot" 3;
|
||||
"~*Sqlmap" 3;
|
||||
"~*Sqlworm" 3;
|
||||
"~*Sqworm" 3;
|
||||
"~*Steeler" 3;
|
||||
"~*Stripper" 3;
|
||||
"~*Sucker" 3;
|
||||
"~*Sucuri" 3;
|
||||
|
|
@ -528,10 +560,10 @@ map $http_user_agent $bad_bot {
|
|||
"~*SuperHTTP" 3;
|
||||
"~*Surfbot" 3;
|
||||
"~*SurveyBot" 3;
|
||||
"~*suzuran" 3;
|
||||
"~*Suzuran" 3;
|
||||
"~*Swiftbot" 3;
|
||||
"~*SISTRIX\ Crawler" 3;
|
||||
"~*Szukacz" 3;
|
||||
"~*T0PHackTeam" 3;
|
||||
"~*T8Abot" 3;
|
||||
"~*tAkeOut" 3;
|
||||
"~*Teleport" 3;
|
||||
|
|
@ -543,91 +575,102 @@ map $http_user_agent $bad_bot {
|
|||
"~*TheNomad" 3;
|
||||
"~*TightTwatBot" 3;
|
||||
"~*Titan" 3;
|
||||
"~*T0PHackTeam" 3;
|
||||
"~*Toata" 3;
|
||||
"~*Toweyabot" 3;
|
||||
"~*Trendictionbot" 3;
|
||||
"~*True_Robot" 3;
|
||||
"~*Turingos" 3;
|
||||
"~*TurnitinBot" 3;
|
||||
"~*Turnitin\ Robot" 3;
|
||||
"~*Turnitin\ Bot" 3;
|
||||
"~*Turnitin\ Robot" 3;
|
||||
"~*TurnitinBot" 3;
|
||||
"~*TwengaBot" 3;
|
||||
"~*Twice" 3;
|
||||
"~*Typhoeus" 3;
|
||||
"~*UptimeRobot/2.0" 3;
|
||||
"~*URLy.Warning" 3;
|
||||
"~*URLy\ Warning" 3;
|
||||
"~*Vacuum" 3;
|
||||
"~*Vagabondo" 3;
|
||||
"~*VB\ Project" 3;
|
||||
"~*VCI" 3;
|
||||
"~*VeriCiteCrawler" 3;
|
||||
"~*VidibleScraper" 3;
|
||||
"~*VoidEYE" 3;
|
||||
"~*Voil" 3;
|
||||
"~*Voltron" 3;
|
||||
"~*WallpapersHD" 3;
|
||||
"~*WASALive-Bot" 3;
|
||||
"~*WBSearchBot" 3;
|
||||
"~*Web\ Auto" 3;
|
||||
"~*Web\ Bandit" 3;
|
||||
"~*Web\ Collage" 3;
|
||||
"~*Web\ Copier" 3;
|
||||
"~*Web\ Enhancer" 3;
|
||||
"~*Web\ Fetch" 3;
|
||||
"~*Web\ Fuck" 3;
|
||||
"~*Web\ Image\ Collector" 3;
|
||||
"~*Web\ Pix" 3;
|
||||
"~*Web\ Reaper" 3;
|
||||
"~*Web\ Sauger" 3;
|
||||
"~*Web\ Stripper" 3;
|
||||
"~*Web\ Sucker" 3;
|
||||
"~*Web\ Whacker" 3;
|
||||
"~*Webalta" 3;
|
||||
"~*WebAuto" 3;
|
||||
"~*WebBandit" 3;
|
||||
"~*Web\ Bandit" 3;
|
||||
"~*WebCollage" 3;
|
||||
"~*WebCopier" 3;
|
||||
"~*Web\ Copier" 3;
|
||||
"~*WEBDAV" 3;
|
||||
"~*WEBDAV\ Client" 3;
|
||||
"~*WebEnhancer" 3;
|
||||
"~*Web\ Enhancer" 3;
|
||||
"~*WebFetch" 3;
|
||||
"~*Web\ Fetch" 3;
|
||||
"~*WebFuck" 3;
|
||||
"~*Web\ Fuck" 3;
|
||||
"~*WebImageCollector" 3;
|
||||
"~*WebGo\ IS" 3;
|
||||
"~*Web\ Image\ Collector" 3;
|
||||
"~*WebLeacher" 3;
|
||||
"~*WebmasterWorldForumBot" 3;
|
||||
"~*WebPix" 3;
|
||||
"~*Web\ Pix" 3;
|
||||
"~*WebReaper" 3;
|
||||
"~*Web\ Reaper" 3;
|
||||
"~*WebSauger" 3;
|
||||
"~*Web\ Sauger" 3;
|
||||
"~*webshag" 3;
|
||||
"~*WebsiteExtractor" 3;
|
||||
"~*Webshag" 3;
|
||||
"~*Website\ Extractor" 3;
|
||||
"~*WebsiteQuester" 3;
|
||||
"~*Website\ Quester" 3;
|
||||
"~*WebsiteExtractor" 3;
|
||||
"~*WebsiteQuester" 3;
|
||||
"~*Webster" 3;
|
||||
"~*WebStripper" 3;
|
||||
"~*Web\ Stripper" 3;
|
||||
"~*WebSucker" 3;
|
||||
"~*Web\ Sucker" 3;
|
||||
"~*WebWhacker" 3;
|
||||
"~*Web\ Whacker" 3;
|
||||
"~*WebZIP" 3;
|
||||
"~*WeSEE" 3;
|
||||
"~*Whack" 3;
|
||||
"~*Whacker" 3;
|
||||
"~*whatweb" 3;
|
||||
"~*Whatweb" 3;
|
||||
"~*Widow" 3;
|
||||
"~*WinHTTrack" 3;
|
||||
"~*WiseGuys\ Robot" 3;
|
||||
"~*WISENutbot" 3;
|
||||
"~*wonderbot" 3;
|
||||
"~*woobot" 3;
|
||||
"~*Wonderbot" 3;
|
||||
"~*Woobot" 3;
|
||||
"~*Wotbox" 3;
|
||||
"~*wprecon" 3;
|
||||
"~*WWWOFFLE" 3;
|
||||
"~*WWW::Mechanize" 3;
|
||||
"~*Wprecon" 3;
|
||||
"~*WWW-Collector-E" 3;
|
||||
"~*WWW-Mechanize" 3;
|
||||
"~*WWW::Mechanize" 3;
|
||||
"~*WWWOFFLE" 3;
|
||||
"~*Xaldon_WebSpider" 3;
|
||||
"~*Xaldon\ WebSpider" 3;
|
||||
"~*Xenu" 3;
|
||||
"~*Y!J-ASR" 3;
|
||||
"~*YoudaoBot" 3;
|
||||
"~*Zade" 3;
|
||||
"~*Zermelo" 3;
|
||||
"~*Zeus" 3;
|
||||
"~*zgrab" 3;
|
||||
"~*zitebot" 3;
|
||||
"~*Zgrab" 3;
|
||||
"~*Zitebot" 3;
|
||||
"~*ZmEu" 3;
|
||||
"~*ZumBot" 3;
|
||||
"~*Zyborg" 3;
|
||||
"~*ZyBorg" 3;
|
||||
# END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ###
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,28 +1,20 @@
|
|||
User-agent: Acunetix
|
||||
Disallow:/
|
||||
User-agent: FHscan
|
||||
Disallow:/
|
||||
User-agent: masscan
|
||||
Disallow:/
|
||||
User-agent: probethenet
|
||||
Disallow:/
|
||||
User-agent: scanbot
|
||||
User-agent: 360Spider
|
||||
Disallow:/
|
||||
User-agent: 80legs
|
||||
Disallow:/
|
||||
User-agent: 360Spider
|
||||
User-agent: Abonti
|
||||
Disallow:/
|
||||
User-agent: Aboundex
|
||||
Disallow:/
|
||||
User-agent: Abonti
|
||||
User-agent: Acunetix
|
||||
Disallow:/
|
||||
User-agent: AhrefsBot
|
||||
Disallow:/
|
||||
User-agent: AIBOT
|
||||
Disallow:/
|
||||
User-agent: aipbot
|
||||
User-agent: AiHitBot
|
||||
Disallow:/
|
||||
User-agent: aiHitBot
|
||||
User-agent: Aipbot
|
||||
Disallow:/
|
||||
User-agent: Alexibot
|
||||
Disallow:/
|
||||
|
|
@ -34,11 +26,11 @@ User-agent: Anarchie
|
|||
Disallow:/
|
||||
User-agent: Apexoo
|
||||
Disallow:/
|
||||
User-agent: asterias
|
||||
Disallow:/
|
||||
User-agent: ASPSeek
|
||||
Disallow:/
|
||||
User-agent: attach
|
||||
User-agent: Asterias
|
||||
Disallow:/
|
||||
User-agent: Attach
|
||||
Disallow:/
|
||||
User-agent: autoemailspider
|
||||
Disallow:/
|
||||
|
|
@ -60,6 +52,8 @@ User-agent: BBBike
|
|||
Disallow:/
|
||||
User-agent: BDFetch
|
||||
Disallow:/
|
||||
User-agent: BetaBot
|
||||
Disallow:/
|
||||
User-agent: Bigfoot
|
||||
Disallow:/
|
||||
User-agent: Bitacle
|
||||
|
|
@ -70,11 +64,13 @@ User-agent: BlackWidow
|
|||
Disallow:/
|
||||
User-agent: BLEXBot
|
||||
Disallow:/
|
||||
User-agent: BlowFish
|
||||
Disallow:/
|
||||
User-agent: Blow
|
||||
Disallow:/
|
||||
User-agent: boardreader
|
||||
User-agent: BlowFish
|
||||
Disallow:/
|
||||
User-agent: Boardreader
|
||||
Disallow:/
|
||||
User-agent: Bolt
|
||||
Disallow:/
|
||||
User-agent: BotALot
|
||||
Disallow:/
|
||||
|
|
@ -82,6 +78,8 @@ User-agent: Brandprotect
|
|||
Disallow:/
|
||||
User-agent: Brandwatch
|
||||
Disallow:/
|
||||
User-agent: Bubing
|
||||
Disallow:/
|
||||
User-agent: Buddy
|
||||
Disallow:/
|
||||
User-agent: BuiltBotTough
|
||||
|
|
@ -94,7 +92,9 @@ User-agent: BunnySlippers
|
|||
Disallow:/
|
||||
User-agent: BuzzSumo
|
||||
Disallow:/
|
||||
User-agent: calculon spider
|
||||
User-agent: Calculon
|
||||
Disallow:/
|
||||
User-agent: CazoodleBot
|
||||
Disallow:/
|
||||
User-agent: CCBot
|
||||
Disallow:/
|
||||
|
|
@ -106,7 +106,7 @@ User-agent: CherryPicker
|
|||
Disallow:/
|
||||
User-agent: ChinaClaw
|
||||
Disallow:/
|
||||
User-agent: chlooe
|
||||
User-agent: Chlooe
|
||||
Disallow:/
|
||||
User-agent: Claritybot
|
||||
Disallow:/
|
||||
|
|
@ -124,7 +124,7 @@ User-agent: Copyscape
|
|||
Disallow:/
|
||||
User-agent: Cosmos
|
||||
Disallow:/
|
||||
User-agent: craftbot@yahoo.com
|
||||
User-agent: Craftbot
|
||||
Disallow:/
|
||||
User-agent: CrazyWebCrawler
|
||||
Disallow:/
|
||||
|
|
@ -132,25 +132,27 @@ User-agent: Crescent
|
|||
Disallow:/
|
||||
User-agent: CSHttp
|
||||
Disallow:/
|
||||
User-agent: Curious
|
||||
Disallow:/
|
||||
User-agent: Custo
|
||||
Disallow:/
|
||||
User-agent: DBLBot
|
||||
Disallow:/
|
||||
User-agent: Demon
|
||||
Disallow:/
|
||||
User-agent: deusu
|
||||
User-agent: Deusu
|
||||
Disallow:/
|
||||
User-agent: Devil
|
||||
Disallow:/
|
||||
User-agent: Disco
|
||||
Disallow:/
|
||||
User-agent: DIIbot
|
||||
Disallow:/
|
||||
User-agent: dirbuster
|
||||
User-agent: Dirbuster
|
||||
Disallow:/
|
||||
User-agent: discobot
|
||||
User-agent: Disco
|
||||
Disallow:/
|
||||
User-agent: discoverybot
|
||||
User-agent: Discobot
|
||||
Disallow:/
|
||||
User-agent: Discoverybot
|
||||
Disallow:/
|
||||
User-agent: DittoSpyder
|
||||
Disallow:/
|
||||
|
|
@ -158,7 +160,7 @@ User-agent: DomainAppender
|
|||
Disallow:/
|
||||
User-agent: DomainSigmaCrawler
|
||||
Disallow:/
|
||||
User-agent: dotbot
|
||||
User-agent: Dotbot
|
||||
Disallow:/
|
||||
User-agent: Download Demon
|
||||
Disallow:/
|
||||
|
|
@ -166,7 +168,7 @@ User-agent: Download Devil
|
|||
Disallow:/
|
||||
User-agent: Download Wonder
|
||||
Disallow:/
|
||||
User-agent: dragonfly
|
||||
User-agent: Dragonfly
|
||||
Disallow:/
|
||||
User-agent: Drip
|
||||
Disallow:/
|
||||
|
|
@ -174,28 +176,24 @@ User-agent: DTS Agent
|
|||
Disallow:/
|
||||
User-agent: EasyDL
|
||||
Disallow:/
|
||||
User-agent: ebingbong
|
||||
User-agent: Ebingbong
|
||||
Disallow:/
|
||||
User-agent: eCatch
|
||||
Disallow:/
|
||||
User-agent: ECCP/1.0
|
||||
Disallow:/
|
||||
User-agent: Ecxi
|
||||
Disallow:/
|
||||
User-agent: EirGrabber
|
||||
Disallow:/
|
||||
User-agent: EMail Collector
|
||||
Disallow:/
|
||||
User-agent: EmailCollector
|
||||
Disallow:/
|
||||
User-agent: EMail Extractor
|
||||
Disallow:/
|
||||
User-agent: EMail Siphon
|
||||
Disallow:/
|
||||
User-agent: EmailSiphon
|
||||
Disallow:/
|
||||
User-agent: EMail Wolf
|
||||
Disallow:/
|
||||
User-agent: EmailWolf
|
||||
Disallow:/
|
||||
User-agent: EroCrawler
|
||||
Disallow:/
|
||||
User-agent: Evil
|
||||
|
|
@ -210,17 +208,21 @@ User-agent: ExtractorPro
|
|||
Disallow:/
|
||||
User-agent: EyeNetIE
|
||||
Disallow:/
|
||||
User-agent: ezooms
|
||||
User-agent: Ezooms
|
||||
Disallow:/
|
||||
User-agent: fimap
|
||||
User-agent: FHscan
|
||||
Disallow:/
|
||||
User-agent: Fimap
|
||||
Disallow:/
|
||||
User-agent: Findxbot
|
||||
Disallow:/
|
||||
User-agent: FlashGet
|
||||
Disallow:/
|
||||
User-agent: flunky
|
||||
User-agent: Flunky
|
||||
Disallow:/
|
||||
User-agent: Foobot
|
||||
Disallow:/
|
||||
User-agent: freeuploader
|
||||
User-agent: Freeuploader
|
||||
Disallow:/
|
||||
User-agent: FrontPage
|
||||
Disallow:/
|
||||
|
|
@ -228,34 +230,40 @@ User-agent: GalaxyBot
|
|||
Disallow:/
|
||||
User-agent: Genieo
|
||||
Disallow:/
|
||||
User-agent: getintent
|
||||
User-agent: Getintent
|
||||
Disallow:/
|
||||
User-agent: GetRight
|
||||
Disallow:/
|
||||
User-agent: GetWeb
|
||||
Disallow:/
|
||||
User-agent: gigablast
|
||||
User-agent: Gigablast
|
||||
Disallow:/
|
||||
User-agent: gigabot
|
||||
User-agent: Gigabot
|
||||
Disallow:/
|
||||
User-agent: Go-Ahead-Got-It
|
||||
Disallow:/
|
||||
User-agent: gotit
|
||||
Disallow:/
|
||||
User-agent: Go!Zilla
|
||||
Disallow:/
|
||||
User-agent: Gotit
|
||||
Disallow:/
|
||||
User-agent: GoZilla
|
||||
Disallow:/
|
||||
User-agent: Grabber
|
||||
Disallow:/
|
||||
User-agent: GrabNet
|
||||
Disallow:/
|
||||
User-agent: Grafula
|
||||
Disallow:/
|
||||
User-agent: grapeFX
|
||||
User-agent: GrapeFX
|
||||
Disallow:/
|
||||
User-agent: GrapeshotCrawler
|
||||
Disallow:/
|
||||
User-agent: GridBot
|
||||
Disallow:/
|
||||
User-agent: GT::WWW
|
||||
Disallow:/
|
||||
User-agent: GuzzleHttp
|
||||
Disallow:/
|
||||
User-agent: HaosouSpider
|
||||
Disallow:/
|
||||
User-agent: Harvest
|
||||
|
|
@ -264,24 +272,26 @@ User-agent: Havij
|
|||
Disallow:/
|
||||
User-agent: Heritrix
|
||||
Disallow:/
|
||||
User-agent: hloader
|
||||
User-agent: Hloader
|
||||
Disallow:/
|
||||
User-agent: HMView
|
||||
Disallow:/
|
||||
User-agent: HTTrack
|
||||
Disallow:/
|
||||
User-agent: HTMLparser
|
||||
Disallow:/
|
||||
User-agent: Huaweisymantecspider
|
||||
User-agent: HTTP::Lite
|
||||
Disallow:/
|
||||
User-agent: humanlinks
|
||||
User-agent: HTTrack
|
||||
Disallow:/
|
||||
User-agent: iblog
|
||||
User-agent: Humanlinks
|
||||
Disallow:/
|
||||
User-agent: Iblog
|
||||
Disallow:/
|
||||
User-agent: Id-search
|
||||
Disallow:/
|
||||
User-agent: IDBot
|
||||
Disallow:/
|
||||
User-agent: IlseBot
|
||||
Disallow:/
|
||||
User-agent: imagefetch
|
||||
Disallow:/
|
||||
User-agent: Image Fetch
|
||||
Disallow:/
|
||||
User-agent: Image Stripper
|
||||
|
|
@ -296,28 +306,30 @@ User-agent: InfoTekies
|
|||
Disallow:/
|
||||
User-agent: Intelliseek
|
||||
Disallow:/
|
||||
User-agent: InterGET
|
||||
Disallow:/
|
||||
User-agent: Internet Ninja
|
||||
Disallow:/
|
||||
User-agent: InternetSeer
|
||||
Disallow:/
|
||||
User-agent: internetVista monitor
|
||||
Disallow:/
|
||||
User-agent: InterGET
|
||||
Disallow:/
|
||||
User-agent: Iria
|
||||
Disallow:/
|
||||
User-agent: iskanie
|
||||
User-agent: IRLbot
|
||||
Disallow:/
|
||||
User-agent: Iskanie
|
||||
Disallow:/
|
||||
User-agent: JamesBOT
|
||||
Disallow:/
|
||||
User-agent: Jbrofuzz
|
||||
Disallow:/
|
||||
User-agent: JennyBot
|
||||
Disallow:/
|
||||
User-agent: JetCar
|
||||
Disallow:/
|
||||
User-agent: jbrofuzz
|
||||
Disallow:/
|
||||
User-agent: JikeSpider
|
||||
Disallow:/
|
||||
User-agent: JOC
|
||||
Disallow:/
|
||||
User-agent: JOC Web Spider
|
||||
Disallow:/
|
||||
User-agent: Joomla
|
||||
|
|
@ -330,11 +342,11 @@ User-agent: Kenjin Spider
|
|||
Disallow:/
|
||||
User-agent: Keyword Density
|
||||
Disallow:/
|
||||
User-agent: kraken
|
||||
User-agent: Kraken
|
||||
Disallow:/
|
||||
User-agent: lanshanbot
|
||||
User-agent: Lanshanbot
|
||||
Disallow:/
|
||||
User-agent: larbin
|
||||
User-agent: Larbin
|
||||
Disallow:/
|
||||
User-agent: LeechFTP
|
||||
Disallow:/
|
||||
|
|
@ -342,53 +354,59 @@ User-agent: LeechGet
|
|||
Disallow:/
|
||||
User-agent: LexiBot
|
||||
Disallow:/
|
||||
User-agent: lftp
|
||||
User-agent: Lftp
|
||||
Disallow:/
|
||||
User-agent: libcurl
|
||||
User-agent: LibWeb
|
||||
Disallow:/
|
||||
User-agent: libWeb
|
||||
User-agent: Libwhisker
|
||||
Disallow:/
|
||||
User-agent: libwhisker
|
||||
User-agent: Lightspeedsystems
|
||||
Disallow:/
|
||||
User-agent: likse
|
||||
User-agent: Likse
|
||||
Disallow:/
|
||||
User-agent: linkdexbot
|
||||
Disallow:/
|
||||
User-agent: LinkpadBot
|
||||
User-agent: Linkdexbot
|
||||
Disallow:/
|
||||
User-agent: LinkextractorPro
|
||||
Disallow:/
|
||||
User-agent: LinkpadBot
|
||||
Disallow:/
|
||||
User-agent: LinkScan
|
||||
Disallow:/
|
||||
User-agent: LinqiaMetadataDownloaderBot
|
||||
Disallow:/
|
||||
User-agent: LinqiaScrapeBot
|
||||
Disallow:/
|
||||
User-agent: LinqiaRSSBot
|
||||
Disallow:/
|
||||
User-agent: lmspider
|
||||
Disallow:/
|
||||
User-agent: LNSpiderguy
|
||||
User-agent: LinksManager
|
||||
Disallow:/
|
||||
User-agent: LinkWalker
|
||||
Disallow:/
|
||||
User-agent: ltx71
|
||||
User-agent: LinqiaMetadataDownloaderBot
|
||||
Disallow:/
|
||||
User-agent: LinqiaRSSBot
|
||||
Disallow:/
|
||||
User-agent: LinqiaScrapeBot
|
||||
Disallow:/
|
||||
User-agent: Lipperhey
|
||||
Disallow:/
|
||||
User-agent: Litemage_walker
|
||||
Disallow:/
|
||||
User-agent: Lmspider
|
||||
Disallow:/
|
||||
User-agent: LNSpiderguy
|
||||
Disallow:/
|
||||
User-agent: Ltx71
|
||||
Disallow:/
|
||||
User-agent: lwp-request
|
||||
Disallow:/
|
||||
User-agent: lwp-trivial
|
||||
Disallow:/
|
||||
User-agent: LWP::Simple
|
||||
Disallow:/
|
||||
User-agent: magpie-crawler
|
||||
Disallow:/
|
||||
User-agent: Magnet
|
||||
Disallow:/
|
||||
User-agent: Mag-Net
|
||||
Disallow:/
|
||||
User-agent: mail.ru
|
||||
User-agent: Magnet
|
||||
Disallow:/
|
||||
User-agent: Mail.RU_Bot
|
||||
User-agent: magpie-crawler
|
||||
Disallow:/
|
||||
User-agent: majestic12
|
||||
User-agent: Mail.ru
|
||||
Disallow:/
|
||||
User-agent: Majestic12
|
||||
Disallow:/
|
||||
User-agent: MarkMonitor
|
||||
Disallow:/
|
||||
|
|
@ -396,12 +414,20 @@ User-agent: MarkWatch
|
|||
Disallow:/
|
||||
User-agent: Mass Downloader
|
||||
Disallow:/
|
||||
User-agent: Masscan
|
||||
Disallow:/
|
||||
User-agent: Mata Hari
|
||||
Disallow:/
|
||||
User-agent: Maxthon
|
||||
Disallow:/
|
||||
User-agent: Meanpathbot
|
||||
Disallow:/
|
||||
User-agent: MegaIndex.ru
|
||||
Disallow:/
|
||||
User-agent: Metauri
|
||||
Disallow:/
|
||||
User-agent: MFC_Tear_Sample
|
||||
Disallow:/
|
||||
User-agent: Microsoft Data Access
|
||||
Disallow:/
|
||||
User-agent: Microsoft URL Control
|
||||
|
|
@ -414,17 +440,19 @@ User-agent: Mister PiX
|
|||
Disallow:/
|
||||
User-agent: MJ12bot
|
||||
Disallow:/
|
||||
User-agent: mojeek
|
||||
Disallow:/
|
||||
User-agent: MSIECrawler
|
||||
Disallow:/
|
||||
User-agent: MSIE 6.0
|
||||
User-agent: Mojeek
|
||||
Disallow:/
|
||||
User-agent: MS Web Services Client Protocol
|
||||
Disallow:/
|
||||
User-agent: msrabot
|
||||
User-agent: MSFrontPage
|
||||
Disallow:/
|
||||
User-agent: musobot
|
||||
User-agent: MSIE 6.0
|
||||
Disallow:/
|
||||
User-agent: MSIECrawler
|
||||
Disallow:/
|
||||
User-agent: Msrabot
|
||||
Disallow:/
|
||||
User-agent: Musobot
|
||||
Disallow:/
|
||||
User-agent: Name Intelligence
|
||||
Disallow:/
|
||||
|
|
@ -434,9 +462,11 @@ User-agent: Navroad
|
|||
Disallow:/
|
||||
User-agent: NearSite
|
||||
Disallow:/
|
||||
User-agent: needle
|
||||
User-agent: Needle
|
||||
Disallow:/
|
||||
User-agent: nessus
|
||||
User-agent: Nessus
|
||||
Disallow:/
|
||||
User-agent: Net Vampire
|
||||
Disallow:/
|
||||
User-agent: NetAnts
|
||||
Disallow:/
|
||||
|
|
@ -444,25 +474,25 @@ User-agent: Netcraft
|
|||
Disallow:/
|
||||
User-agent: netEstate NE Crawler
|
||||
Disallow:/
|
||||
User-agent: NetLyzer
|
||||
Disallow:/
|
||||
User-agent: NetMechanic
|
||||
Disallow:/
|
||||
User-agent: NetSpider
|
||||
Disallow:/
|
||||
User-agent: nettrack
|
||||
Disallow:/
|
||||
User-agent: Net Vampire
|
||||
User-agent: Nettrack
|
||||
Disallow:/
|
||||
User-agent: NetZIP
|
||||
Disallow:/
|
||||
User-agent: NextGenSearchBot
|
||||
Disallow:/
|
||||
User-agent: nibbler
|
||||
User-agent: Nibbler
|
||||
Disallow:/
|
||||
User-agent: NICErsPRO
|
||||
Disallow:/
|
||||
User-agent: niki-bot
|
||||
User-agent: Niki-bot
|
||||
Disallow:/
|
||||
User-agent: nikto
|
||||
User-agent: Nikto
|
||||
Disallow:/
|
||||
User-agent: NimbleCrawler
|
||||
Disallow:/
|
||||
|
|
@ -480,12 +510,12 @@ User-agent: Offline Explorer
|
|||
Disallow:/
|
||||
User-agent: Offline Navigator
|
||||
Disallow:/
|
||||
User-agent: omgilibot
|
||||
Disallow:/
|
||||
User-agent: OpenLinkProfiler
|
||||
User-agent: Omgilibot
|
||||
Disallow:/
|
||||
User-agent: Openfind
|
||||
Disallow:/
|
||||
User-agent: OpenLinkProfiler
|
||||
Disallow:/
|
||||
User-agent: Openvas
|
||||
Disallow:/
|
||||
User-agent: OrangeBot
|
||||
|
|
@ -494,45 +524,57 @@ User-agent: OrangeSpider
|
|||
Disallow:/
|
||||
User-agent: OutfoxBot
|
||||
Disallow:/
|
||||
User-agent: PageAnalyzer
|
||||
Disallow:/
|
||||
User-agent: Page Analyzer
|
||||
Disallow:/
|
||||
User-agent: PageGrabber
|
||||
Disallow:/
|
||||
User-agent: Page Grabber
|
||||
Disallow:/
|
||||
User-agent: page scorer
|
||||
Disallow:/
|
||||
User-agent: PageAnalyzer
|
||||
Disallow:/
|
||||
User-agent: PageGrabber
|
||||
Disallow:/
|
||||
User-agent: PageScorer
|
||||
Disallow:/
|
||||
User-agent: Panscient
|
||||
Disallow:/
|
||||
User-agent: Papa Foto
|
||||
Disallow:/
|
||||
User-agent: pavuk
|
||||
User-agent: Pavuk
|
||||
Disallow:/
|
||||
User-agent: pcBrowser
|
||||
Disallow:/
|
||||
User-agent: PECL::HTTP
|
||||
Disallow:/
|
||||
User-agent: PeoplePal
|
||||
Disallow:/
|
||||
User-agent: PHPCrawl
|
||||
Disallow:/
|
||||
User-agent: Pi-Monster
|
||||
Disallow:/
|
||||
User-agent: Picscout
|
||||
Disallow:/
|
||||
User-agent: Picsearch
|
||||
Disallow:/
|
||||
User-agent: pimonster
|
||||
Disallow:/
|
||||
User-agent: Pi-Monster
|
||||
User-agent: Pimonster
|
||||
Disallow:/
|
||||
User-agent: Pixray
|
||||
Disallow:/
|
||||
User-agent: PleaseCrawl
|
||||
Disallow:/
|
||||
User-agent: Pockey
|
||||
Disallow:/
|
||||
User-agent: POE-Component-Client-HTTP
|
||||
Disallow:/
|
||||
User-agent: Probethenet
|
||||
Disallow:/
|
||||
User-agent: ProPowerBot
|
||||
Disallow:/
|
||||
User-agent: ProWebWalker
|
||||
Disallow:/
|
||||
User-agent: proximic
|
||||
User-agent: Proximic
|
||||
Disallow:/
|
||||
User-agent: psbot
|
||||
User-agent: Psbot
|
||||
Disallow:/
|
||||
User-agent: Pump
|
||||
Disallow:/
|
||||
|
|
@ -540,6 +582,8 @@ User-agent: PyCurl
|
|||
Disallow:/
|
||||
User-agent: QueryN Metasearch
|
||||
Disallow:/
|
||||
User-agent: Qwantify
|
||||
Disallow:/
|
||||
User-agent: RankActiveLinkBot
|
||||
Disallow:/
|
||||
User-agent: Rankivabot
|
||||
|
|
@ -548,25 +592,35 @@ User-agent: RealDownload
|
|||
Disallow:/
|
||||
User-agent: Reaper
|
||||
Disallow:/
|
||||
User-agent: Ripper
|
||||
Disallow:/
|
||||
User-agent: Recorder
|
||||
Disallow:/
|
||||
User-agent: RedesScrapy
|
||||
Disallow:/
|
||||
User-agent: ReGet
|
||||
Disallow:/
|
||||
User-agent: RepoMonkey
|
||||
Disallow:/
|
||||
User-agent: Ripper
|
||||
Disallow:/
|
||||
User-agent: RocketCrawler
|
||||
Disallow:/
|
||||
User-agent: Rogerbot
|
||||
Disallow:/
|
||||
User-agent: SBIder
|
||||
Disallow:/
|
||||
User-agent: ScanAlert
|
||||
Disallow:/
|
||||
User-agent: Scanbot
|
||||
Disallow:/
|
||||
User-agent: Scrapy
|
||||
Disallow:/
|
||||
User-agent: Screaming Frog SEO Spider
|
||||
Disallow:/
|
||||
User-agent: ScreenerBot
|
||||
Disallow:/
|
||||
User-agent: searchestate
|
||||
User-agent: SeaMonkey
|
||||
Disallow:/
|
||||
User-agent: Searchestate
|
||||
Disallow:/
|
||||
User-agent: SearchmetricsBot
|
||||
Disallow:/
|
||||
|
|
@ -580,15 +634,23 @@ User-agent: SEOkicks-Robot
|
|||
Disallow:/
|
||||
User-agent: SEOlyticsCrawler
|
||||
Disallow:/
|
||||
User-agent: seomoz
|
||||
User-agent: Seomoz
|
||||
Disallow:/
|
||||
User-agent: SEOprofiler
|
||||
Disallow:/
|
||||
User-agent: SEOstats
|
||||
Disallow:/
|
||||
User-agent: SISTRIX
|
||||
User-agent: SeznamBot
|
||||
Disallow:/
|
||||
User-agent: Siphon
|
||||
Disallow:/
|
||||
User-agent: sitebeam
|
||||
User-agent: SISTRIX
|
||||
Disallow:/
|
||||
User-agent: SISTRIX Crawler
|
||||
Disallow:/
|
||||
User-agent: Site Sucker
|
||||
Disallow:/
|
||||
User-agent: Sitebeam
|
||||
Disallow:/
|
||||
User-agent: SiteExplorer
|
||||
Disallow:/
|
||||
|
|
@ -596,13 +658,11 @@ User-agent: Siteimprove
|
|||
Disallow:/
|
||||
User-agent: SiteLockSpider
|
||||
Disallow:/
|
||||
User-agent: SiteSucker
|
||||
Disallow:/
|
||||
User-agent: Site Sucker
|
||||
Disallow:/
|
||||
User-agent: SiteSnagger
|
||||
Disallow:/
|
||||
User-agent: sitevigil
|
||||
User-agent: SiteSucker
|
||||
Disallow:/
|
||||
User-agent: Sitevigil
|
||||
Disallow:/
|
||||
User-agent: Slackbot-LinkExpanding
|
||||
Disallow:/
|
||||
|
|
@ -624,22 +684,26 @@ User-agent: Sosospider
|
|||
Disallow:/
|
||||
User-agent: SpaceBison
|
||||
Disallow:/
|
||||
User-agent: spammen
|
||||
Disallow:/
|
||||
User-agent: spanner
|
||||
User-agent: Spammen
|
||||
Disallow:/
|
||||
User-agent: SpankBot
|
||||
Disallow:/
|
||||
User-agent: Spanner
|
||||
Disallow:/
|
||||
User-agent: Spbot
|
||||
Disallow:/
|
||||
User-agent: Spinn3r
|
||||
Disallow:/
|
||||
User-agent: spbot
|
||||
User-agent: SputnikBot
|
||||
Disallow:/
|
||||
User-agent: sqlmap
|
||||
User-agent: Sqlmap
|
||||
Disallow:/
|
||||
User-agent: Sqlworm
|
||||
Disallow:/
|
||||
User-agent: Sqworm
|
||||
Disallow:/
|
||||
User-agent: Steeler
|
||||
Disallow:/
|
||||
User-agent: Stripper
|
||||
Disallow:/
|
||||
User-agent: Sucker
|
||||
|
|
@ -654,14 +718,14 @@ User-agent: Surfbot
|
|||
Disallow:/
|
||||
User-agent: SurveyBot
|
||||
Disallow:/
|
||||
User-agent: suzuran
|
||||
User-agent: Suzuran
|
||||
Disallow:/
|
||||
User-agent: Swiftbot
|
||||
Disallow:/
|
||||
User-agent: SISTRIX Crawler
|
||||
Disallow:/
|
||||
User-agent: Szukacz
|
||||
Disallow:/
|
||||
User-agent: T0PHackTeam
|
||||
Disallow:/
|
||||
User-agent: T8Abot
|
||||
Disallow:/
|
||||
User-agent: tAkeOut
|
||||
|
|
@ -684,7 +748,7 @@ User-agent: TightTwatBot
|
|||
Disallow:/
|
||||
User-agent: Titan
|
||||
Disallow:/
|
||||
User-agent: T0PHackTeam
|
||||
User-agent: Toata
|
||||
Disallow:/
|
||||
User-agent: Toweyabot
|
||||
Disallow:/
|
||||
|
|
@ -694,14 +758,18 @@ User-agent: True_Robot
|
|||
Disallow:/
|
||||
User-agent: Turingos
|
||||
Disallow:/
|
||||
User-agent: TurnitinBot
|
||||
User-agent: Turnitin Bot
|
||||
Disallow:/
|
||||
User-agent: Turnitin Robot
|
||||
Disallow:/
|
||||
User-agent: Turnitin Bot
|
||||
User-agent: TurnitinBot
|
||||
Disallow:/
|
||||
User-agent: TwengaBot
|
||||
Disallow:/
|
||||
User-agent: Twice
|
||||
Disallow:/
|
||||
User-agent: Typhoeus
|
||||
Disallow:/
|
||||
User-agent: UptimeRobot/2.0
|
||||
Disallow:/
|
||||
User-agent: URLy.Warning
|
||||
|
|
@ -710,6 +778,8 @@ User-agent: URLy Warning
|
|||
Disallow:/
|
||||
User-agent: Vacuum
|
||||
Disallow:/
|
||||
User-agent: Vagabondo
|
||||
Disallow:/
|
||||
User-agent: VB Project
|
||||
Disallow:/
|
||||
User-agent: VCI
|
||||
|
|
@ -722,82 +792,94 @@ User-agent: VoidEYE
|
|||
Disallow:/
|
||||
User-agent: Voil
|
||||
Disallow:/
|
||||
User-agent: Voltron
|
||||
Disallow:/
|
||||
User-agent: WallpapersHD
|
||||
Disallow:/
|
||||
User-agent: WASALive-Bot
|
||||
Disallow:/
|
||||
User-agent: WBSearchBot
|
||||
Disallow:/
|
||||
User-agent: Web Auto
|
||||
Disallow:/
|
||||
User-agent: Web Bandit
|
||||
Disallow:/
|
||||
User-agent: Web Collage
|
||||
Disallow:/
|
||||
User-agent: Web Copier
|
||||
Disallow:/
|
||||
User-agent: Web Enhancer
|
||||
Disallow:/
|
||||
User-agent: Web Fetch
|
||||
Disallow:/
|
||||
User-agent: Web Fuck
|
||||
Disallow:/
|
||||
User-agent: Web Image Collector
|
||||
Disallow:/
|
||||
User-agent: Web Pix
|
||||
Disallow:/
|
||||
User-agent: Web Reaper
|
||||
Disallow:/
|
||||
User-agent: Web Sauger
|
||||
Disallow:/
|
||||
User-agent: Web Stripper
|
||||
Disallow:/
|
||||
User-agent: Web Sucker
|
||||
Disallow:/
|
||||
User-agent: Web Whacker
|
||||
Disallow:/
|
||||
User-agent: Webalta
|
||||
Disallow:/
|
||||
User-agent: WebAuto
|
||||
Disallow:/
|
||||
User-agent: WebBandit
|
||||
Disallow:/
|
||||
User-agent: Web Bandit
|
||||
User-agent: WebCollage
|
||||
Disallow:/
|
||||
User-agent: WebCopier
|
||||
Disallow:/
|
||||
User-agent: Web Copier
|
||||
Disallow:/
|
||||
User-agent: WEBDAV
|
||||
Disallow:/
|
||||
User-agent: WEBDAV Client
|
||||
Disallow:/
|
||||
User-agent: WebEnhancer
|
||||
Disallow:/
|
||||
User-agent: Web Enhancer
|
||||
Disallow:/
|
||||
User-agent: WebFetch
|
||||
Disallow:/
|
||||
User-agent: Web Fetch
|
||||
Disallow:/
|
||||
User-agent: WebFuck
|
||||
Disallow:/
|
||||
User-agent: Web Fuck
|
||||
User-agent: WebImageCollector
|
||||
Disallow:/
|
||||
User-agent: WebGo IS
|
||||
Disallow:/
|
||||
User-agent: Web Image Collector
|
||||
Disallow:/
|
||||
User-agent: WebLeacher
|
||||
Disallow:/
|
||||
User-agent: WebmasterWorldForumBot
|
||||
Disallow:/
|
||||
User-agent: WebPix
|
||||
Disallow:/
|
||||
User-agent: Web Pix
|
||||
Disallow:/
|
||||
User-agent: WebReaper
|
||||
Disallow:/
|
||||
User-agent: Web Reaper
|
||||
Disallow:/
|
||||
User-agent: WebSauger
|
||||
Disallow:/
|
||||
User-agent: Web Sauger
|
||||
Disallow:/
|
||||
User-agent: webshag
|
||||
Disallow:/
|
||||
User-agent: WebsiteExtractor
|
||||
User-agent: Webshag
|
||||
Disallow:/
|
||||
User-agent: Website Extractor
|
||||
Disallow:/
|
||||
User-agent: WebsiteQuester
|
||||
Disallow:/
|
||||
User-agent: Website Quester
|
||||
Disallow:/
|
||||
User-agent: WebsiteExtractor
|
||||
Disallow:/
|
||||
User-agent: WebsiteQuester
|
||||
Disallow:/
|
||||
User-agent: Webster
|
||||
Disallow:/
|
||||
User-agent: WebStripper
|
||||
Disallow:/
|
||||
User-agent: Web Stripper
|
||||
Disallow:/
|
||||
User-agent: WebSucker
|
||||
Disallow:/
|
||||
User-agent: Web Sucker
|
||||
Disallow:/
|
||||
User-agent: WebWhacker
|
||||
Disallow:/
|
||||
User-agent: Web Whacker
|
||||
Disallow:/
|
||||
User-agent: WebZIP
|
||||
Disallow:/
|
||||
User-agent: WeSEE
|
||||
|
|
@ -806,7 +888,7 @@ User-agent: Whack
|
|||
Disallow:/
|
||||
User-agent: Whacker
|
||||
Disallow:/
|
||||
User-agent: whatweb
|
||||
User-agent: Whatweb
|
||||
Disallow:/
|
||||
User-agent: Widow
|
||||
Disallow:/
|
||||
|
|
@ -816,19 +898,21 @@ User-agent: WiseGuys Robot
|
|||
Disallow:/
|
||||
User-agent: WISENutbot
|
||||
Disallow:/
|
||||
User-agent: wonderbot
|
||||
User-agent: Wonderbot
|
||||
Disallow:/
|
||||
User-agent: woobot
|
||||
User-agent: Woobot
|
||||
Disallow:/
|
||||
User-agent: Wotbox
|
||||
Disallow:/
|
||||
User-agent: wprecon
|
||||
User-agent: Wprecon
|
||||
Disallow:/
|
||||
User-agent: WWWOFFLE
|
||||
User-agent: WWW-Collector-E
|
||||
Disallow:/
|
||||
User-agent: WWW-Mechanize
|
||||
Disallow:/
|
||||
User-agent: WWW::Mechanize
|
||||
Disallow:/
|
||||
User-agent: WWW-Collector-E
|
||||
User-agent: WWWOFFLE
|
||||
Disallow:/
|
||||
User-agent: Xaldon_WebSpider
|
||||
Disallow:/
|
||||
|
|
@ -842,11 +926,13 @@ User-agent: YoudaoBot
|
|||
Disallow:/
|
||||
User-agent: Zade
|
||||
Disallow:/
|
||||
User-agent: Zermelo
|
||||
Disallow:/
|
||||
User-agent: Zeus
|
||||
Disallow:/
|
||||
User-agent: zgrab
|
||||
User-agent: Zgrab
|
||||
Disallow:/
|
||||
User-agent: zitebot
|
||||
User-agent: Zitebot
|
||||
Disallow:/
|
||||
User-agent: ZmEu
|
||||
Disallow:/
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue