diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 73a0a1859..5245272f3 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -6,7 +6,7 @@ ### Version Information # ******************************************** -#### Version: V3.2017.07.643 +#### Version: V3.2017.07.644 #### Bad Referrer Count: 5030 #### Bad Bot Count: 481 ******************************************** diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index e8d38f93e..0474e5043 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -6,7 +6,7 @@ ### Version Information # ******************************************** -#### Version: V3.2017.07.643 +#### Version: V3.2017.07.644 #### Bad Referrer Count: 5030 #### Bad Bot Count: 481 ******************************************** diff --git a/README.md b/README.md index 0c849a688..153666e3c 100755 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ ### Version Information # ******************************************** -#### Version: V3.2017.07.643 +#### Version: V3.2017.07.644 #### Bad Referrer Count: 5030 #### Bad Bot Count: 481 ******************************************** diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index d5547ce7a..30dbea841 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.07.643 -### Updated: Fri Jul 7 11:11:23 SAST 2017 +### Version: V3.2017.07.644 +### Updated: Fri Jul 7 11:55:01 SAST 2017 ### Bad Referrer Count: 5029 ### Bad Bot Count: 481 ################################################### @@ -118,35 +118,35 @@ map $http_user_agent $bad_bot { # *********************************************** # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~adidxbot" 0; - "~AdsBot-Google" 0; - "~aolbuild" 0; - "~bingbot" 0; - "~bingpreview" 0; - "~DoCoMo" 0; - "~duckduckgo" 0; - "~facebookexternalhit" 0; - "~Feedfetcher-Google" 0; - "~Googlebot" 0; - "~Googlebot-Image" 0; - "~Googlebot-Mobile" 0; - "~Googlebot-News" 0; - "~Googlebot/Test" 0; - "~Googlebot-Video" 0; - "~Google-HTTP-Java-Client" 0; - "~gsa-crawler" 0; - "~Jakarta\ Commons" 0; - "~Kraken/0.1" 0; - "~LinkedInBot" 0; - "~Mediapartners-Google" 0; - "~msnbot" 0; - "~msnbot-media" 0; - "~SAMSUNG" 0; - "~slurp" 0; - "~teoma" 0; - "~TwitterBot" 0; - "~Wordpress" 0; - "~yahoo" 0; + "~*adidxbot" 0; + "~*AdsBot-Google" 0; + "~*aolbuild" 0; + "~*bingbot" 0; + "~*bingpreview" 0; + "~*DoCoMo" 0; + "~*duckduckgo" 0; + "~*facebookexternalhit" 0; + "~*Feedfetcher-Google" 0; + "~*Googlebot" 0; + "~*Googlebot-Image" 0; + "~*Googlebot-Mobile" 0; + "~*Googlebot-News" 0; + "~*Googlebot/Test" 0; + "~*Googlebot-Video" 0; + "~*Google-HTTP-Java-Client" 0; + "~*gsa-crawler" 0; + "~*Jakarta\ Commons" 0; + "~*Kraken/0.1" 0; + "~*LinkedInBot" 0; + "~*Mediapartners-Google" 0; + "~*msnbot" 0; + "~*msnbot-media" 0; + "~*SAMSUNG" 0; + "~*slurp" 0; + "~*teoma" 0; + "~*TwitterBot" 0; + "~*Wordpress" 0; + "~*yahoo" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # *************************************************** @@ -157,11 +157,11 @@ map $http_user_agent $bad_bot { # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~jetmon" 1; - "~libwww-perl" 1; - "~Lynx" 1; - "~munin" 1; - "~Wget/1.15" 1; + "~*jetmon" 1; + "~*libwww-perl" 1; + "~*Lynx" 1; + "~*munin" 1; + "~*Wget/1.15" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************************** @@ -169,24 +169,24 @@ map $http_user_agent $bad_bot { # ************************************************************** # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~Alexa" 2; - "~archive.org" 2; - "~Baidu" 2; - "~BUbiNG" 2; - "~FlipboardProxy" 2; - "~ia_archiver" 2; - "~Mozilla/4.0" 2; - "~MSIE\ 7.0" 2; - "~Presto" 2; - "~R6_CommentReader" 2; - "~R6_FeedFetcher" 2; - "~RPT-HTTPClient" 2; - "~SeznamBot" 2; - "~sfFeedReader/0.9" 2; - "~Spaidu" 2; - "~UptimeRobot/2.0" 2; - "~YandexBot" 2; - "~YandexImages" 2; + "~*Alexa" 2; + "~*archive.org" 2; + "~*Baidu" 2; + "~*BUbiNG" 2; + "~*FlipboardProxy" 2; + "~*ia_archiver" 2; + "~*Mozilla/4.0" 2; + "~*MSIE\ 7.0" 2; + "~*Presto" 2; + "~*R6_CommentReader" 2; + "~*R6_FeedFetcher" 2; + "~*RPT-HTTPClient" 2; + "~*SeznamBot" 2; + "~*sfFeedReader/0.9" 2; + "~*Spaidu" 2; + "~*UptimeRobot/2.0" 2; + "~*YandexBot" 2; + "~*YandexImages" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ********************************************* @@ -197,487 +197,487 @@ map $http_user_agent $bad_bot { # Known Vulnerability Scanners (now merged into one section) # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~360Spider" 3; - "~80legs" 3; - "~Abonti" 3; - "~Aboundex" 3; - "~Acunetix" 3; - "~ADmantX" 3; - "~AhrefsBot" 3; - "~AIBOT" 3; - "~AiHitBot" 3; - "~Aipbot" 3; - "~Alexibot" 3; - "~Alligator" 3; - "~AllSubmitter" 3; - "~Anarchie" 3; - "~Apexoo" 3; - "~ASPSeek" 3; - "~Asterias" 3; - "~Attach" 3; - "~autoemailspider" 3; - "~BackDoorBot" 3; - "~BackStreet" 3; - "~BackWeb" 3; - "~Badass" 3; - "~Bandit" 3; - "~BatchFTP" 3; - "~Battleztar\ Bazinga" 3; - "~BBBike" 3; - "~BDFetch" 3; - "~BetaBot" 3; - "~Bigfoot" 3; - "~Bitacle" 3; - "~Blackboard" 3; - "~Black\ Hole" 3; - "~BlackWidow" 3; - "~BLEXBot" 3; - "~Blow" 3; - "~BlowFish" 3; - "~Boardreader" 3; - "~Bolt" 3; - "~BotALot" 3; - "~Brandprotect" 3; - "~Brandwatch" 3; - "~Bubing" 3; - "~Buddy" 3; - "~BuiltBotTough" 3; - "~BuiltWith" 3; - "~Bullseye" 3; - "~BunnySlippers" 3; - "~BuzzSumo" 3; - "~Calculon" 3; - "~CATExplorador" 3; - "~CazoodleBot" 3; - "~CCBot" 3; - "~Cegbfeieh" 3; - "~CheeseBot" 3; - "~CherryPicker" 3; - "~ChinaClaw" 3; - "~Chlooe" 3; - "~Claritybot" 3; - "~Cliqzbot" 3; - "~Cogentbot" 3; - "~cognitiveseo" 3; - "~Collector" 3; - "~Copier" 3; - "~CopyRightCheck" 3; - "~Copyscape" 3; - "~Cosmos" 3; - "~Craftbot" 3; - "~CrazyWebCrawler" 3; - "~Crescent" 3; - "~CSHttp" 3; - "~Curious" 3; - "~Custo" 3; - "~DatabaseDriverMysqli" 3; - "~DBLBot" 3; - "~Demon" 3; - "~Deusu" 3; - "~Devil" 3; - "~DIIbot" 3; - "~Dirbuster" 3; - "~Disco" 3; - "~Discobot" 3; - "~Discoverybot" 3; - "~DittoSpyder" 3; - "~DomainAppender" 3; - "~DomainCrawler" 3; - "~DomainSigmaCrawler" 3; - "~Dotbot" 3; - "~Download\ Demon" 3; - "~Download\ Devil" 3; - "~Download\ Wonder" 3; - "~Dragonfly" 3; - "~Drip" 3; - "~DTS\ Agent" 3; - "~EasyDL" 3; - "~Ebingbong" 3; - "~eCatch" 3; - "~ECCP/1.0" 3; - "~Ecxi" 3; - "~EirGrabber" 3; - "~EMail\ Collector" 3; - "~EMail\ Extractor" 3; - "~EMail\ Siphon" 3; - "~EMail\ Wolf" 3; - "~EroCrawler" 3; - "~Evil" 3; - "~Exabot" 3; - "~Express\ WebPictures" 3; - "~Extractor" 3; - "~ExtractorPro" 3; - "~EyeNetIE" 3; - "~Ezooms" 3; - "~FHscan" 3; - "~Fimap" 3; - "~Findxbot" 3; - "~Firefox/7.0" 3; - "~FlashGet" 3; - "~Flunky" 3; - "~Foobot" 3; - "~Freeuploader" 3; - "~FrontPage" 3; - "~Fyrebot" 3; - "~GalaxyBot" 3; - "~Genieo" 3; - "~Getintent" 3; - "~GetRight" 3; - "~GetWeb" 3; - "~Gigablast" 3; - "~Gigabot" 3; - "~Go-Ahead-Got-It" 3; - "~Gotit" 3; - "~GoZilla" 3; - "~Go!Zilla" 3; - "~Grabber" 3; - "~GrabNet" 3; - "~Grafula" 3; - "~GrapeFX" 3; - "~GrapeshotCrawler" 3; - "~GridBot" 3; - "~GT::WWW" 3; - "~HaosouSpider" 3; - "~Harvest" 3; - "~Havij" 3; - "~HEADMasterSEO" 3; - "~Heritrix" 3; - "~Hloader" 3; - "~HMView" 3; - "~HTMLparser" 3; - "~HTTP::Lite" 3; - "~HTTrack" 3; - "~Humanlinks" 3; - "~HybridBot" 3; - "~Iblog" 3; - "~IDBot" 3; - "~Id-search" 3; - "~IlseBot" 3; - "~Image\ Fetch" 3; - "~Image\ Stripper" 3; - "~Image\ Sucker" 3; - "~Indy\ Library" 3; - "~InfoNaviRobot" 3; - "~InfoTekies" 3; - "~Intelliseek" 3; - "~InterGET" 3; - "~Internet\ Ninja" 3; - "~InternetSeer" 3; - "~internetVista\ monitor" 3; - "~Iria" 3; - "~IRLbot" 3; - "~Iskanie" 3; - "~JamesBOT" 3; - "~Jbrofuzz" 3; - "~JennyBot" 3; - "~JetCar" 3; - "~JikeSpider" 3; - "~JOC\ Web\ Spider" 3; - "~Joomla" 3; - "~JustView" 3; - "~Jyxobot" 3; - "~Kenjin\ Spider" 3; - "~Keyword\ Density" 3; - "~Lanshanbot" 3; - "~Larbin" 3; - "~LeechFTP" 3; - "~LeechGet" 3; - "~LexiBot" 3; - "~Lftp" 3; - "~LibWeb" 3; - "~Libwhisker" 3; - "~Lightspeedsystems" 3; - "~Likse" 3; - "~Linkdexbot" 3; - "~LinkextractorPro" 3; - "~LinkpadBot" 3; - "~LinkScan" 3; - "~LinksManager" 3; - "~LinkWalker" 3; - "~LinqiaMetadataDownloaderBot" 3; - "~LinqiaRSSBot" 3; - "~LinqiaScrapeBot" 3; - "~Lipperhey" 3; - "~Litemage_walker" 3; - "~Lmspider" 3; - "~LNSpiderguy" 3; - "~Ltx71" 3; - "~lwp-request" 3; - "~LWP::Simple" 3; - "~lwp-trivial" 3; - "~Magnet" 3; - "~Mag-Net" 3; - "~magpie-crawler" 3; - "~Mail.ru" 3; - "~Majestic12" 3; - "~MarkMonitor" 3; - "~MarkWatch" 3; - "~Masscan" 3; - "~Mass\ Downloader" 3; - "~Mata\ Hari" 3; - "~Meanpathbot" 3; - "~MegaIndex.ru" 3; - "~Metauri" 3; - "~MFC_Tear_Sample" 3; - "~Microsoft\ Data\ Access" 3; - "~Microsoft\ URL\ Control" 3; - "~MIDown\ tool" 3; - "~MIIxpc" 3; - "~Mister\ PiX" 3; - "~MJ12bot" 3; - "~Mojeek" 3; - "~MSFrontPage" 3; - "~MSIE\ 6.0" 3; - "~MSIECrawler" 3; - "~Msrabot" 3; - "~MS\ Web\ Services\ Client\ Protocol" 3; - "~Musobot" 3; - "~Name\ Intelligence" 3; - "~Nameprotect" 3; - "~Navroad" 3; - "~NearSite" 3; - "~Needle" 3; - "~Nessus" 3; - "~NetAnts" 3; - "~Netcraft" 3; - "~netEstate\ NE\ Crawler" 3; - "~NetLyzer" 3; - "~NetMechanic" 3; - "~NetSpider" 3; - "~Nettrack" 3; - "~Net\ Vampire" 3; - "~NetZIP" 3; - "~NextGenSearchBot" 3; - "~Nibbler" 3; - "~NICErsPRO" 3; - "~Niki-bot" 3; - "~Nikto" 3; - "~NimbleCrawler" 3; - "~Ninja" 3; - "~Nmap" 3; - "~NPbot" 3; - "~Nutch" 3; - "~Octopus" 3; - "~Offline\ Explorer" 3; - "~Offline\ Navigator" 3; - "~Openfind" 3; - "~OpenLinkProfiler" 3; - "~Openvas" 3; - "~OrangeBot" 3; - "~OrangeSpider" 3; - "~OutfoxBot" 3; - "~PageAnalyzer" 3; - "~Page\ Analyzer" 3; - "~PageGrabber" 3; - "~Page\ Grabber" 3; - "~page\ scorer" 3; - "~PageScorer" 3; - "~Panscient" 3; - "~Papa\ Foto" 3; - "~Pavuk" 3; - "~pcBrowser" 3; - "~PECL::HTTP" 3; - "~PeoplePal" 3; - "~PHPCrawl" 3; - "~Picscout" 3; - "~Picsearch" 3; - "~Pimonster" 3; - "~Pi-Monster" 3; - "~Pixray" 3; - "~PleaseCrawl" 3; - "~Pockey" 3; - "~POE-Component-Client-HTTP" 3; - "~Probethenet" 3; - "~ProPowerBot" 3; - "~ProWebWalker" 3; - "~Proximic" 3; - "~Psbot" 3; - "~Pump" 3; - "~PyCurl" 3; - "~QueryN\ Metasearch" 3; - "~Qwantify" 3; - "~RankActiveLinkBot" 3; - "~Rankivabot" 3; - "~RealDownload" 3; - "~Reaper" 3; - "~Recorder" 3; - "~RedesScrapy" 3; - "~ReGet" 3; - "~RepoMonkey" 3; - "~Ripper" 3; - "~RocketCrawler" 3; - "~Rogerbot" 3; - "~SalesIntelligent" 3; - "~SBIder" 3; - "~ScanAlert" 3; - "~Scanbot" 3; - "~Scrapy" 3; - "~Screaming" 3; - "~Screaming\ Frog\ SEO\ Spider" 3; - "~ScreenerBot" 3; - "~Searchestate" 3; - "~SearchmetricsBot" 3; - "~Semrush" 3; - "~SemrushBot" 3; - "~SEOkicks" 3; - "~SEOkicks-Robot" 3; - "~SEOlyticsCrawler" 3; - "~Seomoz" 3; - "~SEOprofiler" 3; - "~seoscanners" 3; - "~SEOstats" 3; - "~Siphon" 3; - "~SISTRIX" 3; - "~SISTRIX\ Crawler" 3; - "~Sitebeam" 3; - "~SiteExplorer" 3; - "~Siteimprove" 3; - "~SiteLockSpider" 3; - "~SiteSnagger" 3; - "~SiteSucker" 3; - "~Site\ Sucker" 3; - "~Sitevigil" 3; - "~Slackbot-LinkExpanding" 3; - "~SlySearch" 3; - "~SmartDownload" 3; - "~Snake" 3; - "~Snapbot" 3; - "~Snoopy" 3; - "~SocialRankIOBot" 3; - "~Sogou\ web\ spider" 3; - "~Sosospider" 3; - "~SpaceBison" 3; - "~Spammen" 3; - "~SpankBot" 3; - "~Spanner" 3; - "~Spbot" 3; - "~Spinn3r" 3; - "~SputnikBot" 3; - "~Sqlmap" 3; - "~Sqlworm" 3; - "~Sqworm" 3; - "~Steeler" 3; - "~Stripper" 3; - "~Sucker" 3; - "~Sucuri" 3; - "~SuperBot" 3; - "~SuperHTTP" 3; - "~Surfbot" 3; - "~SurveyBot" 3; - "~Suzuran" 3; - "~Swiftbot" 3; - "~Szukacz" 3; - "~T0PHackTeam" 3; - "~T8Abot" 3; - "~tAkeOut" 3; - "~Teleport" 3; - "~TeleportPro" 3; - "~Telesoft" 3; - "~Telesphoreo" 3; - "~Telesphorep" 3; - "~The\ Intraformant" 3; - "~TheNomad" 3; - "~TightTwatBot" 3; - "~Titan" 3; - "~Toata" 3; - "~Toweyabot" 3; - "~Trendictionbot" 3; - "~True_Robot" 3; - "~Turingos" 3; - "~TurnitinBot" 3; - "~Turnitin\ Bot" 3; - "~Turnitin\ Robot" 3; - "~TwengaBot" 3; - "~Twice" 3; - "~Typhoeus" 3; - "~UnisterBot" 3; - "~URLy.Warning" 3; - "~URLy\ Warning" 3; - "~Vacuum" 3; - "~Vagabondo" 3; - "~VB\ Project" 3; - "~VCI" 3; - "~VeriCiteCrawler" 3; - "~VidibleScraper" 3; - "~VoidEYE" 3; - "~Voil" 3; - "~Voltron" 3; - "~Wallpapers/3.0" 3; - "~WallpapersHD" 3; - "~WASALive-Bot" 3; - "~WBSearchBot" 3; - "~Webalta" 3; - "~WebAuto" 3; - "~Web\ Auto" 3; - "~WebBandit" 3; - "~Web\ Bandit" 3; - "~WebCollage" 3; - "~Web\ Collage" 3; - "~WebCopier" 3; - "~Web\ Copier" 3; - "~WEBDAV" 3; - "~WEBDAV\ Client" 3; - "~WebEnhancer" 3; - "~Web\ Enhancer" 3; - "~WebFetch" 3; - "~Web\ Fetch" 3; - "~WebFuck" 3; - "~Web\ Fuck" 3; - "~WebGo\ IS" 3; - "~WebImageCollector" 3; - "~Web\ Image\ Collector" 3; - "~WebLeacher" 3; - "~WebmasterWorldForumBot" 3; - "~webmeup-crawler" 3; - "~WebPix" 3; - "~Web\ Pix" 3; - "~WebReaper" 3; - "~Web\ Reaper" 3; - "~WebSauger" 3; - "~Web\ Sauger" 3; - "~Webshag" 3; - "~WebsiteExtractor" 3; - "~Website\ Extractor" 3; - "~WebsiteQuester" 3; - "~Website\ Quester" 3; - "~Webster" 3; - "~WebStripper" 3; - "~Web\ Stripper" 3; - "~WebSucker" 3; - "~Web\ Sucker" 3; - "~WebWhacker" 3; - "~Web\ Whacker" 3; - "~WebZIP" 3; - "~WeSEE" 3; - "~Whack" 3; - "~Whacker" 3; - "~Whatweb" 3; - "~Widow" 3; - "~WinHTTrack" 3; - "~WiseGuys\ Robot" 3; - "~WISENutbot" 3; - "~Wonderbot" 3; - "~Woobot" 3; - "~Wotbox" 3; - "~Wprecon" 3; - "~WPScan" 3; - "~WWW-Collector-E" 3; - "~WWW-Mechanize" 3; - "~WWW::Mechanize" 3; - "~WWWOFFLE" 3; - "~x22Mozilla" 3; - "~Xaldon_WebSpider" 3; - "~Xaldon\ WebSpider" 3; - "~Xenu" 3; - "~YoudaoBot" 3; - "~Zade" 3; - "~Zermelo" 3; - "~Zeus" 3; - "~Zgrab" 3; - "~Zitebot" 3; - "~ZmEu" 3; - "~ZumBot" 3; - "~ZyBorg" 3; + "~*360Spider" 3; + "~*80legs" 3; + "~*Abonti" 3; + "~*Aboundex" 3; + "~*Acunetix" 3; + "~*ADmantX" 3; + "~*AhrefsBot" 3; + "~*AIBOT" 3; + "~*AiHitBot" 3; + "~*Aipbot" 3; + "~*Alexibot" 3; + "~*Alligator" 3; + "~*AllSubmitter" 3; + "~*Anarchie" 3; + "~*Apexoo" 3; + "~*ASPSeek" 3; + "~*Asterias" 3; + "~*Attach" 3; + "~*autoemailspider" 3; + "~*BackDoorBot" 3; + "~*BackStreet" 3; + "~*BackWeb" 3; + "~*Badass" 3; + "~*Bandit" 3; + "~*BatchFTP" 3; + "~*Battleztar\ Bazinga" 3; + "~*BBBike" 3; + "~*BDFetch" 3; + "~*BetaBot" 3; + "~*Bigfoot" 3; + "~*Bitacle" 3; + "~*Blackboard" 3; + "~*Black\ Hole" 3; + "~*BlackWidow" 3; + "~*BLEXBot" 3; + "~*Blow" 3; + "~*BlowFish" 3; + "~*Boardreader" 3; + "~*Bolt" 3; + "~*BotALot" 3; + "~*Brandprotect" 3; + "~*Brandwatch" 3; + "~*Bubing" 3; + "~*Buddy" 3; + "~*BuiltBotTough" 3; + "~*BuiltWith" 3; + "~*Bullseye" 3; + "~*BunnySlippers" 3; + "~*BuzzSumo" 3; + "~*Calculon" 3; + "~*CATExplorador" 3; + "~*CazoodleBot" 3; + "~*CCBot" 3; + "~*Cegbfeieh" 3; + "~*CheeseBot" 3; + "~*CherryPicker" 3; + "~*ChinaClaw" 3; + "~*Chlooe" 3; + "~*Claritybot" 3; + "~*Cliqzbot" 3; + "~*Cogentbot" 3; + "~*cognitiveseo" 3; + "~*Collector" 3; + "~*Copier" 3; + "~*CopyRightCheck" 3; + "~*Copyscape" 3; + "~*Cosmos" 3; + "~*Craftbot" 3; + "~*CrazyWebCrawler" 3; + "~*Crescent" 3; + "~*CSHttp" 3; + "~*Curious" 3; + "~*Custo" 3; + "~*DatabaseDriverMysqli" 3; + "~*DBLBot" 3; + "~*Demon" 3; + "~*Deusu" 3; + "~*Devil" 3; + "~*DIIbot" 3; + "~*Dirbuster" 3; + "~*Disco" 3; + "~*Discobot" 3; + "~*Discoverybot" 3; + "~*DittoSpyder" 3; + "~*DomainAppender" 3; + "~*DomainCrawler" 3; + "~*DomainSigmaCrawler" 3; + "~*Dotbot" 3; + "~*Download\ Demon" 3; + "~*Download\ Devil" 3; + "~*Download\ Wonder" 3; + "~*Dragonfly" 3; + "~*Drip" 3; + "~*DTS\ Agent" 3; + "~*EasyDL" 3; + "~*Ebingbong" 3; + "~*eCatch" 3; + "~*ECCP/1.0" 3; + "~*Ecxi" 3; + "~*EirGrabber" 3; + "~*EMail\ Collector" 3; + "~*EMail\ Extractor" 3; + "~*EMail\ Siphon" 3; + "~*EMail\ Wolf" 3; + "~*EroCrawler" 3; + "~*Evil" 3; + "~*Exabot" 3; + "~*Express\ WebPictures" 3; + "~*Extractor" 3; + "~*ExtractorPro" 3; + "~*EyeNetIE" 3; + "~*Ezooms" 3; + "~*FHscan" 3; + "~*Fimap" 3; + "~*Findxbot" 3; + "~*Firefox/7.0" 3; + "~*FlashGet" 3; + "~*Flunky" 3; + "~*Foobot" 3; + "~*Freeuploader" 3; + "~*FrontPage" 3; + "~*Fyrebot" 3; + "~*GalaxyBot" 3; + "~*Genieo" 3; + "~*Getintent" 3; + "~*GetRight" 3; + "~*GetWeb" 3; + "~*Gigablast" 3; + "~*Gigabot" 3; + "~*Go-Ahead-Got-It" 3; + "~*Gotit" 3; + "~*GoZilla" 3; + "~*Go!Zilla" 3; + "~*Grabber" 3; + "~*GrabNet" 3; + "~*Grafula" 3; + "~*GrapeFX" 3; + "~*GrapeshotCrawler" 3; + "~*GridBot" 3; + "~*GT::WWW" 3; + "~*HaosouSpider" 3; + "~*Harvest" 3; + "~*Havij" 3; + "~*HEADMasterSEO" 3; + "~*Heritrix" 3; + "~*Hloader" 3; + "~*HMView" 3; + "~*HTMLparser" 3; + "~*HTTP::Lite" 3; + "~*HTTrack" 3; + "~*Humanlinks" 3; + "~*HybridBot" 3; + "~*Iblog" 3; + "~*IDBot" 3; + "~*Id-search" 3; + "~*IlseBot" 3; + "~*Image\ Fetch" 3; + "~*Image\ Stripper" 3; + "~*Image\ Sucker" 3; + "~*Indy\ Library" 3; + "~*InfoNaviRobot" 3; + "~*InfoTekies" 3; + "~*Intelliseek" 3; + "~*InterGET" 3; + "~*Internet\ Ninja" 3; + "~*InternetSeer" 3; + "~*internetVista\ monitor" 3; + "~*Iria" 3; + "~*IRLbot" 3; + "~*Iskanie" 3; + "~*JamesBOT" 3; + "~*Jbrofuzz" 3; + "~*JennyBot" 3; + "~*JetCar" 3; + "~*JikeSpider" 3; + "~*JOC\ Web\ Spider" 3; + "~*Joomla" 3; + "~*JustView" 3; + "~*Jyxobot" 3; + "~*Kenjin\ Spider" 3; + "~*Keyword\ Density" 3; + "~*Lanshanbot" 3; + "~*Larbin" 3; + "~*LeechFTP" 3; + "~*LeechGet" 3; + "~*LexiBot" 3; + "~*Lftp" 3; + "~*LibWeb" 3; + "~*Libwhisker" 3; + "~*Lightspeedsystems" 3; + "~*Likse" 3; + "~*Linkdexbot" 3; + "~*LinkextractorPro" 3; + "~*LinkpadBot" 3; + "~*LinkScan" 3; + "~*LinksManager" 3; + "~*LinkWalker" 3; + "~*LinqiaMetadataDownloaderBot" 3; + "~*LinqiaRSSBot" 3; + "~*LinqiaScrapeBot" 3; + "~*Lipperhey" 3; + "~*Litemage_walker" 3; + "~*Lmspider" 3; + "~*LNSpiderguy" 3; + "~*Ltx71" 3; + "~*lwp-request" 3; + "~*LWP::Simple" 3; + "~*lwp-trivial" 3; + "~*Magnet" 3; + "~*Mag-Net" 3; + "~*magpie-crawler" 3; + "~*Mail.ru" 3; + "~*Majestic12" 3; + "~*MarkMonitor" 3; + "~*MarkWatch" 3; + "~*Masscan" 3; + "~*Mass\ Downloader" 3; + "~*Mata\ Hari" 3; + "~*Meanpathbot" 3; + "~*MegaIndex.ru" 3; + "~*Metauri" 3; + "~*MFC_Tear_Sample" 3; + "~*Microsoft\ Data\ Access" 3; + "~*Microsoft\ URL\ Control" 3; + "~*MIDown\ tool" 3; + "~*MIIxpc" 3; + "~*Mister\ PiX" 3; + "~*MJ12bot" 3; + "~*Mojeek" 3; + "~*MSFrontPage" 3; + "~*MSIE\ 6.0" 3; + "~*MSIECrawler" 3; + "~*Msrabot" 3; + "~*MS\ Web\ Services\ Client\ Protocol" 3; + "~*Musobot" 3; + "~*Name\ Intelligence" 3; + "~*Nameprotect" 3; + "~*Navroad" 3; + "~*NearSite" 3; + "~*Needle" 3; + "~*Nessus" 3; + "~*NetAnts" 3; + "~*Netcraft" 3; + "~*netEstate\ NE\ Crawler" 3; + "~*NetLyzer" 3; + "~*NetMechanic" 3; + "~*NetSpider" 3; + "~*Nettrack" 3; + "~*Net\ Vampire" 3; + "~*NetZIP" 3; + "~*NextGenSearchBot" 3; + "~*Nibbler" 3; + "~*NICErsPRO" 3; + "~*Niki-bot" 3; + "~*Nikto" 3; + "~*NimbleCrawler" 3; + "~*Ninja" 3; + "~*Nmap" 3; + "~*NPbot" 3; + "~*Nutch" 3; + "~*Octopus" 3; + "~*Offline\ Explorer" 3; + "~*Offline\ Navigator" 3; + "~*Openfind" 3; + "~*OpenLinkProfiler" 3; + "~*Openvas" 3; + "~*OrangeBot" 3; + "~*OrangeSpider" 3; + "~*OutfoxBot" 3; + "~*PageAnalyzer" 3; + "~*Page\ Analyzer" 3; + "~*PageGrabber" 3; + "~*Page\ Grabber" 3; + "~*page\ scorer" 3; + "~*PageScorer" 3; + "~*Panscient" 3; + "~*Papa\ Foto" 3; + "~*Pavuk" 3; + "~*pcBrowser" 3; + "~*PECL::HTTP" 3; + "~*PeoplePal" 3; + "~*PHPCrawl" 3; + "~*Picscout" 3; + "~*Picsearch" 3; + "~*Pimonster" 3; + "~*Pi-Monster" 3; + "~*Pixray" 3; + "~*PleaseCrawl" 3; + "~*Pockey" 3; + "~*POE-Component-Client-HTTP" 3; + "~*Probethenet" 3; + "~*ProPowerBot" 3; + "~*ProWebWalker" 3; + "~*Proximic" 3; + "~*Psbot" 3; + "~*Pump" 3; + "~*PyCurl" 3; + "~*QueryN\ Metasearch" 3; + "~*Qwantify" 3; + "~*RankActiveLinkBot" 3; + "~*Rankivabot" 3; + "~*RealDownload" 3; + "~*Reaper" 3; + "~*Recorder" 3; + "~*RedesScrapy" 3; + "~*ReGet" 3; + "~*RepoMonkey" 3; + "~*Ripper" 3; + "~*RocketCrawler" 3; + "~*Rogerbot" 3; + "~*SalesIntelligent" 3; + "~*SBIder" 3; + "~*ScanAlert" 3; + "~*Scanbot" 3; + "~*Scrapy" 3; + "~*Screaming" 3; + "~*Screaming\ Frog\ SEO\ Spider" 3; + "~*ScreenerBot" 3; + "~*Searchestate" 3; + "~*SearchmetricsBot" 3; + "~*Semrush" 3; + "~*SemrushBot" 3; + "~*SEOkicks" 3; + "~*SEOkicks-Robot" 3; + "~*SEOlyticsCrawler" 3; + "~*Seomoz" 3; + "~*SEOprofiler" 3; + "~*seoscanners" 3; + "~*SEOstats" 3; + "~*Siphon" 3; + "~*SISTRIX" 3; + "~*SISTRIX\ Crawler" 3; + "~*Sitebeam" 3; + "~*SiteExplorer" 3; + "~*Siteimprove" 3; + "~*SiteLockSpider" 3; + "~*SiteSnagger" 3; + "~*SiteSucker" 3; + "~*Site\ Sucker" 3; + "~*Sitevigil" 3; + "~*Slackbot-LinkExpanding" 3; + "~*SlySearch" 3; + "~*SmartDownload" 3; + "~*Snake" 3; + "~*Snapbot" 3; + "~*Snoopy" 3; + "~*SocialRankIOBot" 3; + "~*Sogou\ web\ spider" 3; + "~*Sosospider" 3; + "~*SpaceBison" 3; + "~*Spammen" 3; + "~*SpankBot" 3; + "~*Spanner" 3; + "~*Spbot" 3; + "~*Spinn3r" 3; + "~*SputnikBot" 3; + "~*Sqlmap" 3; + "~*Sqlworm" 3; + "~*Sqworm" 3; + "~*Steeler" 3; + "~*Stripper" 3; + "~*Sucker" 3; + "~*Sucuri" 3; + "~*SuperBot" 3; + "~*SuperHTTP" 3; + "~*Surfbot" 3; + "~*SurveyBot" 3; + "~*Suzuran" 3; + "~*Swiftbot" 3; + "~*Szukacz" 3; + "~*T0PHackTeam" 3; + "~*T8Abot" 3; + "~*tAkeOut" 3; + "~*Teleport" 3; + "~*TeleportPro" 3; + "~*Telesoft" 3; + "~*Telesphoreo" 3; + "~*Telesphorep" 3; + "~*The\ Intraformant" 3; + "~*TheNomad" 3; + "~*TightTwatBot" 3; + "~*Titan" 3; + "~*Toata" 3; + "~*Toweyabot" 3; + "~*Trendictionbot" 3; + "~*True_Robot" 3; + "~*Turingos" 3; + "~*TurnitinBot" 3; + "~*Turnitin\ Bot" 3; + "~*Turnitin\ Robot" 3; + "~*TwengaBot" 3; + "~*Twice" 3; + "~*Typhoeus" 3; + "~*UnisterBot" 3; + "~*URLy.Warning" 3; + "~*URLy\ Warning" 3; + "~*Vacuum" 3; + "~*Vagabondo" 3; + "~*VB\ Project" 3; + "~*VCI" 3; + "~*VeriCiteCrawler" 3; + "~*VidibleScraper" 3; + "~*VoidEYE" 3; + "~*Voil" 3; + "~*Voltron" 3; + "~*Wallpapers/3.0" 3; + "~*WallpapersHD" 3; + "~*WASALive-Bot" 3; + "~*WBSearchBot" 3; + "~*Webalta" 3; + "~*WebAuto" 3; + "~*Web\ Auto" 3; + "~*WebBandit" 3; + "~*Web\ Bandit" 3; + "~*WebCollage" 3; + "~*Web\ Collage" 3; + "~*WebCopier" 3; + "~*Web\ Copier" 3; + "~*WEBDAV" 3; + "~*WEBDAV\ Client" 3; + "~*WebEnhancer" 3; + "~*Web\ Enhancer" 3; + "~*WebFetch" 3; + "~*Web\ Fetch" 3; + "~*WebFuck" 3; + "~*Web\ Fuck" 3; + "~*WebGo\ IS" 3; + "~*WebImageCollector" 3; + "~*Web\ Image\ Collector" 3; + "~*WebLeacher" 3; + "~*WebmasterWorldForumBot" 3; + "~*webmeup-crawler" 3; + "~*WebPix" 3; + "~*Web\ Pix" 3; + "~*WebReaper" 3; + "~*Web\ Reaper" 3; + "~*WebSauger" 3; + "~*Web\ Sauger" 3; + "~*Webshag" 3; + "~*WebsiteExtractor" 3; + "~*Website\ Extractor" 3; + "~*WebsiteQuester" 3; + "~*Website\ Quester" 3; + "~*Webster" 3; + "~*WebStripper" 3; + "~*Web\ Stripper" 3; + "~*WebSucker" 3; + "~*Web\ Sucker" 3; + "~*WebWhacker" 3; + "~*Web\ Whacker" 3; + "~*WebZIP" 3; + "~*WeSEE" 3; + "~*Whack" 3; + "~*Whacker" 3; + "~*Whatweb" 3; + "~*Widow" 3; + "~*WinHTTrack" 3; + "~*WiseGuys\ Robot" 3; + "~*WISENutbot" 3; + "~*Wonderbot" 3; + "~*Woobot" 3; + "~*Wotbox" 3; + "~*Wprecon" 3; + "~*WPScan" 3; + "~*WWW-Collector-E" 3; + "~*WWW-Mechanize" 3; + "~*WWW::Mechanize" 3; + "~*WWWOFFLE" 3; + "~*x22Mozilla" 3; + "~*Xaldon_WebSpider" 3; + "~*Xaldon\ WebSpider" 3; + "~*Xenu" 3; + "~*YoudaoBot" 3; + "~*Zade" 3; + "~*Zermelo" 3; + "~*Zeus" 3; + "~*Zgrab" 3; + "~*Zitebot" 3; + "~*ZmEu" 3; + "~*ZumBot" 3; + "~*ZyBorg" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index d5547ce7a..30dbea841 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.07.643 -### Updated: Fri Jul 7 11:11:23 SAST 2017 +### Version: V3.2017.07.644 +### Updated: Fri Jul 7 11:55:01 SAST 2017 ### Bad Referrer Count: 5029 ### Bad Bot Count: 481 ################################################### @@ -118,35 +118,35 @@ map $http_user_agent $bad_bot { # *********************************************** # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~adidxbot" 0; - "~AdsBot-Google" 0; - "~aolbuild" 0; - "~bingbot" 0; - "~bingpreview" 0; - "~DoCoMo" 0; - "~duckduckgo" 0; - "~facebookexternalhit" 0; - "~Feedfetcher-Google" 0; - "~Googlebot" 0; - "~Googlebot-Image" 0; - "~Googlebot-Mobile" 0; - "~Googlebot-News" 0; - "~Googlebot/Test" 0; - "~Googlebot-Video" 0; - "~Google-HTTP-Java-Client" 0; - "~gsa-crawler" 0; - "~Jakarta\ Commons" 0; - "~Kraken/0.1" 0; - "~LinkedInBot" 0; - "~Mediapartners-Google" 0; - "~msnbot" 0; - "~msnbot-media" 0; - "~SAMSUNG" 0; - "~slurp" 0; - "~teoma" 0; - "~TwitterBot" 0; - "~Wordpress" 0; - "~yahoo" 0; + "~*adidxbot" 0; + "~*AdsBot-Google" 0; + "~*aolbuild" 0; + "~*bingbot" 0; + "~*bingpreview" 0; + "~*DoCoMo" 0; + "~*duckduckgo" 0; + "~*facebookexternalhit" 0; + "~*Feedfetcher-Google" 0; + "~*Googlebot" 0; + "~*Googlebot-Image" 0; + "~*Googlebot-Mobile" 0; + "~*Googlebot-News" 0; + "~*Googlebot/Test" 0; + "~*Googlebot-Video" 0; + "~*Google-HTTP-Java-Client" 0; + "~*gsa-crawler" 0; + "~*Jakarta\ Commons" 0; + "~*Kraken/0.1" 0; + "~*LinkedInBot" 0; + "~*Mediapartners-Google" 0; + "~*msnbot" 0; + "~*msnbot-media" 0; + "~*SAMSUNG" 0; + "~*slurp" 0; + "~*teoma" 0; + "~*TwitterBot" 0; + "~*Wordpress" 0; + "~*yahoo" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # *************************************************** @@ -157,11 +157,11 @@ map $http_user_agent $bad_bot { # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~jetmon" 1; - "~libwww-perl" 1; - "~Lynx" 1; - "~munin" 1; - "~Wget/1.15" 1; + "~*jetmon" 1; + "~*libwww-perl" 1; + "~*Lynx" 1; + "~*munin" 1; + "~*Wget/1.15" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************************** @@ -169,24 +169,24 @@ map $http_user_agent $bad_bot { # ************************************************************** # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~Alexa" 2; - "~archive.org" 2; - "~Baidu" 2; - "~BUbiNG" 2; - "~FlipboardProxy" 2; - "~ia_archiver" 2; - "~Mozilla/4.0" 2; - "~MSIE\ 7.0" 2; - "~Presto" 2; - "~R6_CommentReader" 2; - "~R6_FeedFetcher" 2; - "~RPT-HTTPClient" 2; - "~SeznamBot" 2; - "~sfFeedReader/0.9" 2; - "~Spaidu" 2; - "~UptimeRobot/2.0" 2; - "~YandexBot" 2; - "~YandexImages" 2; + "~*Alexa" 2; + "~*archive.org" 2; + "~*Baidu" 2; + "~*BUbiNG" 2; + "~*FlipboardProxy" 2; + "~*ia_archiver" 2; + "~*Mozilla/4.0" 2; + "~*MSIE\ 7.0" 2; + "~*Presto" 2; + "~*R6_CommentReader" 2; + "~*R6_FeedFetcher" 2; + "~*RPT-HTTPClient" 2; + "~*SeznamBot" 2; + "~*sfFeedReader/0.9" 2; + "~*Spaidu" 2; + "~*UptimeRobot/2.0" 2; + "~*YandexBot" 2; + "~*YandexImages" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ********************************************* @@ -197,487 +197,487 @@ map $http_user_agent $bad_bot { # Known Vulnerability Scanners (now merged into one section) # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~360Spider" 3; - "~80legs" 3; - "~Abonti" 3; - "~Aboundex" 3; - "~Acunetix" 3; - "~ADmantX" 3; - "~AhrefsBot" 3; - "~AIBOT" 3; - "~AiHitBot" 3; - "~Aipbot" 3; - "~Alexibot" 3; - "~Alligator" 3; - "~AllSubmitter" 3; - "~Anarchie" 3; - "~Apexoo" 3; - "~ASPSeek" 3; - "~Asterias" 3; - "~Attach" 3; - "~autoemailspider" 3; - "~BackDoorBot" 3; - "~BackStreet" 3; - "~BackWeb" 3; - "~Badass" 3; - "~Bandit" 3; - "~BatchFTP" 3; - "~Battleztar\ Bazinga" 3; - "~BBBike" 3; - "~BDFetch" 3; - "~BetaBot" 3; - "~Bigfoot" 3; - "~Bitacle" 3; - "~Blackboard" 3; - "~Black\ Hole" 3; - "~BlackWidow" 3; - "~BLEXBot" 3; - "~Blow" 3; - "~BlowFish" 3; - "~Boardreader" 3; - "~Bolt" 3; - "~BotALot" 3; - "~Brandprotect" 3; - "~Brandwatch" 3; - "~Bubing" 3; - "~Buddy" 3; - "~BuiltBotTough" 3; - "~BuiltWith" 3; - "~Bullseye" 3; - "~BunnySlippers" 3; - "~BuzzSumo" 3; - "~Calculon" 3; - "~CATExplorador" 3; - "~CazoodleBot" 3; - "~CCBot" 3; - "~Cegbfeieh" 3; - "~CheeseBot" 3; - "~CherryPicker" 3; - "~ChinaClaw" 3; - "~Chlooe" 3; - "~Claritybot" 3; - "~Cliqzbot" 3; - "~Cogentbot" 3; - "~cognitiveseo" 3; - "~Collector" 3; - "~Copier" 3; - "~CopyRightCheck" 3; - "~Copyscape" 3; - "~Cosmos" 3; - "~Craftbot" 3; - "~CrazyWebCrawler" 3; - "~Crescent" 3; - "~CSHttp" 3; - "~Curious" 3; - "~Custo" 3; - "~DatabaseDriverMysqli" 3; - "~DBLBot" 3; - "~Demon" 3; - "~Deusu" 3; - "~Devil" 3; - "~DIIbot" 3; - "~Dirbuster" 3; - "~Disco" 3; - "~Discobot" 3; - "~Discoverybot" 3; - "~DittoSpyder" 3; - "~DomainAppender" 3; - "~DomainCrawler" 3; - "~DomainSigmaCrawler" 3; - "~Dotbot" 3; - "~Download\ Demon" 3; - "~Download\ Devil" 3; - "~Download\ Wonder" 3; - "~Dragonfly" 3; - "~Drip" 3; - "~DTS\ Agent" 3; - "~EasyDL" 3; - "~Ebingbong" 3; - "~eCatch" 3; - "~ECCP/1.0" 3; - "~Ecxi" 3; - "~EirGrabber" 3; - "~EMail\ Collector" 3; - "~EMail\ Extractor" 3; - "~EMail\ Siphon" 3; - "~EMail\ Wolf" 3; - "~EroCrawler" 3; - "~Evil" 3; - "~Exabot" 3; - "~Express\ WebPictures" 3; - "~Extractor" 3; - "~ExtractorPro" 3; - "~EyeNetIE" 3; - "~Ezooms" 3; - "~FHscan" 3; - "~Fimap" 3; - "~Findxbot" 3; - "~Firefox/7.0" 3; - "~FlashGet" 3; - "~Flunky" 3; - "~Foobot" 3; - "~Freeuploader" 3; - "~FrontPage" 3; - "~Fyrebot" 3; - "~GalaxyBot" 3; - "~Genieo" 3; - "~Getintent" 3; - "~GetRight" 3; - "~GetWeb" 3; - "~Gigablast" 3; - "~Gigabot" 3; - "~Go-Ahead-Got-It" 3; - "~Gotit" 3; - "~GoZilla" 3; - "~Go!Zilla" 3; - "~Grabber" 3; - "~GrabNet" 3; - "~Grafula" 3; - "~GrapeFX" 3; - "~GrapeshotCrawler" 3; - "~GridBot" 3; - "~GT::WWW" 3; - "~HaosouSpider" 3; - "~Harvest" 3; - "~Havij" 3; - "~HEADMasterSEO" 3; - "~Heritrix" 3; - "~Hloader" 3; - "~HMView" 3; - "~HTMLparser" 3; - "~HTTP::Lite" 3; - "~HTTrack" 3; - "~Humanlinks" 3; - "~HybridBot" 3; - "~Iblog" 3; - "~IDBot" 3; - "~Id-search" 3; - "~IlseBot" 3; - "~Image\ Fetch" 3; - "~Image\ Stripper" 3; - "~Image\ Sucker" 3; - "~Indy\ Library" 3; - "~InfoNaviRobot" 3; - "~InfoTekies" 3; - "~Intelliseek" 3; - "~InterGET" 3; - "~Internet\ Ninja" 3; - "~InternetSeer" 3; - "~internetVista\ monitor" 3; - "~Iria" 3; - "~IRLbot" 3; - "~Iskanie" 3; - "~JamesBOT" 3; - "~Jbrofuzz" 3; - "~JennyBot" 3; - "~JetCar" 3; - "~JikeSpider" 3; - "~JOC\ Web\ Spider" 3; - "~Joomla" 3; - "~JustView" 3; - "~Jyxobot" 3; - "~Kenjin\ Spider" 3; - "~Keyword\ Density" 3; - "~Lanshanbot" 3; - "~Larbin" 3; - "~LeechFTP" 3; - "~LeechGet" 3; - "~LexiBot" 3; - "~Lftp" 3; - "~LibWeb" 3; - "~Libwhisker" 3; - "~Lightspeedsystems" 3; - "~Likse" 3; - "~Linkdexbot" 3; - "~LinkextractorPro" 3; - "~LinkpadBot" 3; - "~LinkScan" 3; - "~LinksManager" 3; - "~LinkWalker" 3; - "~LinqiaMetadataDownloaderBot" 3; - "~LinqiaRSSBot" 3; - "~LinqiaScrapeBot" 3; - "~Lipperhey" 3; - "~Litemage_walker" 3; - "~Lmspider" 3; - "~LNSpiderguy" 3; - "~Ltx71" 3; - "~lwp-request" 3; - "~LWP::Simple" 3; - "~lwp-trivial" 3; - "~Magnet" 3; - "~Mag-Net" 3; - "~magpie-crawler" 3; - "~Mail.ru" 3; - "~Majestic12" 3; - "~MarkMonitor" 3; - "~MarkWatch" 3; - "~Masscan" 3; - "~Mass\ Downloader" 3; - "~Mata\ Hari" 3; - "~Meanpathbot" 3; - "~MegaIndex.ru" 3; - "~Metauri" 3; - "~MFC_Tear_Sample" 3; - "~Microsoft\ Data\ Access" 3; - "~Microsoft\ URL\ Control" 3; - "~MIDown\ tool" 3; - "~MIIxpc" 3; - "~Mister\ PiX" 3; - "~MJ12bot" 3; - "~Mojeek" 3; - "~MSFrontPage" 3; - "~MSIE\ 6.0" 3; - "~MSIECrawler" 3; - "~Msrabot" 3; - "~MS\ Web\ Services\ Client\ Protocol" 3; - "~Musobot" 3; - "~Name\ Intelligence" 3; - "~Nameprotect" 3; - "~Navroad" 3; - "~NearSite" 3; - "~Needle" 3; - "~Nessus" 3; - "~NetAnts" 3; - "~Netcraft" 3; - "~netEstate\ NE\ Crawler" 3; - "~NetLyzer" 3; - "~NetMechanic" 3; - "~NetSpider" 3; - "~Nettrack" 3; - "~Net\ Vampire" 3; - "~NetZIP" 3; - "~NextGenSearchBot" 3; - "~Nibbler" 3; - "~NICErsPRO" 3; - "~Niki-bot" 3; - "~Nikto" 3; - "~NimbleCrawler" 3; - "~Ninja" 3; - "~Nmap" 3; - "~NPbot" 3; - "~Nutch" 3; - "~Octopus" 3; - "~Offline\ Explorer" 3; - "~Offline\ Navigator" 3; - "~Openfind" 3; - "~OpenLinkProfiler" 3; - "~Openvas" 3; - "~OrangeBot" 3; - "~OrangeSpider" 3; - "~OutfoxBot" 3; - "~PageAnalyzer" 3; - "~Page\ Analyzer" 3; - "~PageGrabber" 3; - "~Page\ Grabber" 3; - "~page\ scorer" 3; - "~PageScorer" 3; - "~Panscient" 3; - "~Papa\ Foto" 3; - "~Pavuk" 3; - "~pcBrowser" 3; - "~PECL::HTTP" 3; - "~PeoplePal" 3; - "~PHPCrawl" 3; - "~Picscout" 3; - "~Picsearch" 3; - "~Pimonster" 3; - "~Pi-Monster" 3; - "~Pixray" 3; - "~PleaseCrawl" 3; - "~Pockey" 3; - "~POE-Component-Client-HTTP" 3; - "~Probethenet" 3; - "~ProPowerBot" 3; - "~ProWebWalker" 3; - "~Proximic" 3; - "~Psbot" 3; - "~Pump" 3; - "~PyCurl" 3; - "~QueryN\ Metasearch" 3; - "~Qwantify" 3; - "~RankActiveLinkBot" 3; - "~Rankivabot" 3; - "~RealDownload" 3; - "~Reaper" 3; - "~Recorder" 3; - "~RedesScrapy" 3; - "~ReGet" 3; - "~RepoMonkey" 3; - "~Ripper" 3; - "~RocketCrawler" 3; - "~Rogerbot" 3; - "~SalesIntelligent" 3; - "~SBIder" 3; - "~ScanAlert" 3; - "~Scanbot" 3; - "~Scrapy" 3; - "~Screaming" 3; - "~Screaming\ Frog\ SEO\ Spider" 3; - "~ScreenerBot" 3; - "~Searchestate" 3; - "~SearchmetricsBot" 3; - "~Semrush" 3; - "~SemrushBot" 3; - "~SEOkicks" 3; - "~SEOkicks-Robot" 3; - "~SEOlyticsCrawler" 3; - "~Seomoz" 3; - "~SEOprofiler" 3; - "~seoscanners" 3; - "~SEOstats" 3; - "~Siphon" 3; - "~SISTRIX" 3; - "~SISTRIX\ Crawler" 3; - "~Sitebeam" 3; - "~SiteExplorer" 3; - "~Siteimprove" 3; - "~SiteLockSpider" 3; - "~SiteSnagger" 3; - "~SiteSucker" 3; - "~Site\ Sucker" 3; - "~Sitevigil" 3; - "~Slackbot-LinkExpanding" 3; - "~SlySearch" 3; - "~SmartDownload" 3; - "~Snake" 3; - "~Snapbot" 3; - "~Snoopy" 3; - "~SocialRankIOBot" 3; - "~Sogou\ web\ spider" 3; - "~Sosospider" 3; - "~SpaceBison" 3; - "~Spammen" 3; - "~SpankBot" 3; - "~Spanner" 3; - "~Spbot" 3; - "~Spinn3r" 3; - "~SputnikBot" 3; - "~Sqlmap" 3; - "~Sqlworm" 3; - "~Sqworm" 3; - "~Steeler" 3; - "~Stripper" 3; - "~Sucker" 3; - "~Sucuri" 3; - "~SuperBot" 3; - "~SuperHTTP" 3; - "~Surfbot" 3; - "~SurveyBot" 3; - "~Suzuran" 3; - "~Swiftbot" 3; - "~Szukacz" 3; - "~T0PHackTeam" 3; - "~T8Abot" 3; - "~tAkeOut" 3; - "~Teleport" 3; - "~TeleportPro" 3; - "~Telesoft" 3; - "~Telesphoreo" 3; - "~Telesphorep" 3; - "~The\ Intraformant" 3; - "~TheNomad" 3; - "~TightTwatBot" 3; - "~Titan" 3; - "~Toata" 3; - "~Toweyabot" 3; - "~Trendictionbot" 3; - "~True_Robot" 3; - "~Turingos" 3; - "~TurnitinBot" 3; - "~Turnitin\ Bot" 3; - "~Turnitin\ Robot" 3; - "~TwengaBot" 3; - "~Twice" 3; - "~Typhoeus" 3; - "~UnisterBot" 3; - "~URLy.Warning" 3; - "~URLy\ Warning" 3; - "~Vacuum" 3; - "~Vagabondo" 3; - "~VB\ Project" 3; - "~VCI" 3; - "~VeriCiteCrawler" 3; - "~VidibleScraper" 3; - "~VoidEYE" 3; - "~Voil" 3; - "~Voltron" 3; - "~Wallpapers/3.0" 3; - "~WallpapersHD" 3; - "~WASALive-Bot" 3; - "~WBSearchBot" 3; - "~Webalta" 3; - "~WebAuto" 3; - "~Web\ Auto" 3; - "~WebBandit" 3; - "~Web\ Bandit" 3; - "~WebCollage" 3; - "~Web\ Collage" 3; - "~WebCopier" 3; - "~Web\ Copier" 3; - "~WEBDAV" 3; - "~WEBDAV\ Client" 3; - "~WebEnhancer" 3; - "~Web\ Enhancer" 3; - "~WebFetch" 3; - "~Web\ Fetch" 3; - "~WebFuck" 3; - "~Web\ Fuck" 3; - "~WebGo\ IS" 3; - "~WebImageCollector" 3; - "~Web\ Image\ Collector" 3; - "~WebLeacher" 3; - "~WebmasterWorldForumBot" 3; - "~webmeup-crawler" 3; - "~WebPix" 3; - "~Web\ Pix" 3; - "~WebReaper" 3; - "~Web\ Reaper" 3; - "~WebSauger" 3; - "~Web\ Sauger" 3; - "~Webshag" 3; - "~WebsiteExtractor" 3; - "~Website\ Extractor" 3; - "~WebsiteQuester" 3; - "~Website\ Quester" 3; - "~Webster" 3; - "~WebStripper" 3; - "~Web\ Stripper" 3; - "~WebSucker" 3; - "~Web\ Sucker" 3; - "~WebWhacker" 3; - "~Web\ Whacker" 3; - "~WebZIP" 3; - "~WeSEE" 3; - "~Whack" 3; - "~Whacker" 3; - "~Whatweb" 3; - "~Widow" 3; - "~WinHTTrack" 3; - "~WiseGuys\ Robot" 3; - "~WISENutbot" 3; - "~Wonderbot" 3; - "~Woobot" 3; - "~Wotbox" 3; - "~Wprecon" 3; - "~WPScan" 3; - "~WWW-Collector-E" 3; - "~WWW-Mechanize" 3; - "~WWW::Mechanize" 3; - "~WWWOFFLE" 3; - "~x22Mozilla" 3; - "~Xaldon_WebSpider" 3; - "~Xaldon\ WebSpider" 3; - "~Xenu" 3; - "~YoudaoBot" 3; - "~Zade" 3; - "~Zermelo" 3; - "~Zeus" 3; - "~Zgrab" 3; - "~Zitebot" 3; - "~ZmEu" 3; - "~ZumBot" 3; - "~ZyBorg" 3; + "~*360Spider" 3; + "~*80legs" 3; + "~*Abonti" 3; + "~*Aboundex" 3; + "~*Acunetix" 3; + "~*ADmantX" 3; + "~*AhrefsBot" 3; + "~*AIBOT" 3; + "~*AiHitBot" 3; + "~*Aipbot" 3; + "~*Alexibot" 3; + "~*Alligator" 3; + "~*AllSubmitter" 3; + "~*Anarchie" 3; + "~*Apexoo" 3; + "~*ASPSeek" 3; + "~*Asterias" 3; + "~*Attach" 3; + "~*autoemailspider" 3; + "~*BackDoorBot" 3; + "~*BackStreet" 3; + "~*BackWeb" 3; + "~*Badass" 3; + "~*Bandit" 3; + "~*BatchFTP" 3; + "~*Battleztar\ Bazinga" 3; + "~*BBBike" 3; + "~*BDFetch" 3; + "~*BetaBot" 3; + "~*Bigfoot" 3; + "~*Bitacle" 3; + "~*Blackboard" 3; + "~*Black\ Hole" 3; + "~*BlackWidow" 3; + "~*BLEXBot" 3; + "~*Blow" 3; + "~*BlowFish" 3; + "~*Boardreader" 3; + "~*Bolt" 3; + "~*BotALot" 3; + "~*Brandprotect" 3; + "~*Brandwatch" 3; + "~*Bubing" 3; + "~*Buddy" 3; + "~*BuiltBotTough" 3; + "~*BuiltWith" 3; + "~*Bullseye" 3; + "~*BunnySlippers" 3; + "~*BuzzSumo" 3; + "~*Calculon" 3; + "~*CATExplorador" 3; + "~*CazoodleBot" 3; + "~*CCBot" 3; + "~*Cegbfeieh" 3; + "~*CheeseBot" 3; + "~*CherryPicker" 3; + "~*ChinaClaw" 3; + "~*Chlooe" 3; + "~*Claritybot" 3; + "~*Cliqzbot" 3; + "~*Cogentbot" 3; + "~*cognitiveseo" 3; + "~*Collector" 3; + "~*Copier" 3; + "~*CopyRightCheck" 3; + "~*Copyscape" 3; + "~*Cosmos" 3; + "~*Craftbot" 3; + "~*CrazyWebCrawler" 3; + "~*Crescent" 3; + "~*CSHttp" 3; + "~*Curious" 3; + "~*Custo" 3; + "~*DatabaseDriverMysqli" 3; + "~*DBLBot" 3; + "~*Demon" 3; + "~*Deusu" 3; + "~*Devil" 3; + "~*DIIbot" 3; + "~*Dirbuster" 3; + "~*Disco" 3; + "~*Discobot" 3; + "~*Discoverybot" 3; + "~*DittoSpyder" 3; + "~*DomainAppender" 3; + "~*DomainCrawler" 3; + "~*DomainSigmaCrawler" 3; + "~*Dotbot" 3; + "~*Download\ Demon" 3; + "~*Download\ Devil" 3; + "~*Download\ Wonder" 3; + "~*Dragonfly" 3; + "~*Drip" 3; + "~*DTS\ Agent" 3; + "~*EasyDL" 3; + "~*Ebingbong" 3; + "~*eCatch" 3; + "~*ECCP/1.0" 3; + "~*Ecxi" 3; + "~*EirGrabber" 3; + "~*EMail\ Collector" 3; + "~*EMail\ Extractor" 3; + "~*EMail\ Siphon" 3; + "~*EMail\ Wolf" 3; + "~*EroCrawler" 3; + "~*Evil" 3; + "~*Exabot" 3; + "~*Express\ WebPictures" 3; + "~*Extractor" 3; + "~*ExtractorPro" 3; + "~*EyeNetIE" 3; + "~*Ezooms" 3; + "~*FHscan" 3; + "~*Fimap" 3; + "~*Findxbot" 3; + "~*Firefox/7.0" 3; + "~*FlashGet" 3; + "~*Flunky" 3; + "~*Foobot" 3; + "~*Freeuploader" 3; + "~*FrontPage" 3; + "~*Fyrebot" 3; + "~*GalaxyBot" 3; + "~*Genieo" 3; + "~*Getintent" 3; + "~*GetRight" 3; + "~*GetWeb" 3; + "~*Gigablast" 3; + "~*Gigabot" 3; + "~*Go-Ahead-Got-It" 3; + "~*Gotit" 3; + "~*GoZilla" 3; + "~*Go!Zilla" 3; + "~*Grabber" 3; + "~*GrabNet" 3; + "~*Grafula" 3; + "~*GrapeFX" 3; + "~*GrapeshotCrawler" 3; + "~*GridBot" 3; + "~*GT::WWW" 3; + "~*HaosouSpider" 3; + "~*Harvest" 3; + "~*Havij" 3; + "~*HEADMasterSEO" 3; + "~*Heritrix" 3; + "~*Hloader" 3; + "~*HMView" 3; + "~*HTMLparser" 3; + "~*HTTP::Lite" 3; + "~*HTTrack" 3; + "~*Humanlinks" 3; + "~*HybridBot" 3; + "~*Iblog" 3; + "~*IDBot" 3; + "~*Id-search" 3; + "~*IlseBot" 3; + "~*Image\ Fetch" 3; + "~*Image\ Stripper" 3; + "~*Image\ Sucker" 3; + "~*Indy\ Library" 3; + "~*InfoNaviRobot" 3; + "~*InfoTekies" 3; + "~*Intelliseek" 3; + "~*InterGET" 3; + "~*Internet\ Ninja" 3; + "~*InternetSeer" 3; + "~*internetVista\ monitor" 3; + "~*Iria" 3; + "~*IRLbot" 3; + "~*Iskanie" 3; + "~*JamesBOT" 3; + "~*Jbrofuzz" 3; + "~*JennyBot" 3; + "~*JetCar" 3; + "~*JikeSpider" 3; + "~*JOC\ Web\ Spider" 3; + "~*Joomla" 3; + "~*JustView" 3; + "~*Jyxobot" 3; + "~*Kenjin\ Spider" 3; + "~*Keyword\ Density" 3; + "~*Lanshanbot" 3; + "~*Larbin" 3; + "~*LeechFTP" 3; + "~*LeechGet" 3; + "~*LexiBot" 3; + "~*Lftp" 3; + "~*LibWeb" 3; + "~*Libwhisker" 3; + "~*Lightspeedsystems" 3; + "~*Likse" 3; + "~*Linkdexbot" 3; + "~*LinkextractorPro" 3; + "~*LinkpadBot" 3; + "~*LinkScan" 3; + "~*LinksManager" 3; + "~*LinkWalker" 3; + "~*LinqiaMetadataDownloaderBot" 3; + "~*LinqiaRSSBot" 3; + "~*LinqiaScrapeBot" 3; + "~*Lipperhey" 3; + "~*Litemage_walker" 3; + "~*Lmspider" 3; + "~*LNSpiderguy" 3; + "~*Ltx71" 3; + "~*lwp-request" 3; + "~*LWP::Simple" 3; + "~*lwp-trivial" 3; + "~*Magnet" 3; + "~*Mag-Net" 3; + "~*magpie-crawler" 3; + "~*Mail.ru" 3; + "~*Majestic12" 3; + "~*MarkMonitor" 3; + "~*MarkWatch" 3; + "~*Masscan" 3; + "~*Mass\ Downloader" 3; + "~*Mata\ Hari" 3; + "~*Meanpathbot" 3; + "~*MegaIndex.ru" 3; + "~*Metauri" 3; + "~*MFC_Tear_Sample" 3; + "~*Microsoft\ Data\ Access" 3; + "~*Microsoft\ URL\ Control" 3; + "~*MIDown\ tool" 3; + "~*MIIxpc" 3; + "~*Mister\ PiX" 3; + "~*MJ12bot" 3; + "~*Mojeek" 3; + "~*MSFrontPage" 3; + "~*MSIE\ 6.0" 3; + "~*MSIECrawler" 3; + "~*Msrabot" 3; + "~*MS\ Web\ Services\ Client\ Protocol" 3; + "~*Musobot" 3; + "~*Name\ Intelligence" 3; + "~*Nameprotect" 3; + "~*Navroad" 3; + "~*NearSite" 3; + "~*Needle" 3; + "~*Nessus" 3; + "~*NetAnts" 3; + "~*Netcraft" 3; + "~*netEstate\ NE\ Crawler" 3; + "~*NetLyzer" 3; + "~*NetMechanic" 3; + "~*NetSpider" 3; + "~*Nettrack" 3; + "~*Net\ Vampire" 3; + "~*NetZIP" 3; + "~*NextGenSearchBot" 3; + "~*Nibbler" 3; + "~*NICErsPRO" 3; + "~*Niki-bot" 3; + "~*Nikto" 3; + "~*NimbleCrawler" 3; + "~*Ninja" 3; + "~*Nmap" 3; + "~*NPbot" 3; + "~*Nutch" 3; + "~*Octopus" 3; + "~*Offline\ Explorer" 3; + "~*Offline\ Navigator" 3; + "~*Openfind" 3; + "~*OpenLinkProfiler" 3; + "~*Openvas" 3; + "~*OrangeBot" 3; + "~*OrangeSpider" 3; + "~*OutfoxBot" 3; + "~*PageAnalyzer" 3; + "~*Page\ Analyzer" 3; + "~*PageGrabber" 3; + "~*Page\ Grabber" 3; + "~*page\ scorer" 3; + "~*PageScorer" 3; + "~*Panscient" 3; + "~*Papa\ Foto" 3; + "~*Pavuk" 3; + "~*pcBrowser" 3; + "~*PECL::HTTP" 3; + "~*PeoplePal" 3; + "~*PHPCrawl" 3; + "~*Picscout" 3; + "~*Picsearch" 3; + "~*Pimonster" 3; + "~*Pi-Monster" 3; + "~*Pixray" 3; + "~*PleaseCrawl" 3; + "~*Pockey" 3; + "~*POE-Component-Client-HTTP" 3; + "~*Probethenet" 3; + "~*ProPowerBot" 3; + "~*ProWebWalker" 3; + "~*Proximic" 3; + "~*Psbot" 3; + "~*Pump" 3; + "~*PyCurl" 3; + "~*QueryN\ Metasearch" 3; + "~*Qwantify" 3; + "~*RankActiveLinkBot" 3; + "~*Rankivabot" 3; + "~*RealDownload" 3; + "~*Reaper" 3; + "~*Recorder" 3; + "~*RedesScrapy" 3; + "~*ReGet" 3; + "~*RepoMonkey" 3; + "~*Ripper" 3; + "~*RocketCrawler" 3; + "~*Rogerbot" 3; + "~*SalesIntelligent" 3; + "~*SBIder" 3; + "~*ScanAlert" 3; + "~*Scanbot" 3; + "~*Scrapy" 3; + "~*Screaming" 3; + "~*Screaming\ Frog\ SEO\ Spider" 3; + "~*ScreenerBot" 3; + "~*Searchestate" 3; + "~*SearchmetricsBot" 3; + "~*Semrush" 3; + "~*SemrushBot" 3; + "~*SEOkicks" 3; + "~*SEOkicks-Robot" 3; + "~*SEOlyticsCrawler" 3; + "~*Seomoz" 3; + "~*SEOprofiler" 3; + "~*seoscanners" 3; + "~*SEOstats" 3; + "~*Siphon" 3; + "~*SISTRIX" 3; + "~*SISTRIX\ Crawler" 3; + "~*Sitebeam" 3; + "~*SiteExplorer" 3; + "~*Siteimprove" 3; + "~*SiteLockSpider" 3; + "~*SiteSnagger" 3; + "~*SiteSucker" 3; + "~*Site\ Sucker" 3; + "~*Sitevigil" 3; + "~*Slackbot-LinkExpanding" 3; + "~*SlySearch" 3; + "~*SmartDownload" 3; + "~*Snake" 3; + "~*Snapbot" 3; + "~*Snoopy" 3; + "~*SocialRankIOBot" 3; + "~*Sogou\ web\ spider" 3; + "~*Sosospider" 3; + "~*SpaceBison" 3; + "~*Spammen" 3; + "~*SpankBot" 3; + "~*Spanner" 3; + "~*Spbot" 3; + "~*Spinn3r" 3; + "~*SputnikBot" 3; + "~*Sqlmap" 3; + "~*Sqlworm" 3; + "~*Sqworm" 3; + "~*Steeler" 3; + "~*Stripper" 3; + "~*Sucker" 3; + "~*Sucuri" 3; + "~*SuperBot" 3; + "~*SuperHTTP" 3; + "~*Surfbot" 3; + "~*SurveyBot" 3; + "~*Suzuran" 3; + "~*Swiftbot" 3; + "~*Szukacz" 3; + "~*T0PHackTeam" 3; + "~*T8Abot" 3; + "~*tAkeOut" 3; + "~*Teleport" 3; + "~*TeleportPro" 3; + "~*Telesoft" 3; + "~*Telesphoreo" 3; + "~*Telesphorep" 3; + "~*The\ Intraformant" 3; + "~*TheNomad" 3; + "~*TightTwatBot" 3; + "~*Titan" 3; + "~*Toata" 3; + "~*Toweyabot" 3; + "~*Trendictionbot" 3; + "~*True_Robot" 3; + "~*Turingos" 3; + "~*TurnitinBot" 3; + "~*Turnitin\ Bot" 3; + "~*Turnitin\ Robot" 3; + "~*TwengaBot" 3; + "~*Twice" 3; + "~*Typhoeus" 3; + "~*UnisterBot" 3; + "~*URLy.Warning" 3; + "~*URLy\ Warning" 3; + "~*Vacuum" 3; + "~*Vagabondo" 3; + "~*VB\ Project" 3; + "~*VCI" 3; + "~*VeriCiteCrawler" 3; + "~*VidibleScraper" 3; + "~*VoidEYE" 3; + "~*Voil" 3; + "~*Voltron" 3; + "~*Wallpapers/3.0" 3; + "~*WallpapersHD" 3; + "~*WASALive-Bot" 3; + "~*WBSearchBot" 3; + "~*Webalta" 3; + "~*WebAuto" 3; + "~*Web\ Auto" 3; + "~*WebBandit" 3; + "~*Web\ Bandit" 3; + "~*WebCollage" 3; + "~*Web\ Collage" 3; + "~*WebCopier" 3; + "~*Web\ Copier" 3; + "~*WEBDAV" 3; + "~*WEBDAV\ Client" 3; + "~*WebEnhancer" 3; + "~*Web\ Enhancer" 3; + "~*WebFetch" 3; + "~*Web\ Fetch" 3; + "~*WebFuck" 3; + "~*Web\ Fuck" 3; + "~*WebGo\ IS" 3; + "~*WebImageCollector" 3; + "~*Web\ Image\ Collector" 3; + "~*WebLeacher" 3; + "~*WebmasterWorldForumBot" 3; + "~*webmeup-crawler" 3; + "~*WebPix" 3; + "~*Web\ Pix" 3; + "~*WebReaper" 3; + "~*Web\ Reaper" 3; + "~*WebSauger" 3; + "~*Web\ Sauger" 3; + "~*Webshag" 3; + "~*WebsiteExtractor" 3; + "~*Website\ Extractor" 3; + "~*WebsiteQuester" 3; + "~*Website\ Quester" 3; + "~*Webster" 3; + "~*WebStripper" 3; + "~*Web\ Stripper" 3; + "~*WebSucker" 3; + "~*Web\ Sucker" 3; + "~*WebWhacker" 3; + "~*Web\ Whacker" 3; + "~*WebZIP" 3; + "~*WeSEE" 3; + "~*Whack" 3; + "~*Whacker" 3; + "~*Whatweb" 3; + "~*Widow" 3; + "~*WinHTTrack" 3; + "~*WiseGuys\ Robot" 3; + "~*WISENutbot" 3; + "~*Wonderbot" 3; + "~*Woobot" 3; + "~*Wotbox" 3; + "~*Wprecon" 3; + "~*WPScan" 3; + "~*WWW-Collector-E" 3; + "~*WWW-Mechanize" 3; + "~*WWW::Mechanize" 3; + "~*WWWOFFLE" 3; + "~*x22Mozilla" 3; + "~*Xaldon_WebSpider" 3; + "~*Xaldon\ WebSpider" 3; + "~*Xenu" 3; + "~*YoudaoBot" 3; + "~*Zade" 3; + "~*Zermelo" 3; + "~*Zeus" 3; + "~*Zgrab" 3; + "~*Zitebot" 3; + "~*ZmEu" 3; + "~*ZumBot" 3; + "~*ZyBorg" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### } diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index c4df3689e..698d7a86e 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -1,7 +1,7 @@ ### Version Information # ################################################### -### Version: V3.2017.07.643 -### Updated: Fri Jul 7 11:11:23 SAST 2017 +### Version: V3.2017.07.644 +### Updated: Fri Jul 7 11:55:01 SAST 2017 ### Bad Bot Count: 481 ################################################### ### Version Information ## diff --git a/travisCI/globalblacklist.template b/travisCI/globalblacklist.template index d5547ce7a..30dbea841 100755 --- a/travisCI/globalblacklist.template +++ b/travisCI/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2017.07.643 -### Updated: Fri Jul 7 11:11:23 SAST 2017 +### Version: V3.2017.07.644 +### Updated: Fri Jul 7 11:55:01 SAST 2017 ### Bad Referrer Count: 5029 ### Bad Bot Count: 481 ################################################### @@ -118,35 +118,35 @@ map $http_user_agent $bad_bot { # *********************************************** # START GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~adidxbot" 0; - "~AdsBot-Google" 0; - "~aolbuild" 0; - "~bingbot" 0; - "~bingpreview" 0; - "~DoCoMo" 0; - "~duckduckgo" 0; - "~facebookexternalhit" 0; - "~Feedfetcher-Google" 0; - "~Googlebot" 0; - "~Googlebot-Image" 0; - "~Googlebot-Mobile" 0; - "~Googlebot-News" 0; - "~Googlebot/Test" 0; - "~Googlebot-Video" 0; - "~Google-HTTP-Java-Client" 0; - "~gsa-crawler" 0; - "~Jakarta\ Commons" 0; - "~Kraken/0.1" 0; - "~LinkedInBot" 0; - "~Mediapartners-Google" 0; - "~msnbot" 0; - "~msnbot-media" 0; - "~SAMSUNG" 0; - "~slurp" 0; - "~teoma" 0; - "~TwitterBot" 0; - "~Wordpress" 0; - "~yahoo" 0; + "~*adidxbot" 0; + "~*AdsBot-Google" 0; + "~*aolbuild" 0; + "~*bingbot" 0; + "~*bingpreview" 0; + "~*DoCoMo" 0; + "~*duckduckgo" 0; + "~*facebookexternalhit" 0; + "~*Feedfetcher-Google" 0; + "~*Googlebot" 0; + "~*Googlebot-Image" 0; + "~*Googlebot-Mobile" 0; + "~*Googlebot-News" 0; + "~*Googlebot/Test" 0; + "~*Googlebot-Video" 0; + "~*Google-HTTP-Java-Client" 0; + "~*gsa-crawler" 0; + "~*Jakarta\ Commons" 0; + "~*Kraken/0.1" 0; + "~*LinkedInBot" 0; + "~*Mediapartners-Google" 0; + "~*msnbot" 0; + "~*msnbot-media" 0; + "~*SAMSUNG" 0; + "~*slurp" 0; + "~*teoma" 0; + "~*TwitterBot" 0; + "~*Wordpress" 0; + "~*yahoo" 0; # END GOOD BOTS ### DO NOT EDIT THIS LINE AT ALL ### # *************************************************** @@ -157,11 +157,11 @@ map $http_user_agent $bad_bot { # I allow libwww-perl as I use it for monitoring systems with Munin but it is rate limited # START ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~jetmon" 1; - "~libwww-perl" 1; - "~Lynx" 1; - "~munin" 1; - "~Wget/1.15" 1; + "~*jetmon" 1; + "~*libwww-perl" 1; + "~*Lynx" 1; + "~*munin" 1; + "~*Wget/1.15" 1; # END ALLOWED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ************************************************************** @@ -169,24 +169,24 @@ map $http_user_agent $bad_bot { # ************************************************************** # START LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~Alexa" 2; - "~archive.org" 2; - "~Baidu" 2; - "~BUbiNG" 2; - "~FlipboardProxy" 2; - "~ia_archiver" 2; - "~Mozilla/4.0" 2; - "~MSIE\ 7.0" 2; - "~Presto" 2; - "~R6_CommentReader" 2; - "~R6_FeedFetcher" 2; - "~RPT-HTTPClient" 2; - "~SeznamBot" 2; - "~sfFeedReader/0.9" 2; - "~Spaidu" 2; - "~UptimeRobot/2.0" 2; - "~YandexBot" 2; - "~YandexImages" 2; + "~*Alexa" 2; + "~*archive.org" 2; + "~*Baidu" 2; + "~*BUbiNG" 2; + "~*FlipboardProxy" 2; + "~*ia_archiver" 2; + "~*Mozilla/4.0" 2; + "~*MSIE\ 7.0" 2; + "~*Presto" 2; + "~*R6_CommentReader" 2; + "~*R6_FeedFetcher" 2; + "~*RPT-HTTPClient" 2; + "~*SeznamBot" 2; + "~*sfFeedReader/0.9" 2; + "~*Spaidu" 2; + "~*UptimeRobot/2.0" 2; + "~*YandexBot" 2; + "~*YandexImages" 2; # END LIMITED BOTS ### DO NOT EDIT THIS LINE AT ALL ### # ********************************************* @@ -197,487 +197,487 @@ map $http_user_agent $bad_bot { # Known Vulnerability Scanners (now merged into one section) # START BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### - "~360Spider" 3; - "~80legs" 3; - "~Abonti" 3; - "~Aboundex" 3; - "~Acunetix" 3; - "~ADmantX" 3; - "~AhrefsBot" 3; - "~AIBOT" 3; - "~AiHitBot" 3; - "~Aipbot" 3; - "~Alexibot" 3; - "~Alligator" 3; - "~AllSubmitter" 3; - "~Anarchie" 3; - "~Apexoo" 3; - "~ASPSeek" 3; - "~Asterias" 3; - "~Attach" 3; - "~autoemailspider" 3; - "~BackDoorBot" 3; - "~BackStreet" 3; - "~BackWeb" 3; - "~Badass" 3; - "~Bandit" 3; - "~BatchFTP" 3; - "~Battleztar\ Bazinga" 3; - "~BBBike" 3; - "~BDFetch" 3; - "~BetaBot" 3; - "~Bigfoot" 3; - "~Bitacle" 3; - "~Blackboard" 3; - "~Black\ Hole" 3; - "~BlackWidow" 3; - "~BLEXBot" 3; - "~Blow" 3; - "~BlowFish" 3; - "~Boardreader" 3; - "~Bolt" 3; - "~BotALot" 3; - "~Brandprotect" 3; - "~Brandwatch" 3; - "~Bubing" 3; - "~Buddy" 3; - "~BuiltBotTough" 3; - "~BuiltWith" 3; - "~Bullseye" 3; - "~BunnySlippers" 3; - "~BuzzSumo" 3; - "~Calculon" 3; - "~CATExplorador" 3; - "~CazoodleBot" 3; - "~CCBot" 3; - "~Cegbfeieh" 3; - "~CheeseBot" 3; - "~CherryPicker" 3; - "~ChinaClaw" 3; - "~Chlooe" 3; - "~Claritybot" 3; - "~Cliqzbot" 3; - "~Cogentbot" 3; - "~cognitiveseo" 3; - "~Collector" 3; - "~Copier" 3; - "~CopyRightCheck" 3; - "~Copyscape" 3; - "~Cosmos" 3; - "~Craftbot" 3; - "~CrazyWebCrawler" 3; - "~Crescent" 3; - "~CSHttp" 3; - "~Curious" 3; - "~Custo" 3; - "~DatabaseDriverMysqli" 3; - "~DBLBot" 3; - "~Demon" 3; - "~Deusu" 3; - "~Devil" 3; - "~DIIbot" 3; - "~Dirbuster" 3; - "~Disco" 3; - "~Discobot" 3; - "~Discoverybot" 3; - "~DittoSpyder" 3; - "~DomainAppender" 3; - "~DomainCrawler" 3; - "~DomainSigmaCrawler" 3; - "~Dotbot" 3; - "~Download\ Demon" 3; - "~Download\ Devil" 3; - "~Download\ Wonder" 3; - "~Dragonfly" 3; - "~Drip" 3; - "~DTS\ Agent" 3; - "~EasyDL" 3; - "~Ebingbong" 3; - "~eCatch" 3; - "~ECCP/1.0" 3; - "~Ecxi" 3; - "~EirGrabber" 3; - "~EMail\ Collector" 3; - "~EMail\ Extractor" 3; - "~EMail\ Siphon" 3; - "~EMail\ Wolf" 3; - "~EroCrawler" 3; - "~Evil" 3; - "~Exabot" 3; - "~Express\ WebPictures" 3; - "~Extractor" 3; - "~ExtractorPro" 3; - "~EyeNetIE" 3; - "~Ezooms" 3; - "~FHscan" 3; - "~Fimap" 3; - "~Findxbot" 3; - "~Firefox/7.0" 3; - "~FlashGet" 3; - "~Flunky" 3; - "~Foobot" 3; - "~Freeuploader" 3; - "~FrontPage" 3; - "~Fyrebot" 3; - "~GalaxyBot" 3; - "~Genieo" 3; - "~Getintent" 3; - "~GetRight" 3; - "~GetWeb" 3; - "~Gigablast" 3; - "~Gigabot" 3; - "~Go-Ahead-Got-It" 3; - "~Gotit" 3; - "~GoZilla" 3; - "~Go!Zilla" 3; - "~Grabber" 3; - "~GrabNet" 3; - "~Grafula" 3; - "~GrapeFX" 3; - "~GrapeshotCrawler" 3; - "~GridBot" 3; - "~GT::WWW" 3; - "~HaosouSpider" 3; - "~Harvest" 3; - "~Havij" 3; - "~HEADMasterSEO" 3; - "~Heritrix" 3; - "~Hloader" 3; - "~HMView" 3; - "~HTMLparser" 3; - "~HTTP::Lite" 3; - "~HTTrack" 3; - "~Humanlinks" 3; - "~HybridBot" 3; - "~Iblog" 3; - "~IDBot" 3; - "~Id-search" 3; - "~IlseBot" 3; - "~Image\ Fetch" 3; - "~Image\ Stripper" 3; - "~Image\ Sucker" 3; - "~Indy\ Library" 3; - "~InfoNaviRobot" 3; - "~InfoTekies" 3; - "~Intelliseek" 3; - "~InterGET" 3; - "~Internet\ Ninja" 3; - "~InternetSeer" 3; - "~internetVista\ monitor" 3; - "~Iria" 3; - "~IRLbot" 3; - "~Iskanie" 3; - "~JamesBOT" 3; - "~Jbrofuzz" 3; - "~JennyBot" 3; - "~JetCar" 3; - "~JikeSpider" 3; - "~JOC\ Web\ Spider" 3; - "~Joomla" 3; - "~JustView" 3; - "~Jyxobot" 3; - "~Kenjin\ Spider" 3; - "~Keyword\ Density" 3; - "~Lanshanbot" 3; - "~Larbin" 3; - "~LeechFTP" 3; - "~LeechGet" 3; - "~LexiBot" 3; - "~Lftp" 3; - "~LibWeb" 3; - "~Libwhisker" 3; - "~Lightspeedsystems" 3; - "~Likse" 3; - "~Linkdexbot" 3; - "~LinkextractorPro" 3; - "~LinkpadBot" 3; - "~LinkScan" 3; - "~LinksManager" 3; - "~LinkWalker" 3; - "~LinqiaMetadataDownloaderBot" 3; - "~LinqiaRSSBot" 3; - "~LinqiaScrapeBot" 3; - "~Lipperhey" 3; - "~Litemage_walker" 3; - "~Lmspider" 3; - "~LNSpiderguy" 3; - "~Ltx71" 3; - "~lwp-request" 3; - "~LWP::Simple" 3; - "~lwp-trivial" 3; - "~Magnet" 3; - "~Mag-Net" 3; - "~magpie-crawler" 3; - "~Mail.ru" 3; - "~Majestic12" 3; - "~MarkMonitor" 3; - "~MarkWatch" 3; - "~Masscan" 3; - "~Mass\ Downloader" 3; - "~Mata\ Hari" 3; - "~Meanpathbot" 3; - "~MegaIndex.ru" 3; - "~Metauri" 3; - "~MFC_Tear_Sample" 3; - "~Microsoft\ Data\ Access" 3; - "~Microsoft\ URL\ Control" 3; - "~MIDown\ tool" 3; - "~MIIxpc" 3; - "~Mister\ PiX" 3; - "~MJ12bot" 3; - "~Mojeek" 3; - "~MSFrontPage" 3; - "~MSIE\ 6.0" 3; - "~MSIECrawler" 3; - "~Msrabot" 3; - "~MS\ Web\ Services\ Client\ Protocol" 3; - "~Musobot" 3; - "~Name\ Intelligence" 3; - "~Nameprotect" 3; - "~Navroad" 3; - "~NearSite" 3; - "~Needle" 3; - "~Nessus" 3; - "~NetAnts" 3; - "~Netcraft" 3; - "~netEstate\ NE\ Crawler" 3; - "~NetLyzer" 3; - "~NetMechanic" 3; - "~NetSpider" 3; - "~Nettrack" 3; - "~Net\ Vampire" 3; - "~NetZIP" 3; - "~NextGenSearchBot" 3; - "~Nibbler" 3; - "~NICErsPRO" 3; - "~Niki-bot" 3; - "~Nikto" 3; - "~NimbleCrawler" 3; - "~Ninja" 3; - "~Nmap" 3; - "~NPbot" 3; - "~Nutch" 3; - "~Octopus" 3; - "~Offline\ Explorer" 3; - "~Offline\ Navigator" 3; - "~Openfind" 3; - "~OpenLinkProfiler" 3; - "~Openvas" 3; - "~OrangeBot" 3; - "~OrangeSpider" 3; - "~OutfoxBot" 3; - "~PageAnalyzer" 3; - "~Page\ Analyzer" 3; - "~PageGrabber" 3; - "~Page\ Grabber" 3; - "~page\ scorer" 3; - "~PageScorer" 3; - "~Panscient" 3; - "~Papa\ Foto" 3; - "~Pavuk" 3; - "~pcBrowser" 3; - "~PECL::HTTP" 3; - "~PeoplePal" 3; - "~PHPCrawl" 3; - "~Picscout" 3; - "~Picsearch" 3; - "~Pimonster" 3; - "~Pi-Monster" 3; - "~Pixray" 3; - "~PleaseCrawl" 3; - "~Pockey" 3; - "~POE-Component-Client-HTTP" 3; - "~Probethenet" 3; - "~ProPowerBot" 3; - "~ProWebWalker" 3; - "~Proximic" 3; - "~Psbot" 3; - "~Pump" 3; - "~PyCurl" 3; - "~QueryN\ Metasearch" 3; - "~Qwantify" 3; - "~RankActiveLinkBot" 3; - "~Rankivabot" 3; - "~RealDownload" 3; - "~Reaper" 3; - "~Recorder" 3; - "~RedesScrapy" 3; - "~ReGet" 3; - "~RepoMonkey" 3; - "~Ripper" 3; - "~RocketCrawler" 3; - "~Rogerbot" 3; - "~SalesIntelligent" 3; - "~SBIder" 3; - "~ScanAlert" 3; - "~Scanbot" 3; - "~Scrapy" 3; - "~Screaming" 3; - "~Screaming\ Frog\ SEO\ Spider" 3; - "~ScreenerBot" 3; - "~Searchestate" 3; - "~SearchmetricsBot" 3; - "~Semrush" 3; - "~SemrushBot" 3; - "~SEOkicks" 3; - "~SEOkicks-Robot" 3; - "~SEOlyticsCrawler" 3; - "~Seomoz" 3; - "~SEOprofiler" 3; - "~seoscanners" 3; - "~SEOstats" 3; - "~Siphon" 3; - "~SISTRIX" 3; - "~SISTRIX\ Crawler" 3; - "~Sitebeam" 3; - "~SiteExplorer" 3; - "~Siteimprove" 3; - "~SiteLockSpider" 3; - "~SiteSnagger" 3; - "~SiteSucker" 3; - "~Site\ Sucker" 3; - "~Sitevigil" 3; - "~Slackbot-LinkExpanding" 3; - "~SlySearch" 3; - "~SmartDownload" 3; - "~Snake" 3; - "~Snapbot" 3; - "~Snoopy" 3; - "~SocialRankIOBot" 3; - "~Sogou\ web\ spider" 3; - "~Sosospider" 3; - "~SpaceBison" 3; - "~Spammen" 3; - "~SpankBot" 3; - "~Spanner" 3; - "~Spbot" 3; - "~Spinn3r" 3; - "~SputnikBot" 3; - "~Sqlmap" 3; - "~Sqlworm" 3; - "~Sqworm" 3; - "~Steeler" 3; - "~Stripper" 3; - "~Sucker" 3; - "~Sucuri" 3; - "~SuperBot" 3; - "~SuperHTTP" 3; - "~Surfbot" 3; - "~SurveyBot" 3; - "~Suzuran" 3; - "~Swiftbot" 3; - "~Szukacz" 3; - "~T0PHackTeam" 3; - "~T8Abot" 3; - "~tAkeOut" 3; - "~Teleport" 3; - "~TeleportPro" 3; - "~Telesoft" 3; - "~Telesphoreo" 3; - "~Telesphorep" 3; - "~The\ Intraformant" 3; - "~TheNomad" 3; - "~TightTwatBot" 3; - "~Titan" 3; - "~Toata" 3; - "~Toweyabot" 3; - "~Trendictionbot" 3; - "~True_Robot" 3; - "~Turingos" 3; - "~TurnitinBot" 3; - "~Turnitin\ Bot" 3; - "~Turnitin\ Robot" 3; - "~TwengaBot" 3; - "~Twice" 3; - "~Typhoeus" 3; - "~UnisterBot" 3; - "~URLy.Warning" 3; - "~URLy\ Warning" 3; - "~Vacuum" 3; - "~Vagabondo" 3; - "~VB\ Project" 3; - "~VCI" 3; - "~VeriCiteCrawler" 3; - "~VidibleScraper" 3; - "~VoidEYE" 3; - "~Voil" 3; - "~Voltron" 3; - "~Wallpapers/3.0" 3; - "~WallpapersHD" 3; - "~WASALive-Bot" 3; - "~WBSearchBot" 3; - "~Webalta" 3; - "~WebAuto" 3; - "~Web\ Auto" 3; - "~WebBandit" 3; - "~Web\ Bandit" 3; - "~WebCollage" 3; - "~Web\ Collage" 3; - "~WebCopier" 3; - "~Web\ Copier" 3; - "~WEBDAV" 3; - "~WEBDAV\ Client" 3; - "~WebEnhancer" 3; - "~Web\ Enhancer" 3; - "~WebFetch" 3; - "~Web\ Fetch" 3; - "~WebFuck" 3; - "~Web\ Fuck" 3; - "~WebGo\ IS" 3; - "~WebImageCollector" 3; - "~Web\ Image\ Collector" 3; - "~WebLeacher" 3; - "~WebmasterWorldForumBot" 3; - "~webmeup-crawler" 3; - "~WebPix" 3; - "~Web\ Pix" 3; - "~WebReaper" 3; - "~Web\ Reaper" 3; - "~WebSauger" 3; - "~Web\ Sauger" 3; - "~Webshag" 3; - "~WebsiteExtractor" 3; - "~Website\ Extractor" 3; - "~WebsiteQuester" 3; - "~Website\ Quester" 3; - "~Webster" 3; - "~WebStripper" 3; - "~Web\ Stripper" 3; - "~WebSucker" 3; - "~Web\ Sucker" 3; - "~WebWhacker" 3; - "~Web\ Whacker" 3; - "~WebZIP" 3; - "~WeSEE" 3; - "~Whack" 3; - "~Whacker" 3; - "~Whatweb" 3; - "~Widow" 3; - "~WinHTTrack" 3; - "~WiseGuys\ Robot" 3; - "~WISENutbot" 3; - "~Wonderbot" 3; - "~Woobot" 3; - "~Wotbox" 3; - "~Wprecon" 3; - "~WPScan" 3; - "~WWW-Collector-E" 3; - "~WWW-Mechanize" 3; - "~WWW::Mechanize" 3; - "~WWWOFFLE" 3; - "~x22Mozilla" 3; - "~Xaldon_WebSpider" 3; - "~Xaldon\ WebSpider" 3; - "~Xenu" 3; - "~YoudaoBot" 3; - "~Zade" 3; - "~Zermelo" 3; - "~Zeus" 3; - "~Zgrab" 3; - "~Zitebot" 3; - "~ZmEu" 3; - "~ZumBot" 3; - "~ZyBorg" 3; + "~*360Spider" 3; + "~*80legs" 3; + "~*Abonti" 3; + "~*Aboundex" 3; + "~*Acunetix" 3; + "~*ADmantX" 3; + "~*AhrefsBot" 3; + "~*AIBOT" 3; + "~*AiHitBot" 3; + "~*Aipbot" 3; + "~*Alexibot" 3; + "~*Alligator" 3; + "~*AllSubmitter" 3; + "~*Anarchie" 3; + "~*Apexoo" 3; + "~*ASPSeek" 3; + "~*Asterias" 3; + "~*Attach" 3; + "~*autoemailspider" 3; + "~*BackDoorBot" 3; + "~*BackStreet" 3; + "~*BackWeb" 3; + "~*Badass" 3; + "~*Bandit" 3; + "~*BatchFTP" 3; + "~*Battleztar\ Bazinga" 3; + "~*BBBike" 3; + "~*BDFetch" 3; + "~*BetaBot" 3; + "~*Bigfoot" 3; + "~*Bitacle" 3; + "~*Blackboard" 3; + "~*Black\ Hole" 3; + "~*BlackWidow" 3; + "~*BLEXBot" 3; + "~*Blow" 3; + "~*BlowFish" 3; + "~*Boardreader" 3; + "~*Bolt" 3; + "~*BotALot" 3; + "~*Brandprotect" 3; + "~*Brandwatch" 3; + "~*Bubing" 3; + "~*Buddy" 3; + "~*BuiltBotTough" 3; + "~*BuiltWith" 3; + "~*Bullseye" 3; + "~*BunnySlippers" 3; + "~*BuzzSumo" 3; + "~*Calculon" 3; + "~*CATExplorador" 3; + "~*CazoodleBot" 3; + "~*CCBot" 3; + "~*Cegbfeieh" 3; + "~*CheeseBot" 3; + "~*CherryPicker" 3; + "~*ChinaClaw" 3; + "~*Chlooe" 3; + "~*Claritybot" 3; + "~*Cliqzbot" 3; + "~*Cogentbot" 3; + "~*cognitiveseo" 3; + "~*Collector" 3; + "~*Copier" 3; + "~*CopyRightCheck" 3; + "~*Copyscape" 3; + "~*Cosmos" 3; + "~*Craftbot" 3; + "~*CrazyWebCrawler" 3; + "~*Crescent" 3; + "~*CSHttp" 3; + "~*Curious" 3; + "~*Custo" 3; + "~*DatabaseDriverMysqli" 3; + "~*DBLBot" 3; + "~*Demon" 3; + "~*Deusu" 3; + "~*Devil" 3; + "~*DIIbot" 3; + "~*Dirbuster" 3; + "~*Disco" 3; + "~*Discobot" 3; + "~*Discoverybot" 3; + "~*DittoSpyder" 3; + "~*DomainAppender" 3; + "~*DomainCrawler" 3; + "~*DomainSigmaCrawler" 3; + "~*Dotbot" 3; + "~*Download\ Demon" 3; + "~*Download\ Devil" 3; + "~*Download\ Wonder" 3; + "~*Dragonfly" 3; + "~*Drip" 3; + "~*DTS\ Agent" 3; + "~*EasyDL" 3; + "~*Ebingbong" 3; + "~*eCatch" 3; + "~*ECCP/1.0" 3; + "~*Ecxi" 3; + "~*EirGrabber" 3; + "~*EMail\ Collector" 3; + "~*EMail\ Extractor" 3; + "~*EMail\ Siphon" 3; + "~*EMail\ Wolf" 3; + "~*EroCrawler" 3; + "~*Evil" 3; + "~*Exabot" 3; + "~*Express\ WebPictures" 3; + "~*Extractor" 3; + "~*ExtractorPro" 3; + "~*EyeNetIE" 3; + "~*Ezooms" 3; + "~*FHscan" 3; + "~*Fimap" 3; + "~*Findxbot" 3; + "~*Firefox/7.0" 3; + "~*FlashGet" 3; + "~*Flunky" 3; + "~*Foobot" 3; + "~*Freeuploader" 3; + "~*FrontPage" 3; + "~*Fyrebot" 3; + "~*GalaxyBot" 3; + "~*Genieo" 3; + "~*Getintent" 3; + "~*GetRight" 3; + "~*GetWeb" 3; + "~*Gigablast" 3; + "~*Gigabot" 3; + "~*Go-Ahead-Got-It" 3; + "~*Gotit" 3; + "~*GoZilla" 3; + "~*Go!Zilla" 3; + "~*Grabber" 3; + "~*GrabNet" 3; + "~*Grafula" 3; + "~*GrapeFX" 3; + "~*GrapeshotCrawler" 3; + "~*GridBot" 3; + "~*GT::WWW" 3; + "~*HaosouSpider" 3; + "~*Harvest" 3; + "~*Havij" 3; + "~*HEADMasterSEO" 3; + "~*Heritrix" 3; + "~*Hloader" 3; + "~*HMView" 3; + "~*HTMLparser" 3; + "~*HTTP::Lite" 3; + "~*HTTrack" 3; + "~*Humanlinks" 3; + "~*HybridBot" 3; + "~*Iblog" 3; + "~*IDBot" 3; + "~*Id-search" 3; + "~*IlseBot" 3; + "~*Image\ Fetch" 3; + "~*Image\ Stripper" 3; + "~*Image\ Sucker" 3; + "~*Indy\ Library" 3; + "~*InfoNaviRobot" 3; + "~*InfoTekies" 3; + "~*Intelliseek" 3; + "~*InterGET" 3; + "~*Internet\ Ninja" 3; + "~*InternetSeer" 3; + "~*internetVista\ monitor" 3; + "~*Iria" 3; + "~*IRLbot" 3; + "~*Iskanie" 3; + "~*JamesBOT" 3; + "~*Jbrofuzz" 3; + "~*JennyBot" 3; + "~*JetCar" 3; + "~*JikeSpider" 3; + "~*JOC\ Web\ Spider" 3; + "~*Joomla" 3; + "~*JustView" 3; + "~*Jyxobot" 3; + "~*Kenjin\ Spider" 3; + "~*Keyword\ Density" 3; + "~*Lanshanbot" 3; + "~*Larbin" 3; + "~*LeechFTP" 3; + "~*LeechGet" 3; + "~*LexiBot" 3; + "~*Lftp" 3; + "~*LibWeb" 3; + "~*Libwhisker" 3; + "~*Lightspeedsystems" 3; + "~*Likse" 3; + "~*Linkdexbot" 3; + "~*LinkextractorPro" 3; + "~*LinkpadBot" 3; + "~*LinkScan" 3; + "~*LinksManager" 3; + "~*LinkWalker" 3; + "~*LinqiaMetadataDownloaderBot" 3; + "~*LinqiaRSSBot" 3; + "~*LinqiaScrapeBot" 3; + "~*Lipperhey" 3; + "~*Litemage_walker" 3; + "~*Lmspider" 3; + "~*LNSpiderguy" 3; + "~*Ltx71" 3; + "~*lwp-request" 3; + "~*LWP::Simple" 3; + "~*lwp-trivial" 3; + "~*Magnet" 3; + "~*Mag-Net" 3; + "~*magpie-crawler" 3; + "~*Mail.ru" 3; + "~*Majestic12" 3; + "~*MarkMonitor" 3; + "~*MarkWatch" 3; + "~*Masscan" 3; + "~*Mass\ Downloader" 3; + "~*Mata\ Hari" 3; + "~*Meanpathbot" 3; + "~*MegaIndex.ru" 3; + "~*Metauri" 3; + "~*MFC_Tear_Sample" 3; + "~*Microsoft\ Data\ Access" 3; + "~*Microsoft\ URL\ Control" 3; + "~*MIDown\ tool" 3; + "~*MIIxpc" 3; + "~*Mister\ PiX" 3; + "~*MJ12bot" 3; + "~*Mojeek" 3; + "~*MSFrontPage" 3; + "~*MSIE\ 6.0" 3; + "~*MSIECrawler" 3; + "~*Msrabot" 3; + "~*MS\ Web\ Services\ Client\ Protocol" 3; + "~*Musobot" 3; + "~*Name\ Intelligence" 3; + "~*Nameprotect" 3; + "~*Navroad" 3; + "~*NearSite" 3; + "~*Needle" 3; + "~*Nessus" 3; + "~*NetAnts" 3; + "~*Netcraft" 3; + "~*netEstate\ NE\ Crawler" 3; + "~*NetLyzer" 3; + "~*NetMechanic" 3; + "~*NetSpider" 3; + "~*Nettrack" 3; + "~*Net\ Vampire" 3; + "~*NetZIP" 3; + "~*NextGenSearchBot" 3; + "~*Nibbler" 3; + "~*NICErsPRO" 3; + "~*Niki-bot" 3; + "~*Nikto" 3; + "~*NimbleCrawler" 3; + "~*Ninja" 3; + "~*Nmap" 3; + "~*NPbot" 3; + "~*Nutch" 3; + "~*Octopus" 3; + "~*Offline\ Explorer" 3; + "~*Offline\ Navigator" 3; + "~*Openfind" 3; + "~*OpenLinkProfiler" 3; + "~*Openvas" 3; + "~*OrangeBot" 3; + "~*OrangeSpider" 3; + "~*OutfoxBot" 3; + "~*PageAnalyzer" 3; + "~*Page\ Analyzer" 3; + "~*PageGrabber" 3; + "~*Page\ Grabber" 3; + "~*page\ scorer" 3; + "~*PageScorer" 3; + "~*Panscient" 3; + "~*Papa\ Foto" 3; + "~*Pavuk" 3; + "~*pcBrowser" 3; + "~*PECL::HTTP" 3; + "~*PeoplePal" 3; + "~*PHPCrawl" 3; + "~*Picscout" 3; + "~*Picsearch" 3; + "~*Pimonster" 3; + "~*Pi-Monster" 3; + "~*Pixray" 3; + "~*PleaseCrawl" 3; + "~*Pockey" 3; + "~*POE-Component-Client-HTTP" 3; + "~*Probethenet" 3; + "~*ProPowerBot" 3; + "~*ProWebWalker" 3; + "~*Proximic" 3; + "~*Psbot" 3; + "~*Pump" 3; + "~*PyCurl" 3; + "~*QueryN\ Metasearch" 3; + "~*Qwantify" 3; + "~*RankActiveLinkBot" 3; + "~*Rankivabot" 3; + "~*RealDownload" 3; + "~*Reaper" 3; + "~*Recorder" 3; + "~*RedesScrapy" 3; + "~*ReGet" 3; + "~*RepoMonkey" 3; + "~*Ripper" 3; + "~*RocketCrawler" 3; + "~*Rogerbot" 3; + "~*SalesIntelligent" 3; + "~*SBIder" 3; + "~*ScanAlert" 3; + "~*Scanbot" 3; + "~*Scrapy" 3; + "~*Screaming" 3; + "~*Screaming\ Frog\ SEO\ Spider" 3; + "~*ScreenerBot" 3; + "~*Searchestate" 3; + "~*SearchmetricsBot" 3; + "~*Semrush" 3; + "~*SemrushBot" 3; + "~*SEOkicks" 3; + "~*SEOkicks-Robot" 3; + "~*SEOlyticsCrawler" 3; + "~*Seomoz" 3; + "~*SEOprofiler" 3; + "~*seoscanners" 3; + "~*SEOstats" 3; + "~*Siphon" 3; + "~*SISTRIX" 3; + "~*SISTRIX\ Crawler" 3; + "~*Sitebeam" 3; + "~*SiteExplorer" 3; + "~*Siteimprove" 3; + "~*SiteLockSpider" 3; + "~*SiteSnagger" 3; + "~*SiteSucker" 3; + "~*Site\ Sucker" 3; + "~*Sitevigil" 3; + "~*Slackbot-LinkExpanding" 3; + "~*SlySearch" 3; + "~*SmartDownload" 3; + "~*Snake" 3; + "~*Snapbot" 3; + "~*Snoopy" 3; + "~*SocialRankIOBot" 3; + "~*Sogou\ web\ spider" 3; + "~*Sosospider" 3; + "~*SpaceBison" 3; + "~*Spammen" 3; + "~*SpankBot" 3; + "~*Spanner" 3; + "~*Spbot" 3; + "~*Spinn3r" 3; + "~*SputnikBot" 3; + "~*Sqlmap" 3; + "~*Sqlworm" 3; + "~*Sqworm" 3; + "~*Steeler" 3; + "~*Stripper" 3; + "~*Sucker" 3; + "~*Sucuri" 3; + "~*SuperBot" 3; + "~*SuperHTTP" 3; + "~*Surfbot" 3; + "~*SurveyBot" 3; + "~*Suzuran" 3; + "~*Swiftbot" 3; + "~*Szukacz" 3; + "~*T0PHackTeam" 3; + "~*T8Abot" 3; + "~*tAkeOut" 3; + "~*Teleport" 3; + "~*TeleportPro" 3; + "~*Telesoft" 3; + "~*Telesphoreo" 3; + "~*Telesphorep" 3; + "~*The\ Intraformant" 3; + "~*TheNomad" 3; + "~*TightTwatBot" 3; + "~*Titan" 3; + "~*Toata" 3; + "~*Toweyabot" 3; + "~*Trendictionbot" 3; + "~*True_Robot" 3; + "~*Turingos" 3; + "~*TurnitinBot" 3; + "~*Turnitin\ Bot" 3; + "~*Turnitin\ Robot" 3; + "~*TwengaBot" 3; + "~*Twice" 3; + "~*Typhoeus" 3; + "~*UnisterBot" 3; + "~*URLy.Warning" 3; + "~*URLy\ Warning" 3; + "~*Vacuum" 3; + "~*Vagabondo" 3; + "~*VB\ Project" 3; + "~*VCI" 3; + "~*VeriCiteCrawler" 3; + "~*VidibleScraper" 3; + "~*VoidEYE" 3; + "~*Voil" 3; + "~*Voltron" 3; + "~*Wallpapers/3.0" 3; + "~*WallpapersHD" 3; + "~*WASALive-Bot" 3; + "~*WBSearchBot" 3; + "~*Webalta" 3; + "~*WebAuto" 3; + "~*Web\ Auto" 3; + "~*WebBandit" 3; + "~*Web\ Bandit" 3; + "~*WebCollage" 3; + "~*Web\ Collage" 3; + "~*WebCopier" 3; + "~*Web\ Copier" 3; + "~*WEBDAV" 3; + "~*WEBDAV\ Client" 3; + "~*WebEnhancer" 3; + "~*Web\ Enhancer" 3; + "~*WebFetch" 3; + "~*Web\ Fetch" 3; + "~*WebFuck" 3; + "~*Web\ Fuck" 3; + "~*WebGo\ IS" 3; + "~*WebImageCollector" 3; + "~*Web\ Image\ Collector" 3; + "~*WebLeacher" 3; + "~*WebmasterWorldForumBot" 3; + "~*webmeup-crawler" 3; + "~*WebPix" 3; + "~*Web\ Pix" 3; + "~*WebReaper" 3; + "~*Web\ Reaper" 3; + "~*WebSauger" 3; + "~*Web\ Sauger" 3; + "~*Webshag" 3; + "~*WebsiteExtractor" 3; + "~*Website\ Extractor" 3; + "~*WebsiteQuester" 3; + "~*Website\ Quester" 3; + "~*Webster" 3; + "~*WebStripper" 3; + "~*Web\ Stripper" 3; + "~*WebSucker" 3; + "~*Web\ Sucker" 3; + "~*WebWhacker" 3; + "~*Web\ Whacker" 3; + "~*WebZIP" 3; + "~*WeSEE" 3; + "~*Whack" 3; + "~*Whacker" 3; + "~*Whatweb" 3; + "~*Widow" 3; + "~*WinHTTrack" 3; + "~*WiseGuys\ Robot" 3; + "~*WISENutbot" 3; + "~*Wonderbot" 3; + "~*Woobot" 3; + "~*Wotbox" 3; + "~*Wprecon" 3; + "~*WPScan" 3; + "~*WWW-Collector-E" 3; + "~*WWW-Mechanize" 3; + "~*WWW::Mechanize" 3; + "~*WWWOFFLE" 3; + "~*x22Mozilla" 3; + "~*Xaldon_WebSpider" 3; + "~*Xaldon\ WebSpider" 3; + "~*Xenu" 3; + "~*YoudaoBot" 3; + "~*Zade" 3; + "~*Zermelo" 3; + "~*Zeus" 3; + "~*Zgrab" 3; + "~*Zitebot" 3; + "~*ZmEu" 3; + "~*ZumBot" 3; + "~*ZyBorg" 3; # END BAD BOTS ### DO NOT EDIT THIS LINE AT ALL ### }