From 0a99260bcf8913f1dfd66011d16a2c62d1d51eb7 Mon Sep 17 00:00:00 2001 From: Mitchell Krog UB1 Date: Thu, 12 Jan 2017 16:12:00 +0200 Subject: [PATCH] Added robots.txt for People Unable to Use this Bot Blocker --- robots.txt/README.md | 27 ++ robots.txt/robots.txt | 852 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 879 insertions(+) create mode 100644 robots.txt/README.md create mode 100644 robots.txt/robots.txt diff --git a/robots.txt/README.md b/robots.txt/README.md new file mode 100644 index 000000000..d0ed87b2e --- /dev/null +++ b/robots.txt/README.md @@ -0,0 +1,27 @@ +# Robots.txt Directives ONLY for People unable to use this Bot Blocker +## DO NOT use this on any of your sites if you are running the Bad Bot Blocker + +The robots.txt file residing in this directory can be used to ADD onto your existing robots.txt file by placing this after anything you already have in your robots.txt file. + +## Do NOT make this file the only contents of your robots.txt file !!! + +If your existing robots.txt file looks like this: +```User-agent: * +Allow: /wp-admin/admin-ajax.php``` + +Then your new file will look like this: +```User-agent: * +Allow: /wp-admin/admin-ajax.php +User-agent: Acunetix + Disallow:/ +User-agent: FHscan + Disallow:/ +User-agent: masscan + Disallow:/ +........ rest of file contents``` + +# You use this at your own risk + +This will only help stop some bad bots from gaining access to your site +this certainly does not provide nearly as much protection as the full +Nginx Bad Bot Blocker diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt new file mode 100644 index 000000000..561f8753a --- /dev/null +++ b/robots.txt/robots.txt @@ -0,0 +1,852 @@ +User-agent: Acunetix + Disallow:/ +User-agent: FHscan + Disallow:/ +User-agent: masscan + Disallow:/ +User-agent: probethenet + Disallow:/ +User-agent: scanbot + Disallow:/ +User-agent: 80legs + Disallow:/ +User-agent: 360Spider + Disallow:/ +User-agent: Aboundex + Disallow:/ +User-agent: Abonti + Disallow:/ +User-agent: AhrefsBot + Disallow:/ +User-agent: AIBOT + Disallow:/ +User-agent: aipbot + Disallow:/ +User-agent: aiHitBot + Disallow:/ +User-agent: Alexibot + Disallow:/ +User-agent: Alligator + Disallow:/ +User-agent: AllSubmitter + Disallow:/ +User-agent: Anarchie + Disallow:/ +User-agent: Apexoo + Disallow:/ +User-agent: asterias + Disallow:/ +User-agent: ASPSeek + Disallow:/ +User-agent: attach + Disallow:/ +User-agent: autoemailspider + Disallow:/ +User-agent: BackDoorBot + Disallow:/ +User-agent: BackStreet + Disallow:/ +User-agent: BackWeb + Disallow:/ +User-agent: Badass + Disallow:/ +User-agent: Bandit + Disallow:/ +User-agent: BatchFTP + Disallow:/ +User-agent: Battleztar Bazinga + Disallow:/ +User-agent: BBBike + Disallow:/ +User-agent: BDFetch + Disallow:/ +User-agent: Bigfoot + Disallow:/ +User-agent: Bitacle + Disallow:/ +User-agent: Black Hole + Disallow:/ +User-agent: BlackWidow + Disallow:/ +User-agent: BLEXBot + Disallow:/ +User-agent: BlowFish + Disallow:/ +User-agent: Blow + Disallow:/ +User-agent: boardreader + Disallow:/ +User-agent: BotALot + Disallow:/ +User-agent: Brandprotect + Disallow:/ +User-agent: Brandwatch + Disallow:/ +User-agent: Buddy + Disallow:/ +User-agent: BuiltBotTough + Disallow:/ +User-agent: BuiltWith + Disallow:/ +User-agent: Bullseye + Disallow:/ +User-agent: BunnySlippers + Disallow:/ +User-agent: BuzzSumo + Disallow:/ +User-agent: calculon spider + Disallow:/ +User-agent: CCBot + Disallow:/ +User-agent: Cegbfeieh + Disallow:/ +User-agent: CheeseBot + Disallow:/ +User-agent: CherryPicker + Disallow:/ +User-agent: ChinaClaw + Disallow:/ +User-agent: chlooe + Disallow:/ +User-agent: Claritybot + Disallow:/ +User-agent: Cliqzbot + Disallow:/ +User-agent: Cogentbot + Disallow:/ +User-agent: Collector + Disallow:/ +User-agent: Copier + Disallow:/ +User-agent: CopyRightCheck + Disallow:/ +User-agent: Copyscape + Disallow:/ +User-agent: Cosmos + Disallow:/ +User-agent: craftbot@yahoo.com + Disallow:/ +User-agent: CrazyWebCrawler + Disallow:/ +User-agent: Crescent + Disallow:/ +User-agent: CSHttp + Disallow:/ +User-agent: Custo + Disallow:/ +User-agent: DBLBot + Disallow:/ +User-agent: Demon + Disallow:/ +User-agent: deusu + Disallow:/ +User-agent: Devil + Disallow:/ +User-agent: Disco + Disallow:/ +User-agent: DIIbot + Disallow:/ +User-agent: dirbuster + Disallow:/ +User-agent: discobot + Disallow:/ +User-agent: discoverybot + Disallow:/ +User-agent: DittoSpyder + Disallow:/ +User-agent: DomainAppender + Disallow:/ +User-agent: DomainSigmaCrawler + Disallow:/ +User-agent: dotbot + Disallow:/ +User-agent: Download Demon + Disallow:/ +User-agent: Download Devil + Disallow:/ +User-agent: Download Wonder + Disallow:/ +User-agent: dragonfly + Disallow:/ +User-agent: Drip + Disallow:/ +User-agent: DTS Agent + Disallow:/ +User-agent: EasyDL + Disallow:/ +User-agent: ebingbong + Disallow:/ +User-agent: eCatch + Disallow:/ +User-agent: ECCP/1.0 + Disallow:/ +User-agent: EirGrabber + Disallow:/ +User-agent: EMail Collector + Disallow:/ +User-agent: EmailCollector + Disallow:/ +User-agent: EMail Extractor + Disallow:/ +User-agent: EMail Siphon + Disallow:/ +User-agent: EmailSiphon + Disallow:/ +User-agent: EMail Wolf + Disallow:/ +User-agent: EmailWolf + Disallow:/ +User-agent: EroCrawler + Disallow:/ +User-agent: Evil + Disallow:/ +User-agent: Exabot + Disallow:/ +User-agent: Express WebPictures + Disallow:/ +User-agent: Extractor + Disallow:/ +User-agent: ExtractorPro + Disallow:/ +User-agent: EyeNetIE + Disallow:/ +User-agent: ezooms + Disallow:/ +User-agent: fimap + Disallow:/ +User-agent: FlashGet + Disallow:/ +User-agent: flunky + Disallow:/ +User-agent: Foobot + Disallow:/ +User-agent: freeuploader + Disallow:/ +User-agent: FrontPage + Disallow:/ +User-agent: GalaxyBot + Disallow:/ +User-agent: Genieo + Disallow:/ +User-agent: getintent + Disallow:/ +User-agent: GetRight + Disallow:/ +User-agent: GetWeb + Disallow:/ +User-agent: gigablast + Disallow:/ +User-agent: gigabot + Disallow:/ +User-agent: Go-Ahead-Got-It + Disallow:/ +User-agent: gotit + Disallow:/ +User-agent: Go!Zilla + Disallow:/ +User-agent: Grabber + Disallow:/ +User-agent: GrabNet + Disallow:/ +User-agent: Grafula + Disallow:/ +User-agent: grapeFX + Disallow:/ +User-agent: GrapeshotCrawler + Disallow:/ +User-agent: GridBot + Disallow:/ +User-agent: HaosouSpider + Disallow:/ +User-agent: Harvest + Disallow:/ +User-agent: Havij + Disallow:/ +User-agent: Heritrix + Disallow:/ +User-agent: hloader + Disallow:/ +User-agent: HMView + Disallow:/ +User-agent: HTTrack + Disallow:/ +User-agent: HTMLparser + Disallow:/ +User-agent: Huaweisymantecspider + Disallow:/ +User-agent: humanlinks + Disallow:/ +User-agent: iblog + Disallow:/ +User-agent: IlseBot + Disallow:/ +User-agent: imagefetch + Disallow:/ +User-agent: Image Fetch + Disallow:/ +User-agent: Image Stripper + Disallow:/ +User-agent: Image Sucker + Disallow:/ +User-agent: Indy Library + Disallow:/ +User-agent: InfoNaviRobot + Disallow:/ +User-agent: InfoTekies + Disallow:/ +User-agent: Intelliseek + Disallow:/ +User-agent: Internet Ninja + Disallow:/ +User-agent: internetVista monitor + Disallow:/ +User-agent: InterGET + Disallow:/ +User-agent: Iria + Disallow:/ +User-agent: iskanie + Disallow:/ +User-agent: JamesBOT + Disallow:/ +User-agent: JennyBot + Disallow:/ +User-agent: JetCar + Disallow:/ +User-agent: jbrofuzz + Disallow:/ +User-agent: JikeSpider + Disallow:/ +User-agent: JOC + Disallow:/ +User-agent: JOC Web Spider + Disallow:/ +User-agent: Joomla + Disallow:/ +User-agent: JustView + Disallow:/ +User-agent: Jyxobot + Disallow:/ +User-agent: Kenjin Spider + Disallow:/ +User-agent: Keyword Density + Disallow:/ +User-agent: kraken + Disallow:/ +User-agent: lanshanbot + Disallow:/ +User-agent: larbin + Disallow:/ +User-agent: LeechFTP + Disallow:/ +User-agent: LeechGet + Disallow:/ +User-agent: LexiBot + Disallow:/ +User-agent: lftp + Disallow:/ +User-agent: libcurl + Disallow:/ +User-agent: libWeb + Disallow:/ +User-agent: libwhisker + Disallow:/ +User-agent: likse + Disallow:/ +User-agent: linkdexbot + Disallow:/ +User-agent: LinkpadBot + Disallow:/ +User-agent: LinkextractorPro + Disallow:/ +User-agent: LinkScan + Disallow:/ +User-agent: LinqiaMetadataDownloaderBot + Disallow:/ +User-agent: LinqiaScrapeBot + Disallow:/ +User-agent: LinqiaRSSBot + Disallow:/ +User-agent: lmspider + Disallow:/ +User-agent: LNSpiderguy + Disallow:/ +User-agent: LinkWalker + Disallow:/ +User-agent: ltx71 + Disallow:/ +User-agent: lwp-request + Disallow:/ +User-agent: LWP::Simple + Disallow:/ +User-agent: magpie-crawler + Disallow:/ +User-agent: Magnet + Disallow:/ +User-agent: Mag-Net + Disallow:/ +User-agent: mail.ru + Disallow:/ +User-agent: Mail.RU_Bot + Disallow:/ +User-agent: majestic12 + Disallow:/ +User-agent: MarkMonitor + Disallow:/ +User-agent: MarkWatch + Disallow:/ +User-agent: Mass Downloader + Disallow:/ +User-agent: Mata Hari + Disallow:/ +User-agent: MegaIndex.ru + Disallow:/ +User-agent: Metauri + Disallow:/ +User-agent: Microsoft Data Access + Disallow:/ +User-agent: Microsoft URL Control + Disallow:/ +User-agent: MIDown tool + Disallow:/ +User-agent: MIIxpc + Disallow:/ +User-agent: Mister PiX + Disallow:/ +User-agent: MJ12bot + Disallow:/ +User-agent: mojeek + Disallow:/ +User-agent: MSIECrawler + Disallow:/ +User-agent: MSIE 6.0 + Disallow:/ +User-agent: MS Web Services Client Protocol + Disallow:/ +User-agent: msrabot + Disallow:/ +User-agent: musobot + Disallow:/ +User-agent: Name Intelligence + Disallow:/ +User-agent: Nameprotect + Disallow:/ +User-agent: Navroad + Disallow:/ +User-agent: NearSite + Disallow:/ +User-agent: needle + Disallow:/ +User-agent: nessus + Disallow:/ +User-agent: NetAnts + Disallow:/ +User-agent: Netcraft + Disallow:/ +User-agent: netEstate NE Crawler + Disallow:/ +User-agent: NetMechanic + Disallow:/ +User-agent: NetSpider + Disallow:/ +User-agent: nettrack + Disallow:/ +User-agent: Net Vampire + Disallow:/ +User-agent: NetZIP + Disallow:/ +User-agent: NextGenSearchBot + Disallow:/ +User-agent: nibbler + Disallow:/ +User-agent: NICErsPRO + Disallow:/ +User-agent: niki-bot + Disallow:/ +User-agent: nikto + Disallow:/ +User-agent: NimbleCrawler + Disallow:/ +User-agent: Ninja + Disallow:/ +User-agent: Nmap + Disallow:/ +User-agent: NPbot + Disallow:/ +User-agent: Nutch + Disallow:/ +User-agent: Octopus + Disallow:/ +User-agent: Offline Explorer + Disallow:/ +User-agent: Offline Navigator + Disallow:/ +User-agent: omgilibot + Disallow:/ +User-agent: OpenLinkProfiler + Disallow:/ +User-agent: Openfind + Disallow:/ +User-agent: Openvas + Disallow:/ +User-agent: OrangeBot + Disallow:/ +User-agent: OrangeSpider + Disallow:/ +User-agent: OutfoxBot + Disallow:/ +User-agent: PageAnalyzer + Disallow:/ +User-agent: Page Analyzer + Disallow:/ +User-agent: PageGrabber + Disallow:/ +User-agent: Page Grabber + Disallow:/ +User-agent: page scorer + Disallow:/ +User-agent: Papa Foto + Disallow:/ +User-agent: pavuk + Disallow:/ +User-agent: pcBrowser + Disallow:/ +User-agent: PECL::HTTP + Disallow:/ +User-agent: Picscout + Disallow:/ +User-agent: Picsearch + Disallow:/ +User-agent: pimonster + Disallow:/ +User-agent: Pi-Monster + Disallow:/ +User-agent: Pixray + Disallow:/ +User-agent: Pockey + Disallow:/ +User-agent: POE-Component-Client-HTTP + Disallow:/ +User-agent: ProPowerBot + Disallow:/ +User-agent: ProWebWalker + Disallow:/ +User-agent: proximic + Disallow:/ +User-agent: psbot + Disallow:/ +User-agent: Pump + Disallow:/ +User-agent: PyCurl + Disallow:/ +User-agent: QueryN Metasearch + Disallow:/ +User-agent: RankActiveLinkBot + Disallow:/ +User-agent: Rankivabot + Disallow:/ +User-agent: RealDownload + Disallow:/ +User-agent: Reaper + Disallow:/ +User-agent: Ripper + Disallow:/ +User-agent: Recorder + Disallow:/ +User-agent: ReGet + Disallow:/ +User-agent: RepoMonkey + Disallow:/ +User-agent: Rogerbot + Disallow:/ +User-agent: ScanAlert + Disallow:/ +User-agent: Scrapy + Disallow:/ +User-agent: Screaming Frog SEO Spider + Disallow:/ +User-agent: ScreenerBot + Disallow:/ +User-agent: searchestate + Disallow:/ +User-agent: SearchmetricsBot + Disallow:/ +User-agent: Semrush + Disallow:/ +User-agent: SemrushBot + Disallow:/ +User-agent: SEOkicks + Disallow:/ +User-agent: SEOkicks-Robot + Disallow:/ +User-agent: SEOlyticsCrawler + Disallow:/ +User-agent: seomoz + Disallow:/ +User-agent: SEOstats + Disallow:/ +User-agent: SISTRIX + Disallow:/ +User-agent: Siphon + Disallow:/ +User-agent: sitebeam + Disallow:/ +User-agent: SiteExplorer + Disallow:/ +User-agent: Siteimprove + Disallow:/ +User-agent: SiteLockSpider + Disallow:/ +User-agent: SiteSucker + Disallow:/ +User-agent: Site Sucker + Disallow:/ +User-agent: SiteSnagger + Disallow:/ +User-agent: sitevigil + Disallow:/ +User-agent: Slackbot-LinkExpanding + Disallow:/ +User-agent: SlySearch + Disallow:/ +User-agent: SmartDownload + Disallow:/ +User-agent: Snake + Disallow:/ +User-agent: Snapbot + Disallow:/ +User-agent: Snoopy + Disallow:/ +User-agent: SocialRankIOBot + Disallow:/ +User-agent: Sogou web spider + Disallow:/ +User-agent: Sosospider + Disallow:/ +User-agent: SpaceBison + Disallow:/ +User-agent: spammen + Disallow:/ +User-agent: spanner + Disallow:/ +User-agent: SpankBot + Disallow:/ +User-agent: Spinn3r + Disallow:/ +User-agent: spbot + Disallow:/ +User-agent: sqlmap + Disallow:/ +User-agent: Sqlworm + Disallow:/ +User-agent: Sqworm + Disallow:/ +User-agent: Stripper + Disallow:/ +User-agent: Sucker + Disallow:/ +User-agent: Sucuri + Disallow:/ +User-agent: SuperBot + Disallow:/ +User-agent: SuperHTTP + Disallow:/ +User-agent: Surfbot + Disallow:/ +User-agent: SurveyBot + Disallow:/ +User-agent: suzuran + Disallow:/ +User-agent: Swiftbot + Disallow:/ +User-agent: SISTRIX Crawler + Disallow:/ +User-agent: Szukacz + Disallow:/ +User-agent: T8Abot + Disallow:/ +User-agent: tAkeOut + Disallow:/ +User-agent: Teleport + Disallow:/ +User-agent: TeleportPro + Disallow:/ +User-agent: Telesoft + Disallow:/ +User-agent: Telesphoreo + Disallow:/ +User-agent: Telesphorep + Disallow:/ +User-agent: The Intraformant + Disallow:/ +User-agent: TheNomad + Disallow:/ +User-agent: TightTwatBot + Disallow:/ +User-agent: Titan + Disallow:/ +User-agent: T0PHackTeam + Disallow:/ +User-agent: Toweyabot + Disallow:/ +User-agent: Trendictionbot + Disallow:/ +User-agent: True_Robot + Disallow:/ +User-agent: Turingos + Disallow:/ +User-agent: TurnitinBot + Disallow:/ +User-agent: Turnitin Robot + Disallow:/ +User-agent: Turnitin Bot + Disallow:/ +User-agent: Twice + Disallow:/ +User-agent: UptimeRobot/2.0 + Disallow:/ +User-agent: URLy.Warning + Disallow:/ +User-agent: URLy Warning + Disallow:/ +User-agent: Vacuum + Disallow:/ +User-agent: VB Project + Disallow:/ +User-agent: VCI + Disallow:/ +User-agent: VeriCiteCrawler + Disallow:/ +User-agent: VidibleScraper + Disallow:/ +User-agent: VoidEYE + Disallow:/ +User-agent: Voil + Disallow:/ +User-agent: WASALive-Bot + Disallow:/ +User-agent: WBSearchBot + Disallow:/ +User-agent: WebAuto + Disallow:/ +User-agent: WebBandit + Disallow:/ +User-agent: Web Bandit + Disallow:/ +User-agent: WebCopier + Disallow:/ +User-agent: Web Copier + Disallow:/ +User-agent: WEBDAV + Disallow:/ +User-agent: WEBDAV Client + Disallow:/ +User-agent: WebEnhancer + Disallow:/ +User-agent: Web Enhancer + Disallow:/ +User-agent: WebFetch + Disallow:/ +User-agent: Web Fetch + Disallow:/ +User-agent: WebFuck + Disallow:/ +User-agent: Web Fuck + Disallow:/ +User-agent: WebGo IS + Disallow:/ +User-agent: Web Image Collector + Disallow:/ +User-agent: WebLeacher + Disallow:/ +User-agent: WebmasterWorldForumBot + Disallow:/ +User-agent: WebPix + Disallow:/ +User-agent: Web Pix + Disallow:/ +User-agent: WebReaper + Disallow:/ +User-agent: Web Reaper + Disallow:/ +User-agent: WebSauger + Disallow:/ +User-agent: Web Sauger + Disallow:/ +User-agent: webshag + Disallow:/ +User-agent: WebsiteExtractor + Disallow:/ +User-agent: Website Extractor + Disallow:/ +User-agent: WebsiteQuester + Disallow:/ +User-agent: Website Quester + Disallow:/ +User-agent: Webster + Disallow:/ +User-agent: WebStripper + Disallow:/ +User-agent: Web Stripper + Disallow:/ +User-agent: WebSucker + Disallow:/ +User-agent: Web Sucker + Disallow:/ +User-agent: WebWhacker + Disallow:/ +User-agent: Web Whacker + Disallow:/ +User-agent: WebZIP + Disallow:/ +User-agent: WeSEE + Disallow:/ +User-agent: Whack + Disallow:/ +User-agent: Whacker + Disallow:/ +User-agent: whatweb + Disallow:/ +User-agent: Widow + Disallow:/ +User-agent: WinHTTrack + Disallow:/ +User-agent: WiseGuys Robot + Disallow:/ +User-agent: WISENutbot + Disallow:/ +User-agent: wonderbot + Disallow:/ +User-agent: woobot + Disallow:/ +User-agent: Wotbox + Disallow:/ +User-agent: wprecon + Disallow:/ +User-agent: WWWOFFLE + Disallow:/ +User-agent: WWW::Mechanize + Disallow:/ +User-agent: WWW-Collector-E + Disallow:/ +User-agent: Xaldon_WebSpider + Disallow:/ +User-agent: Xaldon WebSpider + Disallow:/ +User-agent: Xenu + Disallow:/ +User-agent: Y!J-ASR + Disallow:/ +User-agent: YoudaoBot + Disallow:/ +User-agent: Zade + Disallow:/ +User-agent: Zeus + Disallow:/ +User-agent: zgrab + Disallow:/ +User-agent: zitebot + Disallow:/ +User-agent: ZmEu + Disallow:/ +User-agent: ZumBot + Disallow:/