From ff1e1bcc860168111d6036999a04678680aeb0b7 Mon Sep 17 00:00:00 2001
From: Renan Bernordi <altendorfme@gmail.com>
Date: Thu, 30 Jan 2025 13:03:15 -0300
Subject: [PATCH] bloqueios no robots

---
 app/robots.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/robots.txt b/app/robots.txt
index a8782dd..32ffde7 100644
--- a/app/robots.txt
+++ b/app/robots.txt
@@ -17,10 +17,11 @@ User-agent: ChatGPT-User
 User-agent: Claude-Web
 User-agent: ClaudeBot
 User-agent: cohere-ai
+User-agent: cohere-training-data-crawler
+User-agent: Crawlspace
 User-agent: Diffbot
 User-agent: DuckAssistBot
 User-agent: FacebookBot
-User-agent: facebookexternalhit
 User-agent: FriendlyCrawler
 User-agent: Google-Extended
 User-agent: GoogleOther
@@ -38,9 +39,11 @@ User-agent: Meta-ExternalFetcher
 User-agent: OAI-SearchBot
 User-agent: omgili
 User-agent: omgilibot
+User-agent: PanguBot
 User-agent: PerplexityBot
 User-agent: PetalBot
 User-agent: Scrapy
+User-agent: SemrushBot
 User-agent: Sidetrade indexer bot
 User-agent: Timpibot
 User-agent: VelenPublicWebCrawler