From a4cfef2d40423b68b63cf8ed6d15ee86d0cad8ec Mon Sep 17 00:00:00 2001
From: Renan Bernordi <altendorfme@gmail.com>
Date: Thu, 28 Nov 2024 17:02:56 -0300
Subject: [PATCH] =?UTF-8?q?adicionado=20robots=20bloqueando=20indexa=C3=A7?=
 =?UTF-8?q?=C3=A3o=20de=20paginas=20internas=20e=20ias?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 app/robots.txt | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 app/robots.txt

diff --git a/app/robots.txt b/app/robots.txt
new file mode 100644
index 0000000..454a50c
--- /dev/null
+++ b/app/robots.txt
@@ -0,0 +1,34 @@
+User-agent: *
+Disallow: /p/
+Disallow: /api/
+
+# Block common AI crawlers
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: FacebookBot
+Disallow: /
+
+User-agent: Claude-Web
+Disallow: /
+
+User-agent: Omgilibot
+Disallow: /
+
+User-agent: Omgili
+Disallow: /
+
+User-agent: FacebookBot
+Disallow: /
\ No newline at end of file