Merge pull request #233 from OpenUpSA/feature/block-ai-bots

Update robots.txt to block the AI bots
OpenUpSA · Oct 4, 2024 · 0e95612 · 0e95612
2 parents 7120468 + 4870990
commit 0e95612
Showing 1 changed file with 14 additions and 0 deletions.
diff --git a/pombola/templates/robots.txt b/pombola/templates/robots.txt
@@ -10,3 +10,17 @@ Disallow: /feedback
 Disallow: /admin
 Disallow: /county-performance
 
+User-agent: meta-externalagent
+Disallow: /
+User-agent: Amazonbot
+Disallow: /
+User-agent: Bytespider
+Disallow: /
+User-Agent: PerplexityBot
+Disallow: /
+User-agent: ClaudeBot
+Disallow: /
+User-agent: OAI-SearchBot
+Disallow: /
+User-agent: GPTBot
+Disallow: /