From 48709906afebf3ab39c545c7c2a2c4adead1e31a Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Fri, 4 Oct 2024 11:21:18 +0200 Subject: [PATCH] Upate robots.txt to block the AI bots --- pombola/templates/robots.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pombola/templates/robots.txt b/pombola/templates/robots.txt index bea65fbbb..d3587d78f 100644 --- a/pombola/templates/robots.txt +++ b/pombola/templates/robots.txt @@ -10,3 +10,17 @@ Disallow: /feedback Disallow: /admin Disallow: /county-performance +User-agent: meta-externalagent +Disallow: / +User-agent: Amazonbot +Disallow: / +User-agent: Bytespider +Disallow: / +User-Agent: PerplexityBot +Disallow: / +User-agent: ClaudeBot +Disallow: / +User-agent: OAI-SearchBot +Disallow: / +User-agent: GPTBot +Disallow: / \ No newline at end of file