chore: update robots.txt disallow list

OpenBeta · May 13, 2024 · f5c5e58 · f5c5e58
1 parent 01663db
commit f5c5e58
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -1,6 +1,12 @@
 User-agent: PetalBot
 Disallow: /
 
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: AhrefsBot
+Disallow: /
+
 User-agent: *
 Disallow: /account/
 Disallow: /api/