Add spiders and handle no user agent for capybara on #4130

benwbrum · May 27, 2024 · e560281 · e560281
1 parent a453fd6
commit e560281
Showing 1 changed file with 4 additions and 5 deletions.
diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb
@@ -31,18 +31,17 @@ class Rack::Attack
   # see https://github.com/benwbrum/fromthepage/issues/4130
   ### Throttle requests by agent ClaudeBot 
   throttle('requests by agent ClaudeBot', limit: 6, period: 1.minute) do |req|
-    req.user_agent.match?(/ClaudeBot/)
+    req.user_agent&.match?(/ClaudeBot/)
   end
 
   ### Throttle requests by agent ByteDance
   throttle('requests by agent ByteDance', limit: 6, period: 1.minute) do |req|
-    req.user_agent.match?(/Bytespider/)
+    req.user_agent&.match?(/Bytespider/)
   end
 
   ### Throttle requests by low-rent SEO bots
-  throttle('requests by agent ByteDance', limit: 6, period: 1.minute) do |req|
-    req.user_agent.match?(/SemrushBot/) ||
-    req.user_agent.match?(/DataForSeoBot/)
+  throttle('requests by various SEO bots', limit: 15, period: 1.minute) do |req|
+    req.user_agent&.match?(/(SemrushBot|AhrefsBot|DataForSeoBot|AhrefsBot|DotBot|MJ12bot|PetalBot)/)
   end