try to handle redirections with empty content from BNF archives (cf #426

, to be tested)
medialab · Oct 29, 2021 · 5075f5a · 5075f5a
1 parent 72f5f80
commit 5075f5a
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/hyphe_backend/crawler/hcicrawler/spiders/pages.py b/hyphe_backend/crawler/hcicrawler/spiders/pages.py
@@ -231,7 +231,8 @@ def handle_response(self, response):
 
         if self.webarchives:
             # Handle transparently redirections from archives to another available timestamp
-            if response.status == 302:
+            if response.status == 302 or \
+              ("archivesinternet.bnf.fr" in self.webarchives["url_prefix"] and 300 <= response.status < 400 and not response.body):
                 redir_url = response.headers['Location']
                 if redir_url.startswith("/"):
                     redir_url = "%s%s" % (self.archivehost, redir_url)