try to not deduplicate requests coming from empty redirections (#426)

medialab · Nov 3, 2021 · 378a6c2 · 378a6c2
1 parent 5075f5a
commit 378a6c2
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/hyphe_backend/crawler/hcicrawler/spiders/pages.py b/hyphe_backend/crawler/hcicrawler/spiders/pages.py
@@ -241,7 +241,7 @@ def handle_response(self, response):
                         response.meta['depth'] -= 1
                     else:
                         response.meta['depth'] = -1
-                    return self._request(redir_url, redirection=True)
+                    return self._request(redir_url, redirection=True, dont_filter=(not response.status))
                 real_url = self.archiveregexp.sub("", redir_url)
                 orig_url = self.archiveregexp.sub("", response.url)
                 match = self.archiveregexp.search(redir_url)