Dealing with could_be_html false negatives

Fix #183
medialab · Aug 17, 2023 · 8ab9889 · 8ab9889
1 parent 020a41a
commit 8ab9889
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 5 deletions.
diff --git a/test/could_be_html_test.py b/test/could_be_html_test.py
@@ -19,6 +19,10 @@
     ("https://www.lemonde.fr/img/figure.jpg", False),
     ("https://www.cosmopolitan.fr/inspirations-mode,2511387.asp1", True),
     ("https://www.cosmopolitan.fr/mode,2002.asp2", True),
+    (
+        "https://www.closermag.fr/people/photos.-les-vacances-des-bleus-adil-rami-assailli-par-les-fans-hugo-lloris-se-detend-en-famille-en-croisiere-851976",
+        True,
+    ),
 ]
 
 

diff --git a/test/lru_trie_test.py b/test/lru_trie_test.py
@@ -92,11 +92,11 @@ def test_punycode(self):
         sentinel = object()
 
         trie = LRUTrie()
-        trie.set('http://françai.se', sentinel)
+        trie.set("http://françai.se", sentinel)
 
-        assert trie.match('http://xn--franai-zua.se') is None
+        assert trie.match("http://xn--franai-zua.se") is None
 
         trie = CanonicalizedLRUTrie()
-        trie.set('http://françai.se', sentinel)
+        trie.set("http://françai.se", sentinel)
 
-        assert trie.match('http://xn--franai-zua.se') is sentinel
+        assert trie.match("http://xn--franai-zua.se") is sentinel
diff --git a/ural/could_be_html.py b/ural/could_be_html.py
@@ -32,7 +32,7 @@ def could_be_html(url):
 
     _, ext = splitext(path)
 
-    if not ext:
+    if not ext or len(ext) > 16:
         return True
 
     return ext in HTML_LIKE_EXTENSIONS