[PATCH] utils.parse: parse invalid XHTML5 documents (streamlink#4210)

Billy2011 · Nov 27, 2021 · a2406c5 · a2406c5
1 parent 7d3c130
commit a2406c5
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 1 deletion.
diff --git a/src/streamlink/utils/parse.py b/src/streamlink/utils/parse.py
@@ -3,7 +3,7 @@
 
 from lxml.etree import HTML, XML
 
-from streamlink.compat import is_py2, is_py3, parse_qsl
+from streamlink.compat import is_py2, is_py3, parse_qsl, str
 from streamlink.plugin import PluginError
 
 
@@ -48,8 +48,11 @@ def parse_html(
     """Wrapper around lxml.etree.HTML with some extras.
 
     Provides these extra features:
+     - Removes XML declarations of invalid XHTML5 documents
      - Wraps errors in custom exception with a snippet of the data in the message
     """
+    if isinstance(data, str) and data.lstrip().startswith("<?xml"):
+        data = re.sub(r"^\s*<\?xml.+?\?>", "", data)
 
     return _parse(HTML, data, name, exception, schema, *args, **kwargs)
 

diff --git a/tests/utils/test_parse.py b/tests/utils/test_parse.py
@@ -93,6 +93,12 @@ def test_parse_html_encoding(self):
         tree = parse_html(b"""<!DOCTYPE html><html><body>\xC3\xA4</body></html>""")
         self.assertEqual(tree.xpath(".//body/text()"), ["Ã¤"])
 
+    def test_parse_html_xhtml5(self):
+        tree = parse_html("""<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE html><html><body>ä?></body></html>""")
+        self.assertEqual(tree.xpath(".//body/text()"), ["ä?>"])
+        tree = parse_html(b"""<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE html><html><body>\xC3\xA4?></body></html>""")
+        self.assertEqual(tree.xpath(".//body/text()"), ["ä?>"])
+
     def test_parse_qsd(self):
         self.assertEqual(
             {"test": "1", "foo": "bar"},