fix an issue where UTF-8 with a BOM would not be detected as UTF-8-SI…

…G ( fix #28 )
PyYoshi · Apr 14, 2017 · f729e11 · f729e11
1 parent 1f1e8c1
commit f729e11
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 1 deletion.
diff --git a/src/ext/uchardet b/src/ext/uchardet
diff --git a/src/tests/test.py b/src/tests/test.py
@@ -102,3 +102,15 @@ def test_decode(self):
                 except LookupError as e:
                     print("LookupError: { file=%s, encoding=%s }" % (testfile, detected_encoding["encoding"]))
                     raise e
+
+    def test_utf8_with_bom(self):
+        sample = b'\xEF\xBB\xBF'
+        detected_encoding = cchardet.detect(sample)
+        eq_(
+            "utf-8-sig",
+            detected_encoding['encoding'].lower(),
+            'Expected %s, but got %s' % (
+                "utf-8-sig",
+                detected_encoding['encoding'].lower()
+            )
+        )