BUG: move utf8 recoding elsewhere, add test case to deal with #2298

pandas-dev · Dec 6, 2012 · 4e49d29 · 4e49d29
1 parent f985aa1
commit 4e49d29
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/pandas/core/common.py b/pandas/core/common.py
@@ -1096,8 +1096,6 @@ def _get_handle(path, mode, encoding=None):
             f = open(path, mode, errors='replace')
     else:
         f = open(path, mode)
-        if encoding is not None and 'r' in mode:
-            f = UTF8Recoder(f, encoding)
     return f
 
 if py3compat.PY3:  # pragma: no cover
@@ -1118,7 +1116,7 @@ class UnicodeReader:
         """
 
         def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
-            # f = UTF8Recoder(f, encoding)
+            f = UTF8Recoder(f, encoding)
             self.reader = csv.reader(f, dialect=dialect, **kwds)
 
         def next(self):

diff --git a/pandas/io/tests/test_parsers.py b/pandas/io/tests/test_parsers.py
@@ -1640,6 +1640,10 @@ def test_utf16_example(self):
         result = self.read_table(path, encoding='utf-16')
         self.assertEquals(len(result), 50)
 
+        buf = BytesIO(open(path, 'rb').read())
+        result = self.read_table(buf, encoding='utf-16')
+        self.assertEquals(len(result), 50)
+
 class TestCParserHighMemory(ParserTests, unittest.TestCase):
 
     def read_csv(self, *args, **kwds):