Fix: Select statement with foreign language fails (dbcli#278)

mssqlcli/mssqlcliclient.py

@@ -11,6 +11,7 @@
 from mssqlcli.packages import special
 from mssqlcli.packages.parseutils.meta import ForeignKey
 from time import sleep
+from utility import encode
 
 logger = logging.getLogger(u'mssqlcli.mssqlcliclient')
 time_wait_if_no_response = 0.05
@@ -264,9 +265,9 @@ def _generate_query_results_to_tuples(self, query, message, column_info=None, re
         if is_error:
             return (), None, message, query, is_error
 
-        columns = [col.column_name for col in column_info] if column_info else None
+        columns = [encode(col.column_name) for col in column_info] if column_info else None
 
-        rows = ([[cell.display_value for cell in result_row.result_cells]
+        rows = ([[encode(cell.display_value) for cell in result_row.result_cells]
                  for result_row in result_rows]) if result_rows else ()
 
         return rows, columns, message, query, is_error

mssqlcli/mssqlcompleter.py

@@ -17,6 +17,7 @@
 from .packages.parseutils.tables import TableReference
 from .packages.mssqlliterals.main import get_literals
 from .packages.prioritization import PrevalenceCounter
+from utility import decode
 
 _logger = logging.getLogger('mssqlcli.mssqlcompleter')
 
@@ -407,9 +408,9 @@ def _match(item):
                     prio2, lexical_priority
                 )
 
-                item = self.ensure_unicode(item)
-                display_meta = self.ensure_unicode(display_meta)
-                display = self.ensure_unicode(display)
+                item = decode(item)
+                display_meta = decode(display_meta)
+                display = decode(display)
 
                 matches.append(
                     Match(
@@ -423,14 +424,6 @@ def _match(item):
                     )
                 )
         return matches
-
-    # In Python 3, all strings are sequences of Unicode characters.
-    # There is a bytes type that holds raw bytes.
-    # In Python 2, a string may be of type str or of type unicode.
-    def ensure_unicode(self, word):
-        if (sys.version_info.major < 3 and isinstance(word, str)):
-            return word.decode('utf-8')
-        return word
 
     def case(self, word):
         return self.casing.get(word, word)

tests/test_localization.py

@@ -2,17 +2,13 @@
 import unittest
 import sys
 import mssqlcli.localized_strings as localized
+from utility import decode
 
 
 class LocalizationTests(unittest.TestCase):
 
     def test_product(self):
-        original = self.unicode(localized.goodbye())
+        original = decode(localized.goodbye())
         localized.translation(languages=['ko']).install()
-        translated = self.unicode(localized.goodbye())
+        translated = decode(localized.goodbye())
         assert original != translated
-
-    def unicode(self, s):
-        if (sys.version_info.major < 3 and isinstance(s, str)):
-            return s.decode('utf-8')
-        return s

tests/test_mssqlcliclient.py

@@ -3,6 +3,8 @@
 import io
 import unittest
 import mssqlcli.sqltoolsclient as sqltoolsclient
+from utility import random_str
+import socket
 
 from mssqlcli.jsonrpc.jsonrpcclient import JsonRpcWriter
 from mssqltestutils import (
@@ -202,6 +204,32 @@ def test_stored_proc_multiple_result_sets(self):
         finally:
             shutdown(client)
 
+
+    def test_select_result_unicode_encoded(self):
+        """
+            Verify the column names and string values in rows returned by select statement are properly encoded as unicode.
+        """
+        local_machine_name = socket.gethostname().replace('-','_').replace('.','_')
+        table_name = '#mssqlcli_{0}_{1}'.format(local_machine_name, random_str())
+        try:
+            client = create_mssql_cli_client()
+            setup_query = u"CREATE TABLE {0} (컬럼1 nvarchar(MAX), 컬럼2 int);"\
+                u"INSERT INTO {0} VALUES (N'테스트1', 1);"\
+                u"INSERT INTO {0} VALUES (N'테스트2', 2);"\
+                .format(table_name)
+            for rows, columns, status, statement, is_error in client.execute_query(setup_query):
+                assert is_error == False
+
+            select_query = u"SELECT * FROM {0};".format(table_name)
+            for rows, columns, status, statement, is_error in client.execute_query(select_query):
+                assert is_error == False
+                assert columns[0] == '\xec\xbb\xac\xeb\x9f\xbc1'
+                assert columns[1] == '\xec\xbb\xac\xeb\x9f\xbc2'
+                assert rows[0][0] == '\xed\x85\x8c\xec\x8a\xa4\xed\x8a\xb81'
+                assert rows[1][0] == '\xed\x85\x8c\xec\x8a\xa4\xed\x8a\xb82'
+        finally:
+            shutdown(client)
+
 
 if __name__ == u'__main__':
     unittest.main()

utility.py

@@ -4,6 +4,8 @@
 import platform
 import shutil
 import sys
+import string
+import random
 
 ROOT_DIR = os.path.abspath(os.path.join(os.path.abspath(__file__), '..'))
 
@@ -78,3 +80,22 @@ def copy_current_platform_mssqltoolsservice():
         mssqltoolsservice.copy_sqltoolsservice(current_platform)
     else:
         print("This platform: {} does not support mssqltoolsservice.".format(platform.system()))
+
+
+def encode(s):
+    if (sys.version_info.major < 3):
+        return s.encode('utf-8')
+    return s
+
+
+# In Python 3, all strings are sequences of Unicode characters.
+# There is a bytes type that holds raw bytes.
+# In Python 2, a string may be of type str or of type unicode.
+def decode(s):
+    if (sys.version_info.major < 3):
+        return s.decode('utf-8')
+    return s
+
+
+def random_str(size=12, chars=string.ascii_uppercase + string.digits):
+    return ''.join(random.choice(chars) for x in range(size))