Make unicode decode error clearer

singlestore-labs · Jan 31, 2025 · 298a773 · 298a773
1 parent 52db5af
commit 298a773
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 2 deletions.
diff --git a/accel.c b/accel.c
@@ -1533,6 +1533,15 @@ static PyObject *read_row_from_packet(
                     if (!py_str) goto error;
                 } else {
                     py_str = PyUnicode_Decode(out, out_l, py_state->encodings[i], py_state->encoding_errors);
+                    if (PyErr_Occurred()) {
+                        PyErr_Clear();
+                        PyErr_Format(
+                            PyExc_UnicodeDecodeError,
+                            "failed to decode string value in column '%S' using encoding '%s'; "
+                            "use the 'encoding_errors' option on the connection to specify how to handle this error",
+                            py_state->py_names[i], py_state->encodings[i]
+                        );
+                    }
                     if (!py_str) goto error;
                 }
                 if (py_state->py_converters[i] == Py_None) {
@@ -1740,6 +1749,15 @@ static PyObject *read_row_from_packet(
                     }
 
                     py_item = PyUnicode_Decode(out, out_l, py_state->encodings[i], py_state->encoding_errors);
+                    if (PyErr_Occurred()) {
+                        PyErr_Clear();
+                        PyErr_Format(
+                            PyExc_UnicodeDecodeError,
+                            "failed to decode string value in column '%S' using encoding '%s'; "
+                            "use the 'encoding_errors' option on the connection to specify how to handle this error",
+                            py_state->py_names[i], py_state->encodings[i]
+                        );
+                    }
                     if (!py_item) goto error;
 
                     // Parse JSON string.

diff --git a/singlestoredb/mysql/connection.py b/singlestoredb/mysql/connection.py
@@ -1841,7 +1841,7 @@ def _read_rowdata_packet(self):
 
     def _read_row_from_packet(self, packet):
         row = []
-        for encoding, converter in self.converters:
+        for i, (encoding, converter) in enumerate(self.converters):
             try:
                 data = packet.read_length_coded_string()
             except IndexError:
@@ -1850,7 +1850,15 @@ def _read_row_from_packet(self, packet):
                 break
             if data is not None:
                 if encoding is not None:
-                    data = data.decode(encoding, errors=self.encoding_errors)
+                    try:
+                        data = data.decode(encoding, errors=self.encoding_errors)
+                    except UnicodeDecodeError:
+                        raise UnicodeDecodeError(
+                            'failed to decode string value in column '
+                            f"'{self.fields[i].name}' using encoding '{encoding}'; " +
+                            "use the 'encoding_errors' option on the connection " +
+                            'to specify how to handle this error',
+                        )
                 if DEBUG:
                     print('DEBUG: DATA = ', data)
                 if converter is not None: