feat(rust, python): add strict parameter to decoding expressions

polars/polars-core/src/chunked_array/binary/encoding.rs

@@ -1,16 +1,26 @@
+use std::borrow::Cow;
+
 use base64::engine::general_purpose;
 use base64::Engine as _;
 use hex;
 
 use crate::prelude::*;
 
 impl BinaryChunked {
-    pub fn hex_decode(&self) -> PolarsResult<BinaryChunked> {
-        self.try_apply(|s| {
-            let bytes =
-                hex::decode(s).map_err(|e| PolarsError::ComputeError(e.to_string().into()))?;
-            Ok(bytes.into())
-        })
+    pub fn hex_decode(&self, strict: bool) -> PolarsResult<BinaryChunked> {
+        if strict {
+            self.try_apply(|s| {
+                let bytes = hex::decode(s).map_err(|_e| {
+                    PolarsError::ComputeError(
+                        "Invalid 'hex' encoding found. Try setting 'strict' to false to ignore."
+                            .into(),
+                    )
+                })?;
+                Ok(bytes.into())
+            })
+        } else {
+            Ok(self.apply_on_opt(|opt_s| opt_s.and_then(|s| hex::decode(s).ok().map(Cow::Owned))))
+        }
     }
 
     pub fn hex_encode(&self) -> Series {
@@ -19,13 +29,22 @@ impl BinaryChunked {
             .unwrap()
     }
 
-    pub fn base64_decode(&self) -> PolarsResult<BinaryChunked> {
-        self.try_apply(|s| {
-            let bytes = general_purpose::STANDARD
-                .decode(s)
-                .map_err(|e| PolarsError::ComputeError(e.to_string().into()))?;
-            Ok(bytes.into())
-        })
+    pub fn base64_decode(&self, strict: bool) -> PolarsResult<BinaryChunked> {
+        if strict {
+            self.try_apply(|s| {
+                let bytes = general_purpose::STANDARD.decode(s).map_err(|_e| {
+                    PolarsError::ComputeError(
+                        "Invalid 'base64' encoding found. Try setting 'strict' to false to ignore."
+                            .into(),
+                    )
+                })?;
+                Ok(bytes.into())
+            })
+        } else {
+            Ok(self.apply_on_opt(|opt_s| {
+                opt_s.and_then(|s| general_purpose::STANDARD.decode(s).ok().map(Cow::Owned))
+            }))
+        }
     }
 
     pub fn base64_encode(&self) -> Series {

polars/polars-core/src/chunked_array/strings/encoding.rs

@@ -11,10 +11,10 @@ impl Utf8Chunked {
     }
 
     #[cfg(feature = "binary_encoding")]
-    pub fn hex_decode(&self) -> PolarsResult<BinaryChunked> {
+    pub fn hex_decode(&self, strict: bool) -> PolarsResult<BinaryChunked> {
         self.cast_unchecked(&DataType::Binary)?
             .binary()?
-            .hex_decode()
+            .hex_decode(strict)
     }
 
     #[must_use]
@@ -28,10 +28,10 @@ impl Utf8Chunked {
     }
 
     #[cfg(feature = "binary_encoding")]
-    pub fn base64_decode(&self) -> PolarsResult<BinaryChunked> {
+    pub fn base64_decode(&self, strict: bool) -> PolarsResult<BinaryChunked> {
         self.cast_unchecked(&DataType::Binary)?
             .binary()?
-            .base64_decode()
+            .base64_decode(strict)
     }
 
     #[must_use]

py-polars/polars/internals/expr/binary.py

@@ -56,20 +56,23 @@ def starts_with(self, sub: bytes) -> pli.Expr:
         """
         return pli.wrap_expr(self._pyexpr.binary_starts_with(sub))
 
-    def decode(self, encoding: TransferEncoding) -> pli.Expr:
+    def decode(self, encoding: TransferEncoding, *, strict: bool = True) -> pli.Expr:
         """
         Decode a value using the provided encoding.
 
         Parameters
         ----------
         encoding : {'hex', 'base64'}
             The encoding to use.
+        strict
+            Raise an error if the underlying value cannot be decoded,
+            otherwise mask out with a null value.
 
         """
         if encoding == "hex":
-            return pli.wrap_expr(self._pyexpr.binary_hex_decode())
+            return pli.wrap_expr(self._pyexpr.binary_hex_decode(strict))
         elif encoding == "base64":
-            return pli.wrap_expr(self._pyexpr.binary_base64_decode())
+            return pli.wrap_expr(self._pyexpr.binary_base64_decode(strict))
         else:
             raise ValueError(
                 f"encoding must be one of {{'hex', 'base64'}}, got {encoding}"

py-polars/polars/internals/expr/string.py

@@ -665,20 +665,23 @@ def json_path_match(self, json_path: str) -> pli.Expr:
         """
         return pli.wrap_expr(self._pyexpr.str_json_path_match(json_path))
 
-    def decode(self, encoding: TransferEncoding) -> pli.Expr:
+    def decode(self, encoding: TransferEncoding, *, strict: bool = True) -> pli.Expr:
         """
         Decode a value using the provided encoding.
 
         Parameters
         ----------
         encoding : {'hex', 'base64'}
             The encoding to use.
+        strict
+            Raise an error if the underlying value cannot be decoded,
+            otherwise mask out with a null value.
 
         """
         if encoding == "hex":
-            return pli.wrap_expr(self._pyexpr.str_hex_decode())
+            return pli.wrap_expr(self._pyexpr.str_hex_decode(strict))
         elif encoding == "base64":
-            return pli.wrap_expr(self._pyexpr.str_base64_decode())
+            return pli.wrap_expr(self._pyexpr.str_base64_decode(strict))
         else:
             raise ValueError(
                 f"encoding must be one of {{'hex', 'base64'}}, got {encoding}"

py-polars/polars/internals/series/binary.py

@@ -56,14 +56,17 @@ def starts_with(self, sub: bytes) -> pli.Series:
 
         """
 
-    def decode(self, encoding: TransferEncoding) -> pli.Series:
+    def decode(self, encoding: TransferEncoding, *, strict: bool = True) -> pli.Series:
         """
         Decode a value using the provided encoding.
 
         Parameters
         ----------
         encoding : {'hex', 'base64'}
             The encoding to use.
+        strict
+            Raise an error if the underlying value cannot be decoded,
+            otherwise mask out with a null value.
 
         """
 

py-polars/polars/internals/series/string.py

@@ -266,14 +266,17 @@ def starts_with(self, sub: str) -> pli.Series:
 
         """
 
-    def decode(self, encoding: TransferEncoding) -> pli.Series:
+    def decode(self, encoding: TransferEncoding, *, strict: bool = True) -> pli.Series:
         """
         Decode a value using the provided encoding.
 
         Parameters
         ----------
         encoding : {'hex', 'base64'}
             The encoding to use.
+        strict
+            Raise an error if the underlying value cannot be decoded,
+            otherwise mask out with a null value.
 
         """
 

py-polars/src/lazy/dsl.rs

@@ -739,11 +739,11 @@ impl PyExpr {
             .with_fmt("str.hex_encode")
             .into()
     }
-    pub fn str_hex_decode(&self) -> PyExpr {
+    pub fn str_hex_decode(&self, strict: bool) -> PyExpr {
         self.clone()
             .inner
             .map(
-                move |s| s.utf8()?.hex_decode().map(|s| s.into_series()),
+                move |s| s.utf8()?.hex_decode(strict).map(|s| s.into_series()),
                 GetOutput::same_type(),
             )
             .with_fmt("str.hex_decode")
@@ -760,11 +760,11 @@ impl PyExpr {
             .into()
     }
 
-    pub fn str_base64_decode(&self) -> PyExpr {
+    pub fn str_base64_decode(&self, strict: bool) -> PyExpr {
         self.clone()
             .inner
             .map(
-                move |s| s.utf8()?.base64_decode().map(|s| s.into_series()),
+                move |s| s.utf8()?.base64_decode(strict).map(|s| s.into_series()),
                 GetOutput::same_type(),
             )
             .with_fmt("str.base64_decode")
@@ -781,11 +781,11 @@ impl PyExpr {
             .with_fmt("binary.hex_encode")
             .into()
     }
-    pub fn binary_hex_decode(&self) -> PyExpr {
+    pub fn binary_hex_decode(&self, strict: bool) -> PyExpr {
         self.clone()
             .inner
             .map(
-                move |s| s.binary()?.hex_decode().map(|s| s.into_series()),
+                move |s| s.binary()?.hex_decode(strict).map(|s| s.into_series()),
                 GetOutput::same_type(),
             )
             .with_fmt("binary.hex_decode")
@@ -802,11 +802,11 @@ impl PyExpr {
             .into()
     }
 
-    pub fn binary_base64_decode(&self) -> PyExpr {
+    pub fn binary_base64_decode(&self, strict: bool) -> PyExpr {
         self.clone()
             .inner
             .map(
-                move |s| s.binary()?.base64_decode().map(|s| s.into_series()),
+                move |s| s.binary()?.base64_decode(strict).map(|s| s.into_series()),
                 GetOutput::same_type(),
             )
             .with_fmt("binary.base64_decode")

py-polars/tests/unit/test_utf8.py

@@ -1,3 +1,5 @@
+import pytest
+
 import polars as pl
 
 
@@ -22,3 +24,14 @@ def test_length_vs_nchars() -> None:
         ]
     )
     assert df.rows() == [("café", 5, 4), ("東京", 6, 2)]
+
+
+def test_decode_strict() -> None:
+    df = pl.DataFrame(
+        {"strings": ["0IbQvTc3", "0J%2FQldCf0JA%3D", "0J%2FRgNC%2B0YHRgtC%2B"]}
+    )
+    assert df.select(pl.col("strings").str.decode("base64", strict=False)).to_dict(
+        False
+    ) == {"strings": [b"\xd0\x86\xd0\xbd77", None, None]}
+    with pytest.raises(pl.ComputeError):
+        df.select(pl.col("strings").str.decode("base64", strict=True))