Skip to content

Commit

Permalink
fix(conversion): convert decimals to the exact precision and scale re…
Browse files Browse the repository at this point in the history
…quested by the input type
  • Loading branch information
cpcloud authored and kszucs committed Feb 12, 2024
1 parent 19b71fb commit 8c1e6f4
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 17 deletions.
9 changes: 7 additions & 2 deletions ibis/common/numeric.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,12 @@
from decimal import Context, Decimal, InvalidOperation


def normalize_decimal(value, precision: int | None = None, scale: int | None = None):
def normalize_decimal(
value,
precision: int | None = None,
scale: int | None = None,
strict: bool = True,
):
context = Context(prec=38 if precision is None else precision)

try:
Expand All @@ -25,7 +30,7 @@ def normalize_decimal(value, precision: int | None = None, scale: int | None = N
)

if scale is not None:
if exponent < -scale:
if strict and exponent < -scale:
raise TypeError(
f"Normalizing {value} with scale {exponent} to scale -{scale} "
"would loose precision"
Expand Down
24 changes: 9 additions & 15 deletions ibis/formats/pandas.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

import contextlib
import datetime
import decimal
import warnings
from functools import partial
from importlib.util import find_spec as _find_spec

import numpy as np
Expand All @@ -13,6 +13,7 @@

import ibis.expr.datatypes as dt
import ibis.expr.schema as sch
from ibis.common.numeric import normalize_decimal
from ibis.common.temporal import normalize_timezone
from ibis.formats import DataMapper, SchemaMapper, TableProxy
from ibis.formats.numpy import NumpyType
Expand Down Expand Up @@ -255,20 +256,13 @@ def convert_String(cls, s, dtype, pandas_type):

@classmethod
def convert_Decimal(cls, s, dtype, pandas_type):
context = decimal.Context(prec=dtype.precision)

if dtype.scale is None:
normalize = context.create_decimal
else:
exponent = decimal.Decimal(10) ** -dtype.scale

def normalize(x, exponent=exponent):
try:
return context.create_decimal(x).quantize(exponent)
except decimal.InvalidOperation:
return x

return s.map(normalize, na_action="ignore").astype(pandas_type)
func = partial(
normalize_decimal,
precision=dtype.precision,
scale=dtype.scale,
strict=False,
)
return s.map(func, na_action="ignore")

@classmethod
def convert_UUID(cls, s, dtype, pandas_type):
Expand Down

0 comments on commit 8c1e6f4

Please sign in to comment.