diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index 99ae60859b68c..1a092d930029e 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -239,7 +239,7 @@ Deprecations - Deprecated :attr:`Rolling.is_datetimelike` (:issue:`38963`) - Deprecated :meth:`core.window.ewm.ExponentialMovingWindow.vol` (:issue:`39220`) - Using ``.astype`` to convert between ``datetime64[ns]`` dtype and :class:`DatetimeTZDtype` is deprecated and will raise in a future version, use ``obj.tz_localize`` or ``obj.dt.tz_localize`` instead (:issue:`38622`) -- +- Deprecated casting ``datetime.date`` objects to ``datetime64`` when used as ``fill_value`` in :meth:`DataFrame.unstack`, :meth:`DataFrame.shift`, :meth:`Series.shift`, and :meth:`DataFrame.reindex`, pass ``pd.Timestamp(dateobj)`` instead (:issue:`39767`) .. --------------------------------------------------------------------------- diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index e27c519304e2e..74d750288bdeb 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -5,7 +5,7 @@ from __future__ import annotations from contextlib import suppress -from datetime import datetime, timedelta +from datetime import date, datetime, timedelta from typing import ( TYPE_CHECKING, Any, @@ -549,16 +549,46 @@ def maybe_promote(dtype: np.dtype, fill_value=np.nan): # returns tuple of (dtype, fill_value) if issubclass(dtype.type, np.datetime64): - if isinstance(fill_value, datetime) and fill_value.tzinfo is not None: - # Trying to insert tzaware into tznaive, have to cast to object - dtype = np.dtype(np.object_) - elif is_integer(fill_value) or is_float(fill_value): - dtype = np.dtype(np.object_) - else: + inferred, fv = infer_dtype_from_scalar(fill_value, pandas_dtype=True) + if inferred == dtype: + return dtype, fv + + # TODO(2.0): once this deprecation is enforced, this whole case + # becomes equivalent to: + # dta = DatetimeArray._from_sequence([], dtype="M8[ns]") + # try: + # fv = dta._validate_setitem_value(fill_value) + # return dta.dtype, fv + # except (ValueError, TypeError): + # return np.dtype(object), fill_value + if isinstance(fill_value, date) and not isinstance(fill_value, datetime): + # deprecate casting of date object to match infer_dtype_from_scalar + # and DatetimeArray._validate_setitem_value try: - fill_value = Timestamp(fill_value).to_datetime64() - except (TypeError, ValueError): - dtype = np.dtype(np.object_) + fv = Timestamp(fill_value).to_datetime64() + except OutOfBoundsDatetime: + pass + else: + warnings.warn( + "Using a `date` object for fill_value with `datetime64[ns]` " + "dtype is deprecated. In a future version, this will be cast " + "to object dtype. Pass `fill_value=Timestamp(date_obj)` instead.", + FutureWarning, + stacklevel=7, + ) + return dtype, fv + elif isinstance(fill_value, str): + try: + # explicitly wrap in str to convert np.str_ + fv = Timestamp(str(fill_value)) + except (ValueError, TypeError): + pass + else: + if fv.tz is None: + return dtype, fv.asm8 + + return np.dtype(object), fill_value + elif issubclass(dtype.type, np.timedelta64): if ( is_integer(fill_value) @@ -723,13 +753,13 @@ def infer_dtype_from_scalar(val, pandas_dtype: bool = False) -> Tuple[DtypeObj, if val is NaT or val.tz is None: dtype = np.dtype("M8[ns]") + val = val.to_datetime64() else: if pandas_dtype: dtype = DatetimeTZDtype(unit="ns", tz=val.tz) else: # return datetimetz as object return np.dtype(object), val - val = val.value elif isinstance(val, (np.timedelta64, timedelta)): try: diff --git a/pandas/core/indexes/interval.py b/pandas/core/indexes/interval.py index af353cf3fb5f7..70adcd841a57d 100644 --- a/pandas/core/indexes/interval.py +++ b/pandas/core/indexes/interval.py @@ -533,6 +533,10 @@ def _maybe_convert_i8(self, key): key_dtype, key_i8 = infer_dtype_from_scalar(key, pandas_dtype=True) if lib.is_period(key): key_i8 = key.ordinal + elif isinstance(key_i8, Timestamp): + key_i8 = key_i8.value + elif isinstance(key_i8, (np.datetime64, np.timedelta64)): + key_i8 = key_i8.view("i8") else: # DatetimeIndex/TimedeltaIndex key_dtype, key_i8 = key.dtype, Index(key.asi8) diff --git a/pandas/tests/dtypes/cast/test_infer_dtype.py b/pandas/tests/dtypes/cast/test_infer_dtype.py index a47c5555d3e9f..2b18d110346e4 100644 --- a/pandas/tests/dtypes/cast/test_infer_dtype.py +++ b/pandas/tests/dtypes/cast/test_infer_dtype.py @@ -105,13 +105,11 @@ def test_infer_from_scalar_tz(tz, pandas_dtype): if pandas_dtype: exp_dtype = f"datetime64[ns, {tz}]" - exp_val = dt.value else: exp_dtype = np.object_ - exp_val = dt assert dtype == exp_dtype - assert val == exp_val + assert val == dt @pytest.mark.parametrize( diff --git a/pandas/tests/dtypes/cast/test_promote.py b/pandas/tests/dtypes/cast/test_promote.py index 08303fc601b3e..786944816bcf6 100644 --- a/pandas/tests/dtypes/cast/test_promote.py +++ b/pandas/tests/dtypes/cast/test_promote.py @@ -24,6 +24,7 @@ from pandas.core.dtypes.missing import isna import pandas as pd +import pandas._testing as tm @pytest.fixture( @@ -403,7 +404,13 @@ def test_maybe_promote_any_with_datetime64( expected_dtype = np.dtype(object) exp_val_for_scalar = fill_value - _check_promote(dtype, fill_value, expected_dtype, exp_val_for_scalar) + warn = None + if type(fill_value) is datetime.date and dtype.kind == "M": + # Casting date to dt64 is deprecated + warn = FutureWarning + + with tm.assert_produces_warning(warn, check_stacklevel=False): + _check_promote(dtype, fill_value, expected_dtype, exp_val_for_scalar) @pytest.mark.parametrize( diff --git a/pandas/tests/frame/methods/test_reindex.py b/pandas/tests/frame/methods/test_reindex.py index e4e2656f4337c..e9c17c9f8cc5d 100644 --- a/pandas/tests/frame/methods/test_reindex.py +++ b/pandas/tests/frame/methods/test_reindex.py @@ -25,6 +25,28 @@ class TestDataFrameSelectReindex: # These are specific reindex-based tests; other indexing tests should go in # test_indexing + def test_reindex_date_fill_value(self): + # passing date to dt64 is deprecated + arr = date_range("2016-01-01", periods=6).values.reshape(3, 2) + df = DataFrame(arr, columns=["A", "B"], index=range(3)) + + ts = df.iloc[0, 0] + fv = ts.date() + + with tm.assert_produces_warning(FutureWarning): + res = df.reindex(index=range(4), columns=["A", "B", "C"], fill_value=fv) + + expected = DataFrame( + {"A": df["A"].tolist() + [ts], "B": df["B"].tolist() + [ts], "C": [ts] * 4} + ) + tm.assert_frame_equal(res, expected) + + # same with a datetime-castable str + res = df.reindex( + index=range(4), columns=["A", "B", "C"], fill_value="2016-01-01" + ) + tm.assert_frame_equal(res, expected) + def test_reindex_with_multi_index(self): # https://github.com/pandas-dev/pandas/issues/29896 # tests for reindexing a multi-indexed DataFrame with a new MultiIndex