From 67b930474234543ba903555da47063629a435da6 Mon Sep 17 00:00:00 2001 From: Brock Date: Sat, 5 Sep 2020 13:05:10 -0700 Subject: [PATCH 1/4] BUG: allowing tznaive datetimes when indexing tzaware datetimeindex --- pandas/core/indexes/datetimes.py | 10 ++++ .../tests/indexes/datetimes/test_indexing.py | 50 +++++++++++++------ pandas/tests/series/indexing/test_datetime.py | 30 ++++++----- pandas/tests/series/methods/test_truncate.py | 8 ++- 4 files changed, 68 insertions(+), 30 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 6dcb9250812d0..de596b44394bf 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -596,6 +596,10 @@ def get_loc(self, key, method=None, tolerance=None): if isinstance(key, self._data._recognized_scalars): # needed to localize naive datetimes + try: + self._data._assert_tzawareness_compat(key) + except TypeError as err: + raise KeyError(key) from err key = self._maybe_cast_for_get_loc(key) elif isinstance(key, str): @@ -654,6 +658,10 @@ def _maybe_cast_slice_bound(self, label, side: str, kind): ------- label : object + Raises + ------ + TypeError : indexing timezone-aware DatetimeIndex with tz-naive datetime + Notes ----- Value of `side` parameter should be validated in caller. @@ -677,6 +685,8 @@ def _maybe_cast_slice_bound(self, label, side: str, kind): if self._is_strictly_monotonic_decreasing and len(self) > 1: return upper if side == "left" else lower return lower if side == "left" else upper + elif isinstance(label, (self._data._recognized_scalars, date)): + self._data._assert_tzawareness_compat(label) return self._maybe_cast_for_get_loc(label) def _get_string_slice(self, key: str, use_lhs: bool = True, use_rhs: bool = True): diff --git a/pandas/tests/indexes/datetimes/test_indexing.py b/pandas/tests/indexes/datetimes/test_indexing.py index 539d9cb8f06a7..a8c64784df5a4 100644 --- a/pandas/tests/indexes/datetimes/test_indexing.py +++ b/pandas/tests/indexes/datetimes/test_indexing.py @@ -675,11 +675,17 @@ def test_get_slice_bounds_datetime_within( self, box, kind, side, expected, tz_aware_fixture ): # GH 35690 - index = bdate_range("2000-01-03", "2000-02-11").tz_localize(tz_aware_fixture) - result = index.get_slice_bound( - box(year=2000, month=1, day=7), kind=kind, side=side - ) - assert result == expected + tz = tz_aware_fixture + index = bdate_range("2000-01-03", "2000-02-11").tz_localize(tz) + key = box(year=2000, month=1, day=7) + if tz is None: + result = index.get_slice_bound(key, kind=kind, side=side) + assert result == expected + else: + # We require tzawareness-compat in indexing + msg = "Cannot compare tz-naive and tz-aware datetime-like objects" + with pytest.raises(TypeError, match=msg): + index.get_slice_bound(key, kind=kind, side=side) @pytest.mark.parametrize("box", [date, datetime, Timestamp]) @pytest.mark.parametrize("kind", ["getitem", "loc", None]) @@ -689,19 +695,31 @@ def test_get_slice_bounds_datetime_outside( self, box, kind, side, year, expected, tz_aware_fixture ): # GH 35690 - index = bdate_range("2000-01-03", "2000-02-11").tz_localize(tz_aware_fixture) - result = index.get_slice_bound( - box(year=year, month=1, day=7), kind=kind, side=side - ) - assert result == expected + tz = tz_aware_fixture + index = bdate_range("2000-01-03", "2000-02-11").tz_localize(tz) + key = box(year=year, month=1, day=7) + if tz is None: + result = index.get_slice_bound(key, kind=kind, side=side) + assert result == expected + else: + # We require tzawareness compat in indexing + msg = "Cannot compare tz-naive and tz-aware datetime-like objects" + with pytest.raises(TypeError, match=msg): + index.get_slice_bound(key, kind=kind, side=side) @pytest.mark.parametrize("box", [date, datetime, Timestamp]) @pytest.mark.parametrize("kind", ["getitem", "loc", None]) def test_slice_datetime_locs(self, box, kind, tz_aware_fixture): # GH 34077 - index = DatetimeIndex(["2010-01-01", "2010-01-03"]).tz_localize( - tz_aware_fixture - ) - result = index.slice_locs(box(2010, 1, 1), box(2010, 1, 2)) - expected = (0, 1) - assert result == expected + tz = tz_aware_fixture + index = DatetimeIndex(["2010-01-01", "2010-01-03"]).tz_localize(tz) + key = box(2010, 1, 1) + if tz is None: + result = index.slice_locs(key, box(2010, 1, 2)) + expected = (0, 1) + assert result == expected + else: + # We require tzawareness-compat in indexing + msg = "Cannot compare tz-naive and tz-aware datetime-like objects" + with pytest.raises(TypeError, match=msg): + index.slice_locs(key, box(2010, 1, 2)) diff --git a/pandas/tests/series/indexing/test_datetime.py b/pandas/tests/series/indexing/test_datetime.py index 088f8681feb99..34d61c251a388 100644 --- a/pandas/tests/series/indexing/test_datetime.py +++ b/pandas/tests/series/indexing/test_datetime.py @@ -11,6 +11,7 @@ from pandas import DataFrame, DatetimeIndex, NaT, Series, Timestamp, date_range import pandas._testing as tm + """ Also test support for datetime64[ns] in Series / DataFrame """ @@ -238,24 +239,27 @@ def test_getitem_setitem_datetimeindex(): expected = ts[4:8] tm.assert_series_equal(result, expected) - # repeat all the above with naive datetimes - result = ts[datetime(1990, 1, 1, 4)] - expected = ts[4] - assert result == expected + # But we do not give datetimes a pass on tzawareness compat + # TODO: do the same with Timestamps and dt64 + msg = "Cannot compare tz-naive and tz-aware datetime-like objects" + naive = datetime(1990, 1, 1, 4) + with pytest.raises(KeyError, match=re.escape(repr(naive))): + ts[naive] result = ts.copy() - result[datetime(1990, 1, 1, 4)] = 0 - result[datetime(1990, 1, 1, 4)] = ts[4] - tm.assert_series_equal(result, ts) + with pytest.raises(TypeError, match=msg): + result[datetime(1990, 1, 1, 4)] = 0 + with pytest.raises(TypeError, match=msg): + result[datetime(1990, 1, 1, 4)] = ts[4] - result = ts[datetime(1990, 1, 1, 4) : datetime(1990, 1, 1, 7)] - expected = ts[4:8] - tm.assert_series_equal(result, expected) + with pytest.raises(TypeError, match=msg): + ts[datetime(1990, 1, 1, 4) : datetime(1990, 1, 1, 7)] result = ts.copy() - result[datetime(1990, 1, 1, 4) : datetime(1990, 1, 1, 7)] = 0 - result[datetime(1990, 1, 1, 4) : datetime(1990, 1, 1, 7)] = ts[4:8] - tm.assert_series_equal(result, ts) + with pytest.raises(TypeError, match=msg): + result[datetime(1990, 1, 1, 4) : datetime(1990, 1, 1, 7)] = 0 + with pytest.raises(TypeError, match=msg): + result[datetime(1990, 1, 1, 4) : datetime(1990, 1, 1, 7)] = ts[4:8] lb = datetime(1990, 1, 1, 4) rb = datetime(1990, 1, 1, 7) diff --git a/pandas/tests/series/methods/test_truncate.py b/pandas/tests/series/methods/test_truncate.py index 45592f8d99b93..db6f537db381c 100644 --- a/pandas/tests/series/methods/test_truncate.py +++ b/pandas/tests/series/methods/test_truncate.py @@ -101,7 +101,13 @@ def test_truncate_datetimeindex_tz(self): # GH 9243 idx = date_range("4/1/2005", "4/30/2005", freq="D", tz="US/Pacific") s = Series(range(len(idx)), index=idx) - result = s.truncate(datetime(2005, 4, 2), datetime(2005, 4, 4)) + msg = "Cannot compare tz-naive and tz-aware datetime-like objects" + with pytest.raises(TypeError, match=msg): + s.truncate(datetime(2005, 4, 2), datetime(2005, 4, 4)) + + lb = idx[1] + ub = idx[3] + result = s.truncate(lb.to_pydatetime(), ub.to_pydatetime()) expected = Series([1, 2, 3], index=idx[1:4]) tm.assert_series_equal(result, expected) From a4eda0e745381d022d95b7f5b626b1c727976b40 Mon Sep 17 00:00:00 2001 From: Brock Date: Sat, 5 Sep 2020 15:53:11 -0700 Subject: [PATCH 2/4] isort fixup --- pandas/tests/series/indexing/test_datetime.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pandas/tests/series/indexing/test_datetime.py b/pandas/tests/series/indexing/test_datetime.py index 34d61c251a388..75f3d4b85a6a4 100644 --- a/pandas/tests/series/indexing/test_datetime.py +++ b/pandas/tests/series/indexing/test_datetime.py @@ -1,3 +1,6 @@ +""" +Also test support for datetime64[ns] in Series / DataFrame +""" from datetime import datetime, timedelta import re @@ -12,11 +15,6 @@ import pandas._testing as tm -""" -Also test support for datetime64[ns] in Series / DataFrame -""" - - def test_fancy_getitem(): dti = date_range( freq="WOM-1FRI", start=datetime(2005, 1, 1), end=datetime(2010, 1, 1) From 4789e635900981d8c805b588b68c047610fb963c Mon Sep 17 00:00:00 2001 From: Brock Date: Thu, 1 Oct 2020 10:11:40 -0700 Subject: [PATCH 3/4] deprecate wrong behavior --- pandas/core/indexes/datetimes.py | 33 +++++++++++----- .../tests/indexes/datetimes/test_indexing.py | 38 ++++++++----------- pandas/tests/series/indexing/test_datetime.py | 28 ++++++++++---- pandas/tests/series/methods/test_truncate.py | 4 +- 4 files changed, 62 insertions(+), 41 deletions(-) diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py index 5a32d75794656..e497f25821e4a 100644 --- a/pandas/core/indexes/datetimes.py +++ b/pandas/core/indexes/datetimes.py @@ -600,6 +600,28 @@ def _validate_partial_date_slice(self, reso: Resolution): # _parsed_string_to_bounds allows it. raise KeyError + def _deprecate_mismatched_indexing(self, key): + # GH#36148 + # we get here with isinstance(key, self._data._recognized_scalars) + try: + self._data._assert_tzawareness_compat(key) + except TypeError: + if self.tz is None: + msg = ( + "Indexing a timezone-naive DatetimeIndex with a " + "timezone-aware datetime is deprecated and will " + "raise KeyError in a future version. " + "Use a timezone-naive object instead." + ) + else: + msg = ( + "Indexing a timezone-aware DatetimeIndex with a " + "timezone-naive datetime is deprecated and will " + "raise KeyError in a future version. " + "Use a timezone-aware object instead." + ) + warnings.warn(msg, FutureWarning, stacklevel=5) + def get_loc(self, key, method=None, tolerance=None): """ Get integer location for requested label @@ -617,10 +639,7 @@ def get_loc(self, key, method=None, tolerance=None): if isinstance(key, self._data._recognized_scalars): # needed to localize naive datetimes - try: - self._data._assert_tzawareness_compat(key) - except TypeError as err: - raise KeyError(key) from err + self._deprecate_mismatched_indexing(key) key = self._maybe_cast_for_get_loc(key) elif isinstance(key, str): @@ -679,10 +698,6 @@ def _maybe_cast_slice_bound(self, label, side: str, kind): ------- label : object - Raises - ------ - TypeError : indexing timezone-aware DatetimeIndex with tz-naive datetime - Notes ----- Value of `side` parameter should be validated in caller. @@ -707,7 +722,7 @@ def _maybe_cast_slice_bound(self, label, side: str, kind): return upper if side == "left" else lower return lower if side == "left" else upper elif isinstance(label, (self._data._recognized_scalars, date)): - self._data._assert_tzawareness_compat(label) + self._deprecate_mismatched_indexing(label) return self._maybe_cast_for_get_loc(label) def _get_string_slice(self, key: str, use_lhs: bool = True, use_rhs: bool = True): diff --git a/pandas/tests/indexes/datetimes/test_indexing.py b/pandas/tests/indexes/datetimes/test_indexing.py index a8c64784df5a4..cb12365f38605 100644 --- a/pandas/tests/indexes/datetimes/test_indexing.py +++ b/pandas/tests/indexes/datetimes/test_indexing.py @@ -678,14 +678,12 @@ def test_get_slice_bounds_datetime_within( tz = tz_aware_fixture index = bdate_range("2000-01-03", "2000-02-11").tz_localize(tz) key = box(year=2000, month=1, day=7) - if tz is None: + + warn = None if tz is None else FutureWarning + with tm.assert_produces_warning(warn, check_stacklevel=False): + # GH#36148 will require tzawareness-compat result = index.get_slice_bound(key, kind=kind, side=side) - assert result == expected - else: - # We require tzawareness-compat in indexing - msg = "Cannot compare tz-naive and tz-aware datetime-like objects" - with pytest.raises(TypeError, match=msg): - index.get_slice_bound(key, kind=kind, side=side) + assert result == expected @pytest.mark.parametrize("box", [date, datetime, Timestamp]) @pytest.mark.parametrize("kind", ["getitem", "loc", None]) @@ -698,14 +696,12 @@ def test_get_slice_bounds_datetime_outside( tz = tz_aware_fixture index = bdate_range("2000-01-03", "2000-02-11").tz_localize(tz) key = box(year=year, month=1, day=7) - if tz is None: + + warn = None if tz is None else FutureWarning + with tm.assert_produces_warning(warn, check_stacklevel=False): + # GH#36148 will require tzawareness-compat result = index.get_slice_bound(key, kind=kind, side=side) - assert result == expected - else: - # We require tzawareness compat in indexing - msg = "Cannot compare tz-naive and tz-aware datetime-like objects" - with pytest.raises(TypeError, match=msg): - index.get_slice_bound(key, kind=kind, side=side) + assert result == expected @pytest.mark.parametrize("box", [date, datetime, Timestamp]) @pytest.mark.parametrize("kind", ["getitem", "loc", None]) @@ -714,12 +710,10 @@ def test_slice_datetime_locs(self, box, kind, tz_aware_fixture): tz = tz_aware_fixture index = DatetimeIndex(["2010-01-01", "2010-01-03"]).tz_localize(tz) key = box(2010, 1, 1) - if tz is None: + + warn = None if tz is None else FutureWarning + with tm.assert_produces_warning(warn, check_stacklevel=False): + # GH#36148 will require tzawareness-compat result = index.slice_locs(key, box(2010, 1, 2)) - expected = (0, 1) - assert result == expected - else: - # We require tzawareness-compat in indexing - msg = "Cannot compare tz-naive and tz-aware datetime-like objects" - with pytest.raises(TypeError, match=msg): - index.slice_locs(key, box(2010, 1, 2)) + expected = (0, 1) + assert result == expected diff --git a/pandas/tests/series/indexing/test_datetime.py b/pandas/tests/series/indexing/test_datetime.py index 30d8f60b92191..7c84f26467f50 100644 --- a/pandas/tests/series/indexing/test_datetime.py +++ b/pandas/tests/series/indexing/test_datetime.py @@ -241,23 +241,35 @@ def test_getitem_setitem_datetimeindex(): # TODO: do the same with Timestamps and dt64 msg = "Cannot compare tz-naive and tz-aware datetime-like objects" naive = datetime(1990, 1, 1, 4) - with pytest.raises(KeyError, match=re.escape(repr(naive))): - ts[naive] + with tm.assert_produces_warning(FutureWarning): + # GH#36148 will require tzawareness compat + result = ts[naive] + expected = ts[4] + assert result == expected result = ts.copy() - with pytest.raises(TypeError, match=msg): + with tm.assert_produces_warning(FutureWarning, check_stacklevel=False): + # GH#36148 will require tzawareness compat result[datetime(1990, 1, 1, 4)] = 0 - with pytest.raises(TypeError, match=msg): + with tm.assert_produces_warning(FutureWarning, check_stacklevel=False): + # GH#36148 will require tzawareness compat result[datetime(1990, 1, 1, 4)] = ts[4] + tm.assert_series_equal(result, ts) - with pytest.raises(TypeError, match=msg): - ts[datetime(1990, 1, 1, 4) : datetime(1990, 1, 1, 7)] + with tm.assert_produces_warning(FutureWarning, check_stacklevel=False): + # GH#36148 will require tzawareness compat + result = ts[datetime(1990, 1, 1, 4) : datetime(1990, 1, 1, 7)] + expected = ts[4:8] + tm.assert_series_equal(result, expected) result = ts.copy() - with pytest.raises(TypeError, match=msg): + with tm.assert_produces_warning(FutureWarning, check_stacklevel=False): + # GH#36148 will require tzawareness compat result[datetime(1990, 1, 1, 4) : datetime(1990, 1, 1, 7)] = 0 - with pytest.raises(TypeError, match=msg): + with tm.assert_produces_warning(FutureWarning, check_stacklevel=False): + # GH#36148 will require tzawareness compat result[datetime(1990, 1, 1, 4) : datetime(1990, 1, 1, 7)] = ts[4:8] + tm.assert_series_equal(result, ts) lb = datetime(1990, 1, 1, 4) rb = datetime(1990, 1, 1, 7) diff --git a/pandas/tests/series/methods/test_truncate.py b/pandas/tests/series/methods/test_truncate.py index db6f537db381c..858b1d6b4df8c 100644 --- a/pandas/tests/series/methods/test_truncate.py +++ b/pandas/tests/series/methods/test_truncate.py @@ -101,8 +101,8 @@ def test_truncate_datetimeindex_tz(self): # GH 9243 idx = date_range("4/1/2005", "4/30/2005", freq="D", tz="US/Pacific") s = Series(range(len(idx)), index=idx) - msg = "Cannot compare tz-naive and tz-aware datetime-like objects" - with pytest.raises(TypeError, match=msg): + with tm.assert_produces_warning(FutureWarning, check_stacklevel=False): + # GH#36148 in the future will require tzawareness compat s.truncate(datetime(2005, 4, 2), datetime(2005, 4, 4)) lb = idx[1] From f48a85443125692e96abc028e08cd9f62101c8aa Mon Sep 17 00:00:00 2001 From: Brock Date: Thu, 1 Oct 2020 10:13:13 -0700 Subject: [PATCH 4/4] whatsnew --- doc/source/whatsnew/v1.2.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index e810fc0239b40..b3561d6133d2b 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -265,6 +265,7 @@ Deprecations - Deprecated indexing :class:`DataFrame` rows with datetime-like strings ``df[string]``, use ``df.loc[string]`` instead (:issue:`36179`) - Deprecated casting an object-dtype index of ``datetime`` objects to :class:`DatetimeIndex` in the :class:`Series` constructor (:issue:`23598`) - Deprecated :meth:`Index.is_all_dates` (:issue:`27744`) +- Deprecated slice-indexing on timezone-aware :class:`DatetimeIndex` with naive ``datetime`` objects, to match scalar indexing behavior (:issue:`36148`) .. ---------------------------------------------------------------------------