From 3d160f4946ce2be089c2110b090b3fe218bd3444 Mon Sep 17 00:00:00 2001 From: Brock Date: Sun, 27 Dec 2020 09:10:33 -0800 Subject: [PATCH 1/7] BUG: inconsistency between frame.any/all with dt64 vs dt64tz --- pandas/core/arrays/datetimelike.py | 11 +++++++++++ pandas/tests/frame/test_reductions.py | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index b2050bf54cad6..98bff7f326f66 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1640,6 +1640,17 @@ def floor(self, freq, ambiguous="raise", nonexistent="raise"): def ceil(self, freq, ambiguous="raise", nonexistent="raise"): return self._round(freq, RoundTo.PLUS_INFTY, ambiguous, nonexistent) + # -------------------------------------------------------------- + # Reductions + + def any(self, *, axis: Optional[int] = None, skipna: bool = True): + # GH#33478 + return nanops.nanany(self._ndarray, axis=axis, skipna=skipna, mask=self.isna()) + + def all(self, *, axis: Optional[int] = None, skipna: bool = True): + # GH#33478 + return nanops.nanall(self._ndarray, axis=axis, skipna=skipna, mask=self.isna()) + # -------------------------------------------------------------- # Frequency Methods diff --git a/pandas/tests/frame/test_reductions.py b/pandas/tests/frame/test_reductions.py index d33d91f2cefca..d843d4b0e9504 100644 --- a/pandas/tests/frame/test_reductions.py +++ b/pandas/tests/frame/test_reductions.py @@ -1091,9 +1091,13 @@ def test_any_all_bool_only(self): (np.all, {"A": Series([0, 1], dtype=int)}, False), (np.any, {"A": Series([0, 1], dtype=int)}, True), pytest.param(np.all, {"A": Series([0, 1], dtype="M8[ns]")}, False), + pytest.param(np.all, {"A": Series([0, 1], dtype="M8[ns, UTC]")}, False), pytest.param(np.any, {"A": Series([0, 1], dtype="M8[ns]")}, True), + pytest.param(np.any, {"A": Series([0, 1], dtype="M8[ns, UTC]")}, True), pytest.param(np.all, {"A": Series([1, 2], dtype="M8[ns]")}, True), + pytest.param(np.all, {"A": Series([1, 2], dtype="M8[ns, UTC]")}, True), pytest.param(np.any, {"A": Series([1, 2], dtype="M8[ns]")}, True), + pytest.param(np.any, {"A": Series([1, 2], dtype="M8[ns, UTC]")}, True), pytest.param(np.all, {"A": Series([0, 1], dtype="m8[ns]")}, False), pytest.param(np.any, {"A": Series([0, 1], dtype="m8[ns]")}, True), pytest.param(np.all, {"A": Series([1, 2], dtype="m8[ns]")}, True), From 17b981fa008154c58021c2ed74d253efa5010562 Mon Sep 17 00:00:00 2001 From: Brock Date: Sun, 27 Dec 2020 09:12:31 -0800 Subject: [PATCH 2/7] whatsnew --- doc/source/whatsnew/v1.3.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index c7573ee860744..ffd06a1b40fad 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -185,6 +185,7 @@ Datetimelike - Bug in :class:`DataFrame` and :class:`Series` constructors sometimes dropping nanoseconds from :class:`Timestamp` (resp. :class:`Timedelta`) ``data``, with ``dtype=datetime64[ns]`` (resp. ``timedelta64[ns]``) (:issue:`38032`) - Bug in :meth:`DataFrame.first` and :meth:`Series.first` returning two months for offset one month when first day is last calendar day (:issue:`29623`) - Bug in constructing a :class:`DataFrame` or :class:`Series` with mismatched ``datetime64`` data and ``timedelta64`` dtype, or vice-versa, failing to raise ``TypeError`` (:issue:`38575`) +- Bug in :meth:`DataFrame.any` and :meth:`DataFrame.all` behaving differently for tznaive vs tzaware ``datetime64`` dtypes (:issue:`????`) Timedelta ^^^^^^^^^ From 64305ef0e85ba0cbf3de8a34f746e4b7e4682d1f Mon Sep 17 00:00:00 2001 From: Brock Date: Sun, 27 Dec 2020 09:12:48 -0800 Subject: [PATCH 3/7] GH ref --- doc/source/whatsnew/v1.3.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.3.0.rst b/doc/source/whatsnew/v1.3.0.rst index ffd06a1b40fad..bf1b01ca3d672 100644 --- a/doc/source/whatsnew/v1.3.0.rst +++ b/doc/source/whatsnew/v1.3.0.rst @@ -185,7 +185,7 @@ Datetimelike - Bug in :class:`DataFrame` and :class:`Series` constructors sometimes dropping nanoseconds from :class:`Timestamp` (resp. :class:`Timedelta`) ``data``, with ``dtype=datetime64[ns]`` (resp. ``timedelta64[ns]``) (:issue:`38032`) - Bug in :meth:`DataFrame.first` and :meth:`Series.first` returning two months for offset one month when first day is last calendar day (:issue:`29623`) - Bug in constructing a :class:`DataFrame` or :class:`Series` with mismatched ``datetime64`` data and ``timedelta64`` dtype, or vice-versa, failing to raise ``TypeError`` (:issue:`38575`) -- Bug in :meth:`DataFrame.any` and :meth:`DataFrame.all` behaving differently for tznaive vs tzaware ``datetime64`` dtypes (:issue:`????`) +- Bug in :meth:`DataFrame.any` and :meth:`DataFrame.all` behaving differently for tznaive vs tzaware ``datetime64`` dtypes (:issue:`38723`) Timedelta ^^^^^^^^^ From a2939784ad13f8f8f8f8f76f99ab1d897485117e Mon Sep 17 00:00:00 2001 From: Brock Date: Sun, 27 Dec 2020 12:37:48 -0800 Subject: [PATCH 4/7] typo fixup --- pandas/core/arrays/datetimelike.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 98bff7f326f66..d6dd68b1cf463 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1644,11 +1644,11 @@ def ceil(self, freq, ambiguous="raise", nonexistent="raise"): # Reductions def any(self, *, axis: Optional[int] = None, skipna: bool = True): - # GH#33478 + # GH#33479 discussion of desired behavior long-term return nanops.nanany(self._ndarray, axis=axis, skipna=skipna, mask=self.isna()) def all(self, *, axis: Optional[int] = None, skipna: bool = True): - # GH#33478 + # GH#33479 discussion of desired behavior long-term return nanops.nanall(self._ndarray, axis=axis, skipna=skipna, mask=self.isna()) # -------------------------------------------------------------- From feea7757f97b890620b7b6807682d7009a6c5f66 Mon Sep 17 00:00:00 2001 From: Brock Date: Sun, 27 Dec 2020 12:38:51 -0800 Subject: [PATCH 5/7] tpyo fixup --- pandas/core/arrays/datetimelike.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index d6dd68b1cf463..65cd44478e980 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -1644,11 +1644,11 @@ def ceil(self, freq, ambiguous="raise", nonexistent="raise"): # Reductions def any(self, *, axis: Optional[int] = None, skipna: bool = True): - # GH#33479 discussion of desired behavior long-term + # GH#34479 discussion of desired behavior long-term return nanops.nanany(self._ndarray, axis=axis, skipna=skipna, mask=self.isna()) def all(self, *, axis: Optional[int] = None, skipna: bool = True): - # GH#33479 discussion of desired behavior long-term + # GH#34479 discussion of desired behavior long-term return nanops.nanall(self._ndarray, axis=axis, skipna=skipna, mask=self.isna()) # -------------------------------------------------------------- From b2e3fd9fd74a5e9ac03bfb7eb024c0436a434045 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Tue, 29 Dec 2020 13:31:36 -0800 Subject: [PATCH 6/7] Update doc/source/whatsnew/v1.2.1.rst Co-authored-by: Joris Van den Bossche --- doc/source/whatsnew/v1.2.1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.2.1.rst b/doc/source/whatsnew/v1.2.1.rst index df987fcb7a889..893593124b18e 100644 --- a/doc/source/whatsnew/v1.2.1.rst +++ b/doc/source/whatsnew/v1.2.1.rst @@ -17,7 +17,7 @@ Fixed regressions - The deprecated attributes ``_AXIS_NAMES`` and ``_AXIS_NUMBERS`` of :class:`DataFrame` and :class:`Series` will no longer show up in ``dir`` or ``inspect.getmembers`` calls (:issue:`38740`) - :meth:`to_csv` created corrupted zip files when there were more rows than ``chunksize`` (issue:`38714`) - Bug in repr of float-like strings of an ``object`` dtype having trailing 0's truncated after the decimal (:issue:`38708`) -- Bug in :meth:`DataFrame.any` and :meth:`DataFrame.all` behaving differently for tznaive vs tzaware ``datetime64`` dtypes (:issue:`38723`) +- Fixed regression in :meth:`DataFrame.any` and :meth:`DataFrame.all` not returning a result for tz-aware ``datetime64`` columns (:issue:`38723`) - .. --------------------------------------------------------------------------- From 4a85ab7354bc45488c7b074f1499742c63cf04e1 Mon Sep 17 00:00:00 2001 From: Brock Date: Tue, 29 Dec 2020 13:44:35 -0800 Subject: [PATCH 7/7] TST: test for Series any/all --- pandas/tests/reductions/test_reductions.py | 43 ++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/pandas/tests/reductions/test_reductions.py b/pandas/tests/reductions/test_reductions.py index 8c2297699807d..94afa204db891 100644 --- a/pandas/tests/reductions/test_reductions.py +++ b/pandas/tests/reductions/test_reductions.py @@ -17,6 +17,7 @@ Timedelta, TimedeltaIndex, Timestamp, + date_range, isna, timedelta_range, to_timedelta, @@ -923,6 +924,48 @@ def test_any_axis1_bool_only(self): expected = Series([True, False]) tm.assert_series_equal(result, expected) + def test_any_all_datetimelike(self): + # GH#38723 these may not be the desired long-term behavior (GH#34479) + # but in the interim should be internally consistent + dta = date_range("1995-01-02", periods=3)._data + ser = Series(dta) + df = DataFrame(ser) + + assert dta.all() + assert dta.any() + + assert ser.all() + assert ser.any() + + assert df.any().all() + assert df.all().all() + + dta = dta.tz_localize("UTC") + ser = Series(dta) + df = DataFrame(ser) + + assert dta.all() + assert dta.any() + + assert ser.all() + assert ser.any() + + assert df.any().all() + assert df.all().all() + + tda = dta - dta[0] + ser = Series(tda) + df = DataFrame(ser) + + assert tda.any() + assert not tda.all() + + assert ser.any() + assert not ser.all() + + assert df.any().all() + assert not df.all().any() + def test_timedelta64_analytics(self): # index min/max