From 3882c39561ab7672ae72dfcaeb731a74ed4335da Mon Sep 17 00:00:00 2001 From: XF Date: Tue, 1 Aug 2017 14:42:27 -0500 Subject: [PATCH 01/10] Update window.pyx --- pandas/_libs/window.pyx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index 2450eea5500cd..7c094778b3bf7 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -1427,16 +1427,18 @@ def roll_generic(ndarray[float64_t, cast=True] input, n = len(input) if n == 0: return input - + + counts = roll_sum(np.concatenate([np.isfinite(input).astype(float), + np.array([0.] * offset)]), + win, minp, index, closed)[offset:] + start, end, N, win, minp, is_variable = get_window_indexer(input, win, minp, index, closed, floor=0) output = np.empty(N, dtype=float) - counts = roll_sum(np.concatenate([np.isfinite(input).astype(float), - np.array([0.] * offset)]), - win, minp, index, closed)[offset:] + if is_variable: From 97adfd950dbc7b275ebb93508d8b33d61b82e1c4 Mon Sep 17 00:00:00 2001 From: fxocena Date: Thu, 3 Aug 2017 22:51:23 -0500 Subject: [PATCH 02/10] fix roll_generic in window.pyx add new test method for min_periods in test_window::TestRolling --- pandas/_libs/window.pyx | 9 +++++---- pandas/tests/test_window.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index 7c094778b3bf7..77374e37afeed 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -1428,14 +1428,15 @@ def roll_generic(ndarray[float64_t, cast=True] input, if n == 0: return input - counts = roll_sum(np.concatenate([np.isfinite(input).astype(float), - np.array([0.] * offset)]), - win, minp, index, closed)[offset:] - start, end, N, win, minp, is_variable = get_window_indexer(input, win, minp, index, closed, floor=0) + + counts = roll_sum(np.concatenate([np.isfinite(input).astype(float), + np.array([0.] * offset)]), + win, minp, index, closed)[offset:] + output = np.empty(N, dtype=float) diff --git a/pandas/tests/test_window.py b/pandas/tests/test_window.py index 5ab33bd6cc5e1..f76b2beaeef4f 100644 --- a/pandas/tests/test_window.py +++ b/pandas/tests/test_window.py @@ -423,6 +423,26 @@ def test_constructor_with_timedelta_window(self): expected = df.rolling('3D').sum() tm.assert_frame_equal(result, expected) + def test_constructor_with_timedelta_window_and_minperiods(self): + # GH 15440 + n = 10 + df = pd.DataFrame({'value': np.arange(n)}, + index=pd.date_range('2015-12-24', + periods=n, + freq="D")) + expected_data = np.append([np.NaN, 1.], np.arange(3., 27., 3)) + for window in [timedelta(days=3), pd.Timedelta(days=3)]: + result_roll_sum = df.rolling(window=window, min_periods=2).sum() + result_roll_generic = df.rolling(window=window, min_periods=2).apply(sum) + expected = pd.DataFrame({'value': expected_data}, + index=pd.date_range('2015-12-24', + periods=n, + freq="D")) + tm.assert_frame_equal(result_roll_sum, expected) + tm.assert_frame_equal(result_roll_generic, expected) + expected = df.rolling('3D', min_periods=2).sum() + tm.assert_frame_equal(result_roll_generic, expected) + def test_numpy_compat(self): # see gh-12811 r = rwindow.Rolling(Series([2, 4, 6]), window=2) From 0a7db15a07661ce6dcc9c7cf7310bf35cc29d01c Mon Sep 17 00:00:00 2001 From: fxocena Date: Thu, 3 Aug 2017 22:57:18 -0500 Subject: [PATCH 03/10] fix roll_generic in window.pyx --- pandas/_libs/window.pyx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index 77374e37afeed..dfba468ec96d7 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -1428,14 +1428,14 @@ def roll_generic(ndarray[float64_t, cast=True] input, if n == 0: return input + counts = roll_sum(np.concatenate([np.isfinite(input).astype(float), + np.array([0.] * offset)]), + win, minp, index, closed)[offset:] + start, end, N, win, minp, is_variable = get_window_indexer(input, win, minp, index, closed, floor=0) - - counts = roll_sum(np.concatenate([np.isfinite(input).astype(float), - np.array([0.] * offset)]), - win, minp, index, closed)[offset:] output = np.empty(N, dtype=float) From a569538b251588df67c116ffefa6d7ba9393b987 Mon Sep 17 00:00:00 2001 From: fxocena Date: Fri, 4 Aug 2017 08:02:51 -0500 Subject: [PATCH 04/10] fix PEP8 issues --- pandas/tests/test_window.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/tests/test_window.py b/pandas/tests/test_window.py index f76b2beaeef4f..dd40267e401b7 100644 --- a/pandas/tests/test_window.py +++ b/pandas/tests/test_window.py @@ -433,7 +433,8 @@ def test_constructor_with_timedelta_window_and_minperiods(self): expected_data = np.append([np.NaN, 1.], np.arange(3., 27., 3)) for window in [timedelta(days=3), pd.Timedelta(days=3)]: result_roll_sum = df.rolling(window=window, min_periods=2).sum() - result_roll_generic = df.rolling(window=window, min_periods=2).apply(sum) + result_roll_generic = df.rolling(window=window, + min_periods=2).apply(sum) expected = pd.DataFrame({'value': expected_data}, index=pd.date_range('2015-12-24', periods=n, @@ -442,7 +443,7 @@ def test_constructor_with_timedelta_window_and_minperiods(self): tm.assert_frame_equal(result_roll_generic, expected) expected = df.rolling('3D', min_periods=2).sum() tm.assert_frame_equal(result_roll_generic, expected) - + def test_numpy_compat(self): # see gh-12811 r = rwindow.Rolling(Series([2, 4, 6]), window=2) From eb3434835e096446ebc498e7c1913d202708531a Mon Sep 17 00:00:00 2001 From: FXocena Date: Sat, 5 Aug 2017 10:55:39 -0500 Subject: [PATCH 05/10] for Travis --- pandas/_libs/window.pyx | 2 -- pandas/tests/test_window.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index dfba468ec96d7..679ed7222b26f 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -1439,8 +1439,6 @@ def roll_generic(ndarray[float64_t, cast=True] input, output = np.empty(N, dtype=float) - - if is_variable: # variable window diff --git a/pandas/tests/test_window.py b/pandas/tests/test_window.py index dd40267e401b7..a093131885be0 100644 --- a/pandas/tests/test_window.py +++ b/pandas/tests/test_window.py @@ -433,7 +433,7 @@ def test_constructor_with_timedelta_window_and_minperiods(self): expected_data = np.append([np.NaN, 1.], np.arange(3., 27., 3)) for window in [timedelta(days=3), pd.Timedelta(days=3)]: result_roll_sum = df.rolling(window=window, min_periods=2).sum() - result_roll_generic = df.rolling(window=window, + result_roll_generic = df.rolling(window=window, min_periods=2).apply(sum) expected = pd.DataFrame({'value': expected_data}, index=pd.date_range('2015-12-24', From 679ad4e62cc528e7a68c842949386b06a4eac82e Mon Sep 17 00:00:00 2001 From: fxocena Date: Sat, 5 Aug 2017 14:19:51 -0500 Subject: [PATCH 06/10] fix issue #15305, update _libs\window.pyx and tests\test_window.py --- doc/source/whatsnew/v0.21.0.txt | 2 ++ pandas/tests/test_window.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.21.0.txt b/doc/source/whatsnew/v0.21.0.txt index 1931ffff4b217..8fa4bcf28b04e 100644 --- a/doc/source/whatsnew/v0.21.0.txt +++ b/doc/source/whatsnew/v0.21.0.txt @@ -296,6 +296,8 @@ Groupby/Resample/Rolling - Bug in :func:`infer_freq` causing indices with 2-day gaps during the working week to be wrongly inferred as business daily (:issue:`16624`) - Bug in ``.rolling(...).quantile()`` which incorrectly used different defaults than :func:`Series.quantile()` and :func:`DataFrame.quantile()` (:issue:`9413`, :issue:`16211`) - Bug in ``groupby.transform()`` that would coerce boolean dtypes back to float (:issue:`16875`) +- Bug in ``.rolling(...).apply(...)`` where when rolling apply on DataFrame + with Datetime index and min_periods set to be greater than 1 (:issue:`15305`) Sparse ^^^^^^ diff --git a/pandas/tests/test_window.py b/pandas/tests/test_window.py index a093131885be0..7e31bd4e2645a 100644 --- a/pandas/tests/test_window.py +++ b/pandas/tests/test_window.py @@ -424,7 +424,7 @@ def test_constructor_with_timedelta_window(self): tm.assert_frame_equal(result, expected) def test_constructor_with_timedelta_window_and_minperiods(self): - # GH 15440 + # GH 15305 n = 10 df = pd.DataFrame({'value': np.arange(n)}, index=pd.date_range('2015-12-24', From 5890579988070d27471a852839dc0dc059b1a455 Mon Sep 17 00:00:00 2001 From: fxocena Date: Tue, 8 Aug 2017 08:25:46 -0500 Subject: [PATCH 07/10] changed test_window.py::TestRolling --- pandas/tests/test_window.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/pandas/tests/test_window.py b/pandas/tests/test_window.py index 7e31bd4e2645a..ef1b74798cda9 100644 --- a/pandas/tests/test_window.py +++ b/pandas/tests/test_window.py @@ -427,22 +427,18 @@ def test_constructor_with_timedelta_window_and_minperiods(self): # GH 15305 n = 10 df = pd.DataFrame({'value': np.arange(n)}, - index=pd.date_range('2015-12-24', + index=pd.date_range('2017-08-08', periods=n, freq="D")) - expected_data = np.append([np.NaN, 1.], np.arange(3., 27., 3)) - for window in [timedelta(days=3), pd.Timedelta(days=3)]: + expected = pd.DataFrame( + {'value':np.append([np.NaN, 1.], np.arange(3., 27., 3))}, + index=pd.date_range('2017-08-08', periods=n, freq="D")) + for window in [timedelta(days=3), pd.Timedelta(days=3), '3D']: result_roll_sum = df.rolling(window=window, min_periods=2).sum() result_roll_generic = df.rolling(window=window, min_periods=2).apply(sum) - expected = pd.DataFrame({'value': expected_data}, - index=pd.date_range('2015-12-24', - periods=n, - freq="D")) tm.assert_frame_equal(result_roll_sum, expected) tm.assert_frame_equal(result_roll_generic, expected) - expected = df.rolling('3D', min_periods=2).sum() - tm.assert_frame_equal(result_roll_generic, expected) def test_numpy_compat(self): # see gh-12811 From 4148bb3ecc872cb5f028d9181f4dc892b80d714b Mon Sep 17 00:00:00 2001 From: fxocena Date: Tue, 8 Aug 2017 08:53:52 -0500 Subject: [PATCH 08/10] changed test_window.py::TestRolling --- pandas/tests/test_window.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pandas/tests/test_window.py b/pandas/tests/test_window.py index ef1b74798cda9..ab6c8fba016cb 100644 --- a/pandas/tests/test_window.py +++ b/pandas/tests/test_window.py @@ -430,9 +430,11 @@ def test_constructor_with_timedelta_window_and_minperiods(self): index=pd.date_range('2017-08-08', periods=n, freq="D")) - expected = pd.DataFrame( - {'value':np.append([np.NaN, 1.], np.arange(3., 27., 3))}, - index=pd.date_range('2017-08-08', periods=n, freq="D")) + expected = pd.DataFrame({'value':np.append([np.NaN, 1.], + np.arange(3., 27., 3))}, + index=pd.date_range('2017-08-08', + periods=n, + freq="D")) for window in [timedelta(days=3), pd.Timedelta(days=3), '3D']: result_roll_sum = df.rolling(window=window, min_periods=2).sum() result_roll_generic = df.rolling(window=window, From 334a7d1b43f2b48b7cb6fb8ee8548684fbdf3bb4 Mon Sep 17 00:00:00 2001 From: fxocena Date: Tue, 8 Aug 2017 09:33:56 -0500 Subject: [PATCH 09/10] changed test_window.py::TestRolling --- pandas/tests/test_window.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/test_window.py b/pandas/tests/test_window.py index ab6c8fba016cb..0e701d62233a5 100644 --- a/pandas/tests/test_window.py +++ b/pandas/tests/test_window.py @@ -430,8 +430,8 @@ def test_constructor_with_timedelta_window_and_minperiods(self): index=pd.date_range('2017-08-08', periods=n, freq="D")) - expected = pd.DataFrame({'value':np.append([np.NaN, 1.], - np.arange(3., 27., 3))}, + expected = pd.DataFrame({'value': np.append([np.NaN, 1.], + np.arange(3., 27., 3))}, index=pd.date_range('2017-08-08', periods=n, freq="D")) From 9c5e1c16bd1510188bfee95d90e1e8220e5b043a Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Tue, 8 Aug 2017 19:37:51 -0400 Subject: [PATCH 10/10] clean up whatsnew, parametrize tests, fix some whitespace issues --- doc/source/whatsnew/v0.21.0.txt | 3 +-- pandas/_libs/window.pyx | 4 ++-- pandas/tests/test_window.py | 15 ++++++++------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/source/whatsnew/v0.21.0.txt b/doc/source/whatsnew/v0.21.0.txt index 8fa4bcf28b04e..a5b933fe74ff5 100644 --- a/doc/source/whatsnew/v0.21.0.txt +++ b/doc/source/whatsnew/v0.21.0.txt @@ -296,8 +296,7 @@ Groupby/Resample/Rolling - Bug in :func:`infer_freq` causing indices with 2-day gaps during the working week to be wrongly inferred as business daily (:issue:`16624`) - Bug in ``.rolling(...).quantile()`` which incorrectly used different defaults than :func:`Series.quantile()` and :func:`DataFrame.quantile()` (:issue:`9413`, :issue:`16211`) - Bug in ``groupby.transform()`` that would coerce boolean dtypes back to float (:issue:`16875`) -- Bug in ``.rolling(...).apply(...)`` where when rolling apply on DataFrame - with Datetime index and min_periods set to be greater than 1 (:issue:`15305`) +- Bug in ``.rolling(...).apply(...)`` with a ``DataFrame`` with a ``DatetimeIndex``, a ``window`` of a timedelta-convertible and ``min_periods >= 1` (:issue:`15305`) Sparse ^^^^^^ diff --git a/pandas/_libs/window.pyx b/pandas/_libs/window.pyx index 679ed7222b26f..bdd371871b6e1 100644 --- a/pandas/_libs/window.pyx +++ b/pandas/_libs/window.pyx @@ -1427,11 +1427,11 @@ def roll_generic(ndarray[float64_t, cast=True] input, n = len(input) if n == 0: return input - + counts = roll_sum(np.concatenate([np.isfinite(input).astype(float), np.array([0.] * offset)]), win, minp, index, closed)[offset:] - + start, end, N, win, minp, is_variable = get_window_indexer(input, win, minp, index, closed, diff --git a/pandas/tests/test_window.py b/pandas/tests/test_window.py index 0e701d62233a5..d94e34c41786b 100644 --- a/pandas/tests/test_window.py +++ b/pandas/tests/test_window.py @@ -423,7 +423,9 @@ def test_constructor_with_timedelta_window(self): expected = df.rolling('3D').sum() tm.assert_frame_equal(result, expected) - def test_constructor_with_timedelta_window_and_minperiods(self): + @pytest.mark.parametrize( + 'window', [timedelta(days=3), pd.Timedelta(days=3), '3D']) + def test_constructor_with_timedelta_window_and_minperiods(self, window): # GH 15305 n = 10 df = pd.DataFrame({'value': np.arange(n)}, @@ -435,12 +437,11 @@ def test_constructor_with_timedelta_window_and_minperiods(self): index=pd.date_range('2017-08-08', periods=n, freq="D")) - for window in [timedelta(days=3), pd.Timedelta(days=3), '3D']: - result_roll_sum = df.rolling(window=window, min_periods=2).sum() - result_roll_generic = df.rolling(window=window, - min_periods=2).apply(sum) - tm.assert_frame_equal(result_roll_sum, expected) - tm.assert_frame_equal(result_roll_generic, expected) + result_roll_sum = df.rolling(window=window, min_periods=2).sum() + result_roll_generic = df.rolling(window=window, + min_periods=2).apply(sum) + tm.assert_frame_equal(result_roll_sum, expected) + tm.assert_frame_equal(result_roll_generic, expected) def test_numpy_compat(self): # see gh-12811