From dc2280fcbbaa7ad53d12424007edb3c1135abffb Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 16 Oct 2018 19:05:34 -0700 Subject: [PATCH 01/30] Make arithmetic code dispatch less redundant, fix datetime64 addition, add missing tests --- pandas/core/arrays/datetimelike.py | 55 ++++++++++-- pandas/core/arrays/datetimes.py | 98 +++++++-------------- pandas/core/arrays/period.py | 40 ++------- pandas/core/arrays/timedeltas.py | 70 +++++---------- pandas/tests/arithmetic/test_period.py | 15 ++++ pandas/tests/arithmetic/test_timedelta64.py | 12 +++ 6 files changed, 134 insertions(+), 156 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index a98b0b3bf35f9..79978c6c4ea62 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -347,21 +347,59 @@ def _validate_frequency(cls, index, freq, **kwargs): # Arithmetic Methods def _add_datelike(self, other): + # Overriden by TimedeltaArray raise TypeError("cannot add {cls} and {typ}" .format(cls=type(self).__name__, typ=type(other).__name__)) + _add_datelike_dti = _add_datelike + def _sub_datelike(self, other): - raise com.AbstractMethodError(self) + # Overridden by DatetimeArray + assert other is not NaT + raise TypeError("cannot subtract a datelike from a {cls}" + .format(cls=type(self).__name__)) + + _sub_datelike_dti = _sub_datelike def _sub_period(self, other): - return NotImplemented + # Overriden by PeriodArray + raise TypeError("cannot subtract Period from a {cls}" + .format(cls=type(self).__name__)) def _add_offset(self, offset): raise com.AbstractMethodError(self) def _add_delta(self, other): - return NotImplemented + """ + Add a timedelta-like, DateOffset, or TimedeltaIndex-like object + to self. + + Parameters + ---------- + delta : {timedelta, np.timedelta64, DateOffset, + TimedeltaIndex, ndarray[timedelta64]} + + Returns + ------- + result : same type as self + + Notes + ----- + The result's name is set outside of _add_delta by the calling + method (__add__ or __sub__), if necessary (i.e. for Indexes). + """ + # Note: The docstring here says the return type is the same type + # as self, which is inaccurate. Once wrapped by the inheriting + # Array classes, this will be accurate. + + if isinstance(other, (Tick, timedelta, np.timedelta64)): + new_values = self._add_delta_td(other) + elif is_timedelta64_dtype(other): + # ndarray[timedelta64] or TimedeltaArray/index + new_values = self._add_delta_tdi(other) + + return new_values def _add_delta_td(self, other): """ @@ -371,8 +409,7 @@ def _add_delta_td(self, other): inc = delta_to_nanoseconds(other) new_values = checked_add_with_arr(self.asi8, inc, arr_mask=self._isnan).view('i8') - if self.hasnans: - new_values[self._isnan] = iNaT + new_values = self._maybe_mask_results(new_values, fill_value=iNaT) return new_values.view('i8') def _add_delta_tdi(self, other): @@ -380,7 +417,7 @@ def _add_delta_tdi(self, other): Add a delta of a TimedeltaIndex return the i8 result view """ - if not len(self) == len(other): + if len(self) != len(other): raise ValueError("cannot add indices of unequal length") if isinstance(other, np.ndarray): @@ -441,7 +478,7 @@ def _sub_period_array(self, other): .format(dtype=other.dtype, cls=type(self).__name__)) - if not len(self) == len(other): + if len(self) != len(other): raise ValueError("cannot subtract arrays/indices of " "unequal length") if self.freq != other.freq: @@ -635,7 +672,7 @@ def __add__(self, other): result = self._addsub_offset_array(other, operator.add) elif is_datetime64_dtype(other) or is_datetime64tz_dtype(other): # DatetimeIndex, ndarray[datetime64] - return self._add_datelike(other) + return self._add_datelike_dti(other) elif is_integer_dtype(other): result = self._addsub_int_array(other, operator.add) elif is_float_dtype(other): @@ -695,7 +732,7 @@ def __sub__(self, other): result = self._addsub_offset_array(other, operator.sub) elif is_datetime64_dtype(other) or is_datetime64tz_dtype(other): # DatetimeIndex, ndarray[datetime64] - result = self._sub_datelike(other) + result = self._sub_datelike_dti(other) elif is_period_dtype(other): # PeriodIndex result = self._sub_period_array(other) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 4cc33d7afd6c8..0f6b8a4f62178 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from datetime import datetime, timedelta, time +from datetime import datetime, time import warnings import numpy as np @@ -12,7 +12,7 @@ conversion, fields, timezones, resolution as libresolution) -from pandas.util._decorators import cache_readonly +from pandas.util._decorators import cache_readonly, Appender from pandas.errors import PerformanceWarning from pandas import compat @@ -21,7 +21,6 @@ is_object_dtype, is_datetime64tz_dtype, is_datetime64_dtype, - is_timedelta64_dtype, ensure_int64) from pandas.core.dtypes.dtypes import DatetimeTZDtype from pandas.core.dtypes.missing import isna @@ -425,10 +424,20 @@ def _assert_tzawareness_compat(self, other): # Arithmetic Methods def _sub_datelike_dti(self, other): - """subtraction of two DatetimeIndexes""" - if not len(self) == len(other): + """subtract DatetimeArray/Index or ndarray[datetime64]""" + if len(self) != len(other): raise ValueError("cannot add indices of unequal length") + if isinstance(other, np.ndarray): + # if other is an ndarray, we assume it is datetime64-dtype + other = type(self)(other) + + if not self._has_same_tz(other): + # require tz compat + raise TypeError("{cls} subtraction must have the same " + "timezones or no timezones" + .format(cls=type(self).__name__)) + self_i8 = self.asi8 other_i8 = other.asi8 new_values = checked_add_with_arr(self_i8, -other_i8, @@ -458,72 +467,27 @@ def _add_offset(self, offset): def _sub_datelike(self, other): # subtract a datetime from myself, yielding a ndarray[timedelta64[ns]] - if isinstance(other, (DatetimeArrayMixin, np.ndarray)): - if isinstance(other, np.ndarray): - # if other is an ndarray, we assume it is datetime64-dtype - other = type(self)(other) - if not self._has_same_tz(other): - # require tz compat - raise TypeError("{cls} subtraction must have the same " - "timezones or no timezones" - .format(cls=type(self).__name__)) - result = self._sub_datelike_dti(other) - elif isinstance(other, (datetime, np.datetime64)): - assert other is not NaT - other = Timestamp(other) - if other is NaT: - return self - NaT + assert isinstance(other, (datetime, np.datetime64)) + assert other is not NaT + other = Timestamp(other) + if other is NaT: + return self - NaT + + if not self._has_same_tz(other): # require tz compat - elif not self._has_same_tz(other): - raise TypeError("Timestamp subtraction must have the same " - "timezones or no timezones") - else: - i8 = self.asi8 - result = checked_add_with_arr(i8, -other.value, - arr_mask=self._isnan) - result = self._maybe_mask_results(result, - fill_value=iNaT) - else: - raise TypeError("cannot subtract {cls} and {typ}" - .format(cls=type(self).__name__, - typ=type(other).__name__)) + raise TypeError("Timestamp subtraction must have the same " + "timezones or no timezones") + + i8 = self.asi8 + result = checked_add_with_arr(i8, -other.value, + arr_mask=self._isnan) + result = self._maybe_mask_results(result, fill_value=iNaT) return result.view('timedelta64[ns]') + @Appender(dtl.DatetimeLikeArrayMixin._add_delta.__doc__) def _add_delta(self, delta): - """ - Add a timedelta-like, DateOffset, or TimedeltaIndex-like object - to self. - - Parameters - ---------- - delta : {timedelta, np.timedelta64, DateOffset, - TimedeltaIndex, ndarray[timedelta64]} - - Returns - ------- - result : same type as self - - Notes - ----- - The result's name is set outside of _add_delta by the calling - method (__add__ or __sub__) - """ - from pandas.core.arrays.timedeltas import TimedeltaArrayMixin - - if isinstance(delta, (Tick, timedelta, np.timedelta64)): - new_values = self._add_delta_td(delta) - elif is_timedelta64_dtype(delta): - if not isinstance(delta, TimedeltaArrayMixin): - delta = TimedeltaArrayMixin(delta) - new_values = self._add_delta_tdi(delta) - else: - new_values = self.astype('O') + delta - - tz = 'UTC' if self.tz is not None else None - result = type(self)(new_values, tz=tz, freq='infer') - if self.tz is not None and self.tz is not utc: - result = result.tz_convert(self.tz) - return result + new_values = dtl.DatetimeLikeArrayMixin._add_delta(self, delta) + return type(self)(new_values, tz=self.tz, freq='infer') # ----------------------------------------------------------------- # Timezone Conversion and Localization Methods diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 8624ddd8965e8..79671ee8aedbd 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -15,10 +15,10 @@ from pandas._libs.tslibs.fields import isleapyear_arr from pandas import compat -from pandas.util._decorators import (cache_readonly, deprecate_kwarg) +from pandas.util._decorators import cache_readonly, deprecate_kwarg, Appender from pandas.core.dtypes.common import ( - is_integer_dtype, is_float_dtype, is_period_dtype, is_timedelta64_dtype, + is_integer_dtype, is_float_dtype, is_period_dtype, is_datetime64_dtype, _TD_DTYPE) from pandas.core.dtypes.dtypes import PeriodDtype from pandas.core.dtypes.generic import ABCSeries @@ -334,10 +334,6 @@ def to_timestamp(self, freq=None, how='start'): _create_comparison_method = classmethod(_period_array_cmp) - def _sub_datelike(self, other): - assert other is not NaT - return NotImplemented - def _sub_period(self, other): # If the operation is well-defined, we return an object-Index # of DateOffsets. Null entries are filled with pd.NaT @@ -349,9 +345,7 @@ def _sub_period(self, other): new_data = asi8 - other.ordinal new_data = np.array([self.freq * x for x in new_data]) - if self.hasnans: - new_data[self._isnan] = NaT - + new_data = self._maybe_mask_results(new_data, fill_value=NaT) return new_data def _add_offset(self, other): @@ -379,20 +373,8 @@ def _add_delta_tdi(self, other): delta = self._check_timedeltalike_freq_compat(other) return self._addsub_int_array(delta, operator.add) + @Appender(DatetimeLikeArrayMixin._add_delta.__doc__) def _add_delta(self, other): - """ - Add a timedelta-like, Tick, or TimedeltaIndex-like object - to self. - - Parameters - ---------- - other : {timedelta, np.timedelta64, Tick, - TimedeltaIndex, ndarray[timedelta64]} - - Returns - ------- - result : same type as self - """ if not isinstance(self.freq, Tick): # We cannot add timedelta-like to non-tick PeriodArray raise IncompatibleFrequency("Input has different freq from " @@ -400,17 +382,9 @@ def _add_delta(self, other): .format(cls=type(self).__name__, freqstr=self.freqstr)) - # TODO: standardize across datetimelike subclasses whether to return - # i8 view or _shallow_copy - if isinstance(other, (Tick, timedelta, np.timedelta64)): - new_values = self._add_delta_td(other) - return self._shallow_copy(new_values) - elif is_timedelta64_dtype(other): - # ndarray[timedelta64] or TimedeltaArray/index - new_values = self._add_delta_tdi(other) - return self._shallow_copy(new_values) - else: # pragma: no cover - raise TypeError(type(other).__name__) + new_values = DatetimeLikeArrayMixin._add_delta(self, other) + + return self._shallow_copy(new_values) @deprecate_kwarg(old_arg_name='n', new_arg_name='periods') def shift(self, periods): diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 4904a90ab7b2b..a774c3fc2fb14 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -8,6 +8,7 @@ from pandas._libs.tslibs.fields import get_timedelta_field from pandas._libs.tslibs.timedeltas import array_to_timedelta64 +from pandas.util._decorators import Appender from pandas import compat from pandas.core.dtypes.common import ( @@ -191,60 +192,35 @@ def _add_offset(self, other): .format(typ=type(other).__name__, cls=type(self).__name__)) - def _sub_datelike(self, other): - assert other is not NaT - raise TypeError("cannot subtract a datelike from a {cls}" - .format(cls=type(self).__name__)) - + @Appender(dtl.DatetimeLikeArrayMixin._add_delta.__doc__) def _add_delta(self, delta): - """ - Add a timedelta-like, Tick, or TimedeltaIndex-like object - to self. - - Parameters - ---------- - delta : timedelta, np.timedelta64, Tick, TimedeltaArray, TimedeltaIndex - - Returns - ------- - result : same type as self + new_values = dtl.DatetimeLikeArrayMixin._add_delta(self, delta) + return type(self)(new_values, freq='infer') - Notes - ----- - The result's name is set outside of _add_delta by the calling - method (__add__ or __sub__) - """ - if isinstance(delta, (Tick, timedelta, np.timedelta64)): - new_values = self._add_delta_td(delta) - elif isinstance(delta, TimedeltaArrayMixin): - new_values = self._add_delta_tdi(delta) - elif is_timedelta64_dtype(delta): - # ndarray[timedelta64] --> wrap in TimedeltaArray/Index - delta = type(self)(delta) - new_values = self._add_delta_tdi(delta) - else: - raise TypeError("cannot add the type {0} to a TimedeltaIndex" - .format(type(delta))) + def _add_datelike_dti(self, other): + """Add DatetimeArray/Index or ndarray[datetime64] to TimedeltaArray""" + if isinstance(other, np.ndarray): + # At this point we have already checked that dtype is datetime64 + from pandas.core.arrays import DatetimeArrayMixin + other = DatetimeArrayMixin(other) - return type(self)(new_values, freq='infer') + # defer to implementation in DatetimeArray + return other + self def _add_datelike(self, other): # adding a timedeltaindex to a datetimelike from pandas.core.arrays import DatetimeArrayMixin - if isinstance(other, (DatetimeArrayMixin, np.ndarray)): - # if other is an ndarray, we assume it is datetime64-dtype - # defer to implementation in DatetimeIndex - if not isinstance(other, DatetimeArrayMixin): - other = DatetimeArrayMixin(other) - return other + self - else: - assert other is not NaT - other = Timestamp(other) - i8 = self.asi8 - result = checked_add_with_arr(i8, other.value, - arr_mask=self._isnan) - result = self._maybe_mask_results(result, fill_value=iNaT) - return DatetimeArrayMixin(result) + + assert other is not NaT + other = Timestamp(other) + if other is NaT: + return self + NaT + + i8 = self.asi8 + result = checked_add_with_arr(i8, other.value, + arr_mask=self._isnan) + result = self._maybe_mask_results(result, fill_value=iNaT) + return DatetimeArrayMixin(result) def _addsub_offset_array(self, other, op): # Add or subtract Array-like of DateOffset objects diff --git a/pandas/tests/arithmetic/test_period.py b/pandas/tests/arithmetic/test_period.py index fe98b74499983..d564b29fdff80 100644 --- a/pandas/tests/arithmetic/test_period.py +++ b/pandas/tests/arithmetic/test_period.py @@ -411,6 +411,21 @@ def test_pi_add_sub_float(self, op, other): with pytest.raises(TypeError): op(pi, other) + @pytest.mark.parametrize('other', [pd.Timestamp.now(), + pd.Timestamp.now().to_pydatetime(), + pd.Timestamp.now().to_datetime64()]) + def test_pi_add_sub_datetime(self, other): + rng = pd.period_range('1/1/2000', freq='D', periods=3) + + with pytest.raises(TypeError): + rng + other + with pytest.raises(TypeError): + other + rng + with pytest.raises(TypeError): + rng - other + with pytest.raises(TypeError): + other - rng + # ----------------------------------------------------------------- # __add__/__sub__ with ndarray[datetime64] and ndarray[timedelta64] diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index fa1a2d9df9a58..aae7ae4b7490b 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -460,6 +460,18 @@ def test_td64arr_add_sub_timestamp(self, box): with pytest.raises(TypeError): tdser - ts + def test_td64arr_add_datetime64_nat(self, box): + other = np.datetime64('NaT') + + tdi = timedelta_range('1 day', periods=3) + expected = pd.DatetimeIndex(["NaT", "NaT", "NaT"]) + + tdser = tm.box_expected(tdi, box) + expected = tm.box_expected(expected, box) + + tm.assert_equal(other + tdser, expected) + tm.assert_equal(tdser + other, expected) + def test_tdi_sub_dt64_array(self, box_df_broadcast_failure): box = box_df_broadcast_failure From 37728ff55ed9a380bc70ec76c3a1c4db9f3879e2 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 16 Oct 2018 19:13:47 -0700 Subject: [PATCH 02/30] remove unnecessary nat_new --- pandas/core/arrays/datetimelike.py | 25 +++------------------ pandas/tests/indexes/datetimes/test_ops.py | 10 --------- pandas/tests/indexes/period/test_ops.py | 11 --------- pandas/tests/indexes/timedeltas/test_ops.py | 11 --------- 4 files changed, 3 insertions(+), 54 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 79978c6c4ea62..00f7ff9e4f0cf 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -246,27 +246,6 @@ def _maybe_mask_results(self, result, fill_value=None, convert=None): result[self._isnan] = fill_value return result - def _nat_new(self, box=True): - """ - Return Array/Index or ndarray filled with NaT which has the same - length as the caller. - - Parameters - ---------- - box : boolean, default True - - If True returns a Array/Index as the same as caller. - - If False returns ndarray of np.int64. - """ - result = np.zeros(len(self), dtype=np.int64) - result.fill(iNaT) - if not box: - return result - - attribs = self._get_attributes_dict() - if not is_period_dtype(self): - attribs['freq'] = None - return self._simple_new(result, **attribs) - # ------------------------------------------------------------------ # Frequency Properties/Methods @@ -444,7 +423,9 @@ def _add_nat(self): # GH#19124 pd.NaT is treated like a timedelta for both timedelta # and datetime dtypes - return self._nat_new(box=True) + result = np.zeros(len(self), dtype=np.int64) + result.fill(iNaT) + return self._shallow_copy(result, freq=None) def _sub_nat(self): """Subtract pd.NaT from self""" diff --git a/pandas/tests/indexes/datetimes/test_ops.py b/pandas/tests/indexes/datetimes/test_ops.py index 9ce77326d37b7..771165eafec7a 100644 --- a/pandas/tests/indexes/datetimes/test_ops.py +++ b/pandas/tests/indexes/datetimes/test_ops.py @@ -337,16 +337,6 @@ def test_infer_freq(self, freq): tm.assert_index_equal(idx, result) assert result.freq == freq - def test_nat_new(self): - idx = pd.date_range('2011-01-01', freq='D', periods=5, name='x') - result = idx._nat_new() - exp = pd.DatetimeIndex([pd.NaT] * 5, name='x') - tm.assert_index_equal(result, exp) - - result = idx._nat_new(box=False) - exp = np.array([tslib.iNaT] * 5, dtype=np.int64) - tm.assert_numpy_array_equal(result, exp) - def test_nat(self, tz_naive_fixture): tz = tz_naive_fixture assert pd.DatetimeIndex._na_value is pd.NaT diff --git a/pandas/tests/indexes/period/test_ops.py b/pandas/tests/indexes/period/test_ops.py index a59efe57f83c4..5e95fcfdd6c3e 100644 --- a/pandas/tests/indexes/period/test_ops.py +++ b/pandas/tests/indexes/period/test_ops.py @@ -312,17 +312,6 @@ def test_order(self): tm.assert_numpy_array_equal(indexer, exp, check_dtype=False) assert ordered.freq == 'D' - def test_nat_new(self): - - idx = pd.period_range('2011-01', freq='M', periods=5, name='x') - result = idx._nat_new() - exp = pd.PeriodIndex([pd.NaT] * 5, freq='M', name='x') - tm.assert_index_equal(result, exp) - - result = idx._nat_new(box=False) - exp = np.array([tslib.iNaT] * 5, dtype=np.int64) - tm.assert_numpy_array_equal(result, exp) - def test_shift(self): # This is tested in test_arithmetic pass diff --git a/pandas/tests/indexes/timedeltas/test_ops.py b/pandas/tests/indexes/timedeltas/test_ops.py index 9f8a3e893c3de..8af32f1f9421b 100644 --- a/pandas/tests/indexes/timedeltas/test_ops.py +++ b/pandas/tests/indexes/timedeltas/test_ops.py @@ -236,17 +236,6 @@ def test_infer_freq(self, freq): tm.assert_index_equal(idx, result) assert result.freq == freq - def test_nat_new(self): - - idx = pd.timedelta_range('1', freq='D', periods=5, name='x') - result = idx._nat_new() - exp = pd.TimedeltaIndex([pd.NaT] * 5, name='x') - tm.assert_index_equal(result, exp) - - result = idx._nat_new(box=False) - exp = np.array([iNaT] * 5, dtype=np.int64) - tm.assert_numpy_array_equal(result, exp) - def test_shift(self): pass # handled in test_arithmetic.py From 15ad0a62e178039b8c687ebb0d60e99db39e15ed Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 16 Oct 2018 20:38:40 -0700 Subject: [PATCH 03/30] Fix interpretation of NaT --- pandas/core/arrays/timedeltas.py | 5 ++++- pandas/tests/arithmetic/test_timedelta64.py | 2 +- pandas/util/testing.py | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index a774c3fc2fb14..45c6e94f742be 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -214,7 +214,10 @@ def _add_datelike(self, other): assert other is not NaT other = Timestamp(other) if other is NaT: - return self + NaT + # In this case we specifically interpret NaT as a datetime, not + # the timedelta interpretation we would get by returning self + NaT + result = self.asi8.view('m8[ms]') + NaT.to_datetime64() + return DatetimeArrayMixin(result) i8 = self.asi8 result = checked_add_with_arr(i8, other.value, diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index aae7ae4b7490b..5bba98890b917 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -469,8 +469,8 @@ def test_td64arr_add_datetime64_nat(self, box): tdser = tm.box_expected(tdi, box) expected = tm.box_expected(expected, box) - tm.assert_equal(other + tdser, expected) tm.assert_equal(tdser + other, expected) + tm.assert_equal(other + tdser, expected) def test_tdi_sub_dt64_array(self, box_df_broadcast_failure): box = box_df_broadcast_failure diff --git a/pandas/util/testing.py b/pandas/util/testing.py index 1bd9043f42634..82be5edc73b55 100644 --- a/pandas/util/testing.py +++ b/pandas/util/testing.py @@ -934,6 +934,7 @@ def assert_attr_equal(attr, left, right, obj='Attributes'): Specify object name being compared, internally used to show appropriate assertion message """ + __tracebackhide__ = True left_attr = getattr(left, attr) right_attr = getattr(right, attr) From 94f174582e49424e20d6b105583aa024248fe8c4 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 17 Oct 2018 15:29:02 -0700 Subject: [PATCH 04/30] move test --- pandas/tests/arithmetic/test_timedelta64.py | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index 5bba98890b917..5fcddd7661f5c 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -460,18 +460,6 @@ def test_td64arr_add_sub_timestamp(self, box): with pytest.raises(TypeError): tdser - ts - def test_td64arr_add_datetime64_nat(self, box): - other = np.datetime64('NaT') - - tdi = timedelta_range('1 day', periods=3) - expected = pd.DatetimeIndex(["NaT", "NaT", "NaT"]) - - tdser = tm.box_expected(tdi, box) - expected = tm.box_expected(expected, box) - - tm.assert_equal(tdser + other, expected) - tm.assert_equal(other + tdser, expected) - def test_tdi_sub_dt64_array(self, box_df_broadcast_failure): box = box_df_broadcast_failure @@ -783,6 +771,18 @@ def test_td64arr_sub_timedeltalike(self, two_hours, box): result = rng - two_hours tm.assert_equal(result, expected) + def test_td64arr_add_datetime64_nat(self, box): + other = np.datetime64('NaT') + + tdi = timedelta_range('1 day', periods=3) + expected = pd.DatetimeIndex(["NaT", "NaT", "NaT"]) + + tdser = tm.box_expected(tdi, box) + expected = tm.box_expected(expected, box) + + tm.assert_equal(tdser + other, expected) + tm.assert_equal(other + tdser, expected) + # ------------------------------------------------------------------ # __add__/__sub__ with DateOffsets and arrays of DateOffsets From 3bdf1047fd328d4a3c638972531d9a6f474535b6 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 17 Oct 2018 16:02:08 -0700 Subject: [PATCH 05/30] whatsnew, fix dropped timezone --- doc/source/whatsnew/v0.24.0.txt | 3 +- pandas/core/arrays/timedeltas.py | 2 +- pandas/tests/arithmetic/test_timedelta64.py | 45 +++++++++------------ 3 files changed, 23 insertions(+), 27 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 16f0b9ee99909..5f99434658b7c 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -825,7 +825,8 @@ Timedelta - Bug in :class:`TimedeltaIndex` incorrectly allowing indexing with ``Timestamp`` object (:issue:`20464`) - Fixed bug where subtracting :class:`Timedelta` from an object-dtyped array would raise ``TypeError`` (:issue:`21980`) - Fixed bug in adding a :class:`DataFrame` with all-`timedelta64[ns]` dtypes to a :class:`DataFrame` with all-integer dtypes returning incorrect results instead of raising ``TypeError`` (:issue:`22696`) -- +- Bug in :class:`TimedeltaIndex` where adding a timezone-aware datetime scalar incorrectly returned a timezone-naive :class:`DatetimeIndex` (:issue:`?????`) +- Bug in :class:`TimedeltaIndex` where adding ``np.timedelta64('NaT')`` incorrectly returned an all-`NaT` :class:`DatetimeIndex` instead of an all-`NaT` :class:`TimedeltaIndex` (:issue:`?????`) Timezones ^^^^^^^^^ diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 45c6e94f742be..b38cf017da9a5 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -223,7 +223,7 @@ def _add_datelike(self, other): result = checked_add_with_arr(i8, other.value, arr_mask=self._isnan) result = self._maybe_mask_results(result, fill_value=iNaT) - return DatetimeArrayMixin(result) + return DatetimeArrayMixin(result, tz=other.tz) # FIXME: what about timezone? def _addsub_offset_array(self, other, op): # Add or subtract Array-like of DateOffset objects diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index 5fcddd7661f5c..acc434a5a2176 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -415,25 +415,20 @@ def test_td64arr_sub_timestamp_raises(self, box): with tm.assert_raises_regex(TypeError, msg): idx - Timestamp('2011-01-01') - def test_td64arr_add_timestamp(self, box): - idx = TimedeltaIndex(['1 day', '2 day']) - expected = DatetimeIndex(['2011-01-02', '2011-01-03']) - - idx = tm.box_expected(idx, box) - expected = tm.box_expected(expected, box) - - result = idx + Timestamp('2011-01-01') - tm.assert_equal(result, expected) + def test_td64arr_add_timestamp(self, box, tz_naive_fixture): + # TODO: parametrize over scalar datetime types? + tz = tz_naive_fixture + other = Timestamp('2011-01-01', tz=tz) - def test_td64_radd_timestamp(self, box): idx = TimedeltaIndex(['1 day', '2 day']) - expected = DatetimeIndex(['2011-01-02', '2011-01-03']) + expected = DatetimeIndex(['2011-01-02', '2011-01-03'], tz=tz) idx = tm.box_expected(idx, box) expected = tm.box_expected(expected, box) - # TODO: parametrize over scalar datetime types? - result = Timestamp('2011-01-01') + idx + result = idx + other + tm.assert_equal(result, expected) + result = other + idx tm.assert_equal(result, expected) def test_td64arr_add_sub_timestamp(self, box): @@ -494,6 +489,18 @@ def test_tdi_add_dt64_array(self, box_df_broadcast_failure): result = dtarr + tdi tm.assert_equal(result, expected) + def test_td64arr_add_datetime64_nat(self, box): + other = np.datetime64('NaT') + + tdi = timedelta_range('1 day', periods=3) + expected = pd.DatetimeIndex(["NaT", "NaT", "NaT"]) + + tdser = tm.box_expected(tdi, box) + expected = tm.box_expected(expected, box) + + tm.assert_equal(tdser + other, expected) + tm.assert_equal(other + tdser, expected) + # ------------------------------------------------------------------ # Operations with int-like others @@ -771,18 +778,6 @@ def test_td64arr_sub_timedeltalike(self, two_hours, box): result = rng - two_hours tm.assert_equal(result, expected) - def test_td64arr_add_datetime64_nat(self, box): - other = np.datetime64('NaT') - - tdi = timedelta_range('1 day', periods=3) - expected = pd.DatetimeIndex(["NaT", "NaT", "NaT"]) - - tdser = tm.box_expected(tdi, box) - expected = tm.box_expected(expected, box) - - tm.assert_equal(tdser + other, expected) - tm.assert_equal(other + tdser, expected) - # ------------------------------------------------------------------ # __add__/__sub__ with DateOffsets and arrays of DateOffsets From 982ea3081094f10dc982714307db13a56dd050e3 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 17 Oct 2018 16:07:23 -0700 Subject: [PATCH 06/30] remove comment --- pandas/core/arrays/timedeltas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index b38cf017da9a5..3824945b2988f 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -223,7 +223,7 @@ def _add_datelike(self, other): result = checked_add_with_arr(i8, other.value, arr_mask=self._isnan) result = self._maybe_mask_results(result, fill_value=iNaT) - return DatetimeArrayMixin(result, tz=other.tz) # FIXME: what about timezone? + return DatetimeArrayMixin(result, tz=other.tz) def _addsub_offset_array(self, other, op): # Add or subtract Array-like of DateOffset objects From d046038b40f9a72131f9830e2e823ee7b6e606d3 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 17 Oct 2018 16:11:46 -0700 Subject: [PATCH 07/30] Add GH references --- doc/source/whatsnew/v0.24.0.txt | 4 ++-- pandas/tests/arithmetic/test_period.py | 1 + pandas/tests/arithmetic/test_timedelta64.py | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 5f99434658b7c..94bc075265e45 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -825,8 +825,8 @@ Timedelta - Bug in :class:`TimedeltaIndex` incorrectly allowing indexing with ``Timestamp`` object (:issue:`20464`) - Fixed bug where subtracting :class:`Timedelta` from an object-dtyped array would raise ``TypeError`` (:issue:`21980`) - Fixed bug in adding a :class:`DataFrame` with all-`timedelta64[ns]` dtypes to a :class:`DataFrame` with all-integer dtypes returning incorrect results instead of raising ``TypeError`` (:issue:`22696`) -- Bug in :class:`TimedeltaIndex` where adding a timezone-aware datetime scalar incorrectly returned a timezone-naive :class:`DatetimeIndex` (:issue:`?????`) -- Bug in :class:`TimedeltaIndex` where adding ``np.timedelta64('NaT')`` incorrectly returned an all-`NaT` :class:`DatetimeIndex` instead of an all-`NaT` :class:`TimedeltaIndex` (:issue:`?????`) +- Bug in :class:`TimedeltaIndex` where adding a timezone-aware datetime scalar incorrectly returned a timezone-naive :class:`DatetimeIndex` (:issue:`23215`) +- Bug in :class:`TimedeltaIndex` where adding ``np.timedelta64('NaT')`` incorrectly returned an all-`NaT` :class:`DatetimeIndex` instead of an all-`NaT` :class:`TimedeltaIndex` (:issue:`23215`) Timezones ^^^^^^^^^ diff --git a/pandas/tests/arithmetic/test_period.py b/pandas/tests/arithmetic/test_period.py index d564b29fdff80..ba9c73795bc27 100644 --- a/pandas/tests/arithmetic/test_period.py +++ b/pandas/tests/arithmetic/test_period.py @@ -415,6 +415,7 @@ def test_pi_add_sub_float(self, op, other): pd.Timestamp.now().to_pydatetime(), pd.Timestamp.now().to_datetime64()]) def test_pi_add_sub_datetime(self, other): + # GH#23215 rng = pd.period_range('1/1/2000', freq='D', periods=3) with pytest.raises(TypeError): diff --git a/pandas/tests/arithmetic/test_timedelta64.py b/pandas/tests/arithmetic/test_timedelta64.py index acc434a5a2176..795f44cdc91d0 100644 --- a/pandas/tests/arithmetic/test_timedelta64.py +++ b/pandas/tests/arithmetic/test_timedelta64.py @@ -416,6 +416,7 @@ def test_td64arr_sub_timestamp_raises(self, box): idx - Timestamp('2011-01-01') def test_td64arr_add_timestamp(self, box, tz_naive_fixture): + # GH#23215 # TODO: parametrize over scalar datetime types? tz = tz_naive_fixture other = Timestamp('2011-01-01', tz=tz) @@ -490,6 +491,7 @@ def test_tdi_add_dt64_array(self, box_df_broadcast_failure): tm.assert_equal(result, expected) def test_td64arr_add_datetime64_nat(self, box): + # GH#23215 other = np.datetime64('NaT') tdi = timedelta_range('1 day', periods=3) From a0c1a851391b5d756435e3be49ed62514620b396 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Wed, 17 Oct 2018 18:28:27 -0700 Subject: [PATCH 08/30] remove unused imports --- pandas/tests/indexes/datetimes/test_ops.py | 1 - pandas/tests/indexes/timedeltas/test_ops.py | 1 - 2 files changed, 2 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_ops.py b/pandas/tests/indexes/datetimes/test_ops.py index 771165eafec7a..c27207388fd6b 100644 --- a/pandas/tests/indexes/datetimes/test_ops.py +++ b/pandas/tests/indexes/datetimes/test_ops.py @@ -4,7 +4,6 @@ from datetime import datetime import pandas as pd -import pandas._libs.tslib as tslib import pandas.util.testing as tm from pandas import (DatetimeIndex, PeriodIndex, Series, Timestamp, date_range, bdate_range, Index) diff --git a/pandas/tests/indexes/timedeltas/test_ops.py b/pandas/tests/indexes/timedeltas/test_ops.py index 8af32f1f9421b..16d70d79c217e 100644 --- a/pandas/tests/indexes/timedeltas/test_ops.py +++ b/pandas/tests/indexes/timedeltas/test_ops.py @@ -7,7 +7,6 @@ import pandas.util.testing as tm from pandas import (Series, Timedelta, Timestamp, TimedeltaIndex, timedelta_range, to_timedelta) -from pandas._libs.tslib import iNaT from pandas.tests.test_base import Ops from pandas.tseries.offsets import Day, Hour from pandas.core.dtypes.generic import ABCDateOffset From b932121998f208776b3171b8d1ce713b65b6b074 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 18 Oct 2018 15:20:29 -0700 Subject: [PATCH 09/30] dummy commit to force CI From 9f3b18d118d549a9343759d1ad59992207437aaf Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 18 Oct 2018 19:23:36 -0700 Subject: [PATCH 10/30] Fix bug in adding DateOffset to PeriodIndex, Series, Frame --- doc/source/whatsnew/v0.24.0.txt | 3 ++- pandas/_libs/tslibs/offsets.pyx | 4 ++-- pandas/core/arrays/period.py | 7 +++++- pandas/tests/arithmetic/test_period.py | 33 ++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 2dd6076f67b89..55243fb6e049f 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -821,6 +821,7 @@ Datetimelike - Bug in rounding methods of :class:`DatetimeIndex` (:meth:`~DatetimeIndex.round`, :meth:`~DatetimeIndex.ceil`, :meth:`~DatetimeIndex.floor`) and :class:`Timestamp` (:meth:`~Timestamp.round`, :meth:`~Timestamp.ceil`, :meth:`~Timestamp.floor`) could give rise to loss of precision (:issue:`22591`) - Bug in :func:`to_datetime` with an :class:`Index` argument that would drop the ``name`` from the result (:issue:`21697`) - Bug in :class:`PeriodIndex` where adding or subtracting a :class:`timedelta` or :class:`Tick` object produced incorrect results (:issue:`22988`) +- Bug in :class:`PeriodIndex` with attribute ``freq.n`` greater than 1 where adding a :class:`DateOffset` object would return incorrect results (:issue:`23215`) Timedelta ^^^^^^^^^ @@ -863,7 +864,7 @@ Offsets - Bug in :class:`FY5253` where date offsets could incorrectly raise an ``AssertionError`` in arithmetic operatons (:issue:`14774`) - Bug in :class:`DateOffset` where keyword arguments ``week`` and ``milliseconds`` were accepted and ignored. Passing these will now raise ``ValueError`` (:issue:`19398`) -- +- Bug in adding :class:`DateOffset` with :class:`DataFrame` or :class:`PeriodIndex` incorrectly raising ``TypeError`` (:issue:`23215`) Numeric ^^^^^^^ diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx index 393c2cdba8568..5baacfe5f725f 100644 --- a/pandas/_libs/tslibs/offsets.pyx +++ b/pandas/_libs/tslibs/offsets.pyx @@ -344,8 +344,8 @@ class _BaseOffset(object): return {name: kwds[name] for name in kwds if kwds[name] is not None} def __add__(self, other): - if getattr(other, "_typ", None) in ["datetimeindex", - "series", "period"]: + if getattr(other, "_typ", None) in ["datetimeindex", "periodindex", + "series", "period", "dataframe"]: # defer to the other class's implementation return other + self try: diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 8f840a82c8c8f..1d7b57801dad2 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -354,7 +354,12 @@ def _add_offset(self, other): if base != self.freq.rule_code: msg = DIFFERENT_FREQ_INDEX.format(self.freqstr, other.freqstr) raise IncompatibleFrequency(msg) - return self._time_shift(other.n) + + # Note: when calling parent class's _add_delta_td, it will call + # delta_to_nanoseconds(delta). Because delta here is an integer, + # delta_to_nanoseconds will return it unchanged. + result = DatetimeLikeArrayMixin._add_delta_td(self, other.n) + return self._shallow_copy(result) def _add_delta_td(self, other): assert isinstance(self.freq, Tick) # checked by calling function diff --git a/pandas/tests/arithmetic/test_period.py b/pandas/tests/arithmetic/test_period.py index ba9c73795bc27..516386c9f92d6 100644 --- a/pandas/tests/arithmetic/test_period.py +++ b/pandas/tests/arithmetic/test_period.py @@ -16,6 +16,7 @@ import pandas.core.indexes.period as period from pandas.core import ops from pandas import Period, PeriodIndex, period_range, Series +from pandas.tseries.frequencies import to_offset # ------------------------------------------------------------------ @@ -591,6 +592,38 @@ def test_pi_sub_isub_offset(self): rng -= pd.offsets.MonthEnd(5) tm.assert_index_equal(rng, expected) + def test_pi_add_offset_n_gt1(self, box): + # GH#23215 + # add offset to PeriodIndex with freq.n > 1 + per = pd.Period('2016-01', freq='2M') + pi = pd.PeriodIndex([per]) + + expected = pd.PeriodIndex(['2016-03'], freq='2M') + pi = tm.box_expected(pi, box) + expected = tm.box_expected(expected, box) + + result = pi + per.freq + tm.assert_equal(result, expected) + + result = per.freq + pi + tm.assert_equal(result, expected) + + def test_pi_add_offset_n_gt1_not_divisible(self, box): + # GH#23215 + # PeriodIndex with freq.n > 1 add offset with offset.n % freq.n != 0 + + pi = pd.PeriodIndex(['2016-01'], freq='2M') + pi = tm.box_expected(pi, box) + + expected = pd.PeriodIndex(['2016-04'], freq='2M') + expected = tm.box_expected(expected, box) + + result = pi + to_offset('3M') + tm.assert_equal(result, expected) + + result = to_offset('3M') + pi + tm.assert_equal(result, expected) + # --------------------------------------------------------------- # __add__/__sub__ with integer arrays From 7a8232e8ca0a441353194114498057e4e91bea8b Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 18 Oct 2018 22:09:01 -0700 Subject: [PATCH 11/30] Dummy commit to force CI --- pandas/core/arrays/datetimelike.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 00f7ff9e4f0cf..ecfd91f4a9a97 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -351,12 +351,12 @@ def _add_offset(self, offset): def _add_delta(self, other): """ - Add a timedelta-like, DateOffset, or TimedeltaIndex-like object + Add a timedelta-like, Tick or TimedeltaIndex-like object to self. Parameters ---------- - delta : {timedelta, np.timedelta64, DateOffset, + delta : {timedelta, np.timedelta64, Tick, TimedeltaIndex, ndarray[timedelta64]} Returns From a743f74a4d11cc1713c323246229e96f0ca70697 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 19 Oct 2018 09:16:48 -0700 Subject: [PATCH 12/30] revert tracebackhide --- pandas/util/testing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/util/testing.py b/pandas/util/testing.py index 82be5edc73b55..1bd9043f42634 100644 --- a/pandas/util/testing.py +++ b/pandas/util/testing.py @@ -934,7 +934,6 @@ def assert_attr_equal(attr, left, right, obj='Attributes'): Specify object name being compared, internally used to show appropriate assertion message """ - __tracebackhide__ = True left_attr = getattr(left, attr) right_attr = getattr(right, attr) From af4872ea9b472684d76621e6779bd6b3104af8cc Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Fri, 19 Oct 2018 14:24:31 -0700 Subject: [PATCH 13/30] comment and reversion --- pandas/core/arrays/datetimelike.py | 2 +- pandas/core/arrays/datetimes.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index ecfd91f4a9a97..0727bceaa6f7b 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -352,7 +352,7 @@ def _add_offset(self, offset): def _add_delta(self, other): """ Add a timedelta-like, Tick or TimedeltaIndex-like object - to self. + to self, yielding another array of the same type Parameters ---------- diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 786354258aa6e..cf98d2043dfc7 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -487,7 +487,15 @@ def _sub_datelike(self, other): @Appender(dtl.DatetimeLikeArrayMixin._add_delta.__doc__) def _add_delta(self, delta): new_values = dtl.DatetimeLikeArrayMixin._add_delta(self, delta) - return type(self)(new_values, tz=self.tz, freq='infer') + + # Note: this construction is _not_ equivalent to + # type(self)(new_values, tz=self.tz, freq'infer'') + # see GH#23215 + tz = 'UTC' if self.tz is not None else None + result = type(self)(new_values, tz=tz, freq='infer') + if self.tz is not None and self.tz is not utc: + result = result.tz_convert(self.tz) + return result # ----------------------------------------------------------------- # Timezone Conversion and Localization Methods From 6707032f9773e4897a284725f5c76b1901dcf25a Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 20 Oct 2018 09:41:28 -0700 Subject: [PATCH 14/30] revert to simpler version --- pandas/core/arrays/datetimes.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index cf98d2043dfc7..786354258aa6e 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -487,15 +487,7 @@ def _sub_datelike(self, other): @Appender(dtl.DatetimeLikeArrayMixin._add_delta.__doc__) def _add_delta(self, delta): new_values = dtl.DatetimeLikeArrayMixin._add_delta(self, delta) - - # Note: this construction is _not_ equivalent to - # type(self)(new_values, tz=self.tz, freq'infer'') - # see GH#23215 - tz = 'UTC' if self.tz is not None else None - result = type(self)(new_values, tz=tz, freq='infer') - if self.tz is not None and self.tz is not utc: - result = result.tz_convert(self.tz) - return result + return type(self)(new_values, tz=self.tz, freq='infer') # ----------------------------------------------------------------- # Timezone Conversion and Localization Methods From 18ef26d59282736515b8d246fe2ff534e011c8a4 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 20 Oct 2018 09:45:01 -0700 Subject: [PATCH 15/30] Move overriding of addsub_int_array to PeriodArray --- pandas/core/arrays/datetimelike.py | 14 +++----------- pandas/core/arrays/period.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 0727bceaa6f7b..fda33755f5afe 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -491,18 +491,10 @@ def _addsub_int_array(self, other, op): ------- result : same class as self """ + assert not is_period_dtype(self) # overriden by PeriodArray assert op in [operator.add, operator.sub] - if is_period_dtype(self): - # easy case for PeriodIndex - if op is operator.sub: - other = -other - res_values = checked_add_with_arr(self.asi8, other, - arr_mask=self._isnan) - res_values = res_values.view('i8') - res_values[self._isnan] = iNaT - return self._from_ordinals(res_values, freq=self.freq) - - elif self.freq is None: + + if self.freq is None: # GH#19123 raise NullFrequencyError("Cannot shift with no freq") diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 1d7b57801dad2..e2c90c80aa480 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -24,6 +24,7 @@ from pandas.core.dtypes.generic import ABCSeries import pandas.core.common as com +from pandas.core.algorithms import checked_add_with_arr from pandas.tseries import frequencies from pandas.tseries.offsets import Tick, DateOffset @@ -334,6 +335,17 @@ def to_timestamp(self, freq=None, how='start'): _create_comparison_method = classmethod(_period_array_cmp) + @Appender(DatetimeLikeArrayMixin._addsub_int_array.__doc__) + def _addsub_int_array(self, other, op): + assert op in [operator.add, operator.sub] + if op is operator.sub: + other = -other + res_values = checked_add_with_arr(self.asi8, other, + arr_mask=self._isnan) + res_values = res_values.view('i8') + res_values[self._isnan] = iNaT + return self._simple_new(res_values, freq=self.freq) + def _sub_period(self, other): # If the operation is well-defined, we return an object-Index # of DateOffsets. Null entries are filled with pd.NaT From 937242390b042d56a6fcc73f942eff625b1f6e2e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 22 Oct 2018 19:14:43 -0700 Subject: [PATCH 16/30] correct docstrings --- pandas/core/arrays/datetimelike.py | 6 +----- pandas/core/arrays/datetimes.py | 3 ++- pandas/core/arrays/period.py | 3 ++- pandas/core/arrays/timedeltas.py | 3 ++- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index fda33755f5afe..ec982dee0c70d 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -361,17 +361,13 @@ def _add_delta(self, other): Returns ------- - result : same type as self + result : ndarray[int64] Notes ----- The result's name is set outside of _add_delta by the calling method (__add__ or __sub__), if necessary (i.e. for Indexes). """ - # Note: The docstring here says the return type is the same type - # as self, which is inaccurate. Once wrapped by the inheriting - # Array classes, this will be accurate. - if isinstance(other, (Tick, timedelta, np.timedelta64)): new_values = self._add_delta_td(other) elif is_timedelta64_dtype(other): diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 6e7fc10d529be..9b862ce472d21 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -484,7 +484,8 @@ def _sub_datelike(self, other): result = self._maybe_mask_results(result, fill_value=iNaT) return result.view('timedelta64[ns]') - @Appender(dtl.DatetimeLikeArrayMixin._add_delta.__doc__) + @Appender(dtl.DatetimeLikeArrayMixin._add_delta.__doc__.replace( + "ndarray[int64]", "same type as self")) def _add_delta(self, delta): new_values = dtl.DatetimeLikeArrayMixin._add_delta(self, delta) return type(self)(new_values, tz=self.tz, freq='infer') diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index e2c90c80aa480..f9d4396281c4d 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -390,7 +390,8 @@ def _add_delta_tdi(self, other): delta = self._check_timedeltalike_freq_compat(other) return self._addsub_int_array(delta, operator.add) - @Appender(DatetimeLikeArrayMixin._add_delta.__doc__) + @Appender(DatetimeLikeArrayMixin._add_delta.__doc__.replace( + "ndarray[int64]", "same type as self")) def _add_delta(self, other): if not isinstance(self.freq, Tick): # We cannot add timedelta-like to non-tick PeriodArray diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 3824945b2988f..3a57f0a4d9fdb 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -192,7 +192,8 @@ def _add_offset(self, other): .format(typ=type(other).__name__, cls=type(self).__name__)) - @Appender(dtl.DatetimeLikeArrayMixin._add_delta.__doc__) + @Appender(dtl.DatetimeLikeArrayMixin._add_delta.__doc__.replace( + "ndarray[int64]", "same type as self")) def _add_delta(self, delta): new_values = dtl.DatetimeLikeArrayMixin._add_delta(self, delta) return type(self)(new_values, freq='infer') From 2a6268e5834d0f72d659de919448927efe8d7319 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 22 Oct 2018 19:19:56 -0700 Subject: [PATCH 17/30] comments and assertions --- pandas/core/arrays/datetimelike.py | 3 ++- pandas/core/arrays/datetimes.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index ec982dee0c70d..5dd4ca6145e6e 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -487,7 +487,8 @@ def _addsub_int_array(self, other, op): ------- result : same class as self """ - assert not is_period_dtype(self) # overriden by PeriodArray + # _addsub_int_array is overriden by PeriodArray + assert not is_period_dtype(self) assert op in [operator.add, operator.sub] if self.freq is None: diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 9b862ce472d21..fc4cdf631a9fb 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -429,7 +429,7 @@ def _sub_datelike_dti(self, other): raise ValueError("cannot add indices of unequal length") if isinstance(other, np.ndarray): - # if other is an ndarray, we assume it is datetime64-dtype + assert is_datetime64_dtype(other) other = type(self)(other) if not self._has_same_tz(other): From 777f4d90d735248518f8024ab2d7cbd72dd7f039 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 22 Oct 2018 19:23:23 -0700 Subject: [PATCH 18/30] More explicit names for array/scalar add/sub methods --- pandas/core/arrays/datetimelike.py | 16 ++++++++-------- pandas/core/arrays/datetimes.py | 4 ++-- pandas/core/arrays/timedeltas.py | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 5dd4ca6145e6e..d50f964d793a4 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -325,21 +325,21 @@ def _validate_frequency(cls, index, freq, **kwargs): # ------------------------------------------------------------------ # Arithmetic Methods - def _add_datelike(self, other): + def _add_datetimelike_scalar(self, other): # Overriden by TimedeltaArray raise TypeError("cannot add {cls} and {typ}" .format(cls=type(self).__name__, typ=type(other).__name__)) - _add_datelike_dti = _add_datelike + _add_datetime_arraylike = _add_datetimelike_scalar - def _sub_datelike(self, other): + def _sub_datetimelike_scalar(self, other): # Overridden by DatetimeArray assert other is not NaT raise TypeError("cannot subtract a datelike from a {cls}" .format(cls=type(self).__name__)) - _sub_datelike_dti = _sub_datelike + _sub_datetime_arraylike = _sub_datetimelike_scalar def _sub_period(self, other): # Overriden by PeriodArray @@ -627,7 +627,7 @@ def __add__(self, other): # specifically _not_ a Tick result = self._add_offset(other) elif isinstance(other, (datetime, np.datetime64)): - result = self._add_datelike(other) + result = self._add_datetimelike_scalar(other) elif lib.is_integer(other): # This check must come after the check for np.timedelta64 # as is_integer returns True for these @@ -642,7 +642,7 @@ def __add__(self, other): result = self._addsub_offset_array(other, operator.add) elif is_datetime64_dtype(other) or is_datetime64tz_dtype(other): # DatetimeIndex, ndarray[datetime64] - return self._add_datelike_dti(other) + return self._add_datetime_arraylike(other) elif is_integer_dtype(other): result = self._addsub_int_array(other, operator.add) elif is_float_dtype(other): @@ -685,7 +685,7 @@ def __sub__(self, other): # specifically _not_ a Tick result = self._add_offset(-other) elif isinstance(other, (datetime, np.datetime64)): - result = self._sub_datelike(other) + result = self._sub_datetimelike_scalar(other) elif lib.is_integer(other): # This check must come after the check for np.timedelta64 # as is_integer returns True for these @@ -702,7 +702,7 @@ def __sub__(self, other): result = self._addsub_offset_array(other, operator.sub) elif is_datetime64_dtype(other) or is_datetime64tz_dtype(other): # DatetimeIndex, ndarray[datetime64] - result = self._sub_datelike_dti(other) + result = self._sub_datetime_arraylike(other) elif is_period_dtype(other): # PeriodIndex result = self._sub_period_array(other) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index fc4cdf631a9fb..fe372d475b7de 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -423,7 +423,7 @@ def _assert_tzawareness_compat(self, other): # ----------------------------------------------------------------- # Arithmetic Methods - def _sub_datelike_dti(self, other): + def _sub_datetime_arraylike(self, other): """subtract DatetimeArray/Index or ndarray[datetime64]""" if len(self) != len(other): raise ValueError("cannot add indices of unequal length") @@ -465,7 +465,7 @@ def _add_offset(self, offset): return type(self)(result, freq='infer') - def _sub_datelike(self, other): + def _sub_datetimelike_scalar(self, other): # subtract a datetime from myself, yielding a ndarray[timedelta64[ns]] assert isinstance(other, (datetime, np.datetime64)) assert other is not NaT diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 3a57f0a4d9fdb..a4b81d362b92b 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -198,7 +198,7 @@ def _add_delta(self, delta): new_values = dtl.DatetimeLikeArrayMixin._add_delta(self, delta) return type(self)(new_values, freq='infer') - def _add_datelike_dti(self, other): + def _add_datetime_arraylike(self, other): """Add DatetimeArray/Index or ndarray[datetime64] to TimedeltaArray""" if isinstance(other, np.ndarray): # At this point we have already checked that dtype is datetime64 @@ -208,7 +208,7 @@ def _add_datelike_dti(self, other): # defer to implementation in DatetimeArray return other + self - def _add_datelike(self, other): + def _add_datetimelike_scalar(self, other): # adding a timedeltaindex to a datetimelike from pandas.core.arrays import DatetimeArrayMixin From 5f231b287a7aa692b05ccbf29bf9b37c20ebb5aa Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Mon, 22 Oct 2018 20:04:51 -0700 Subject: [PATCH 19/30] oo optimization fixup --- pandas/core/arrays/datetimes.py | 2 +- pandas/core/arrays/period.py | 2 +- pandas/core/arrays/timedeltas.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index 5fd40979c53b6..19cbbf53cec9d 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -484,7 +484,7 @@ def _sub_datetimelike_scalar(self, other): result = self._maybe_mask_results(result, fill_value=iNaT) return result.view('timedelta64[ns]') - @Appender(dtl.DatetimeLikeArrayMixin._add_delta.__doc__.replace( + @Appender((dtl.DatetimeLikeArrayMixin._add_delta.__doc__ or "").replace( "ndarray[int64]", "same type as self")) def _add_delta(self, delta): new_values = dtl.DatetimeLikeArrayMixin._add_delta(self, delta) diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index f9d4396281c4d..41184a58c2396 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -390,7 +390,7 @@ def _add_delta_tdi(self, other): delta = self._check_timedeltalike_freq_compat(other) return self._addsub_int_array(delta, operator.add) - @Appender(DatetimeLikeArrayMixin._add_delta.__doc__.replace( + @Appender((DatetimeLikeArrayMixin._add_delta.__doc__ or "").replace( "ndarray[int64]", "same type as self")) def _add_delta(self, other): if not isinstance(self.freq, Tick): diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 49a67ca6cbe56..985e4e42f7fe9 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -192,7 +192,7 @@ def _add_offset(self, other): .format(typ=type(other).__name__, cls=type(self).__name__)) - @Appender(dtl.DatetimeLikeArrayMixin._add_delta.__doc__.replace( + @Appender((dtl.DatetimeLikeArrayMixin._add_delta.__doc__ or "").replace( "ndarray[int64]", "same type as self")) def _add_delta(self, delta): new_values = dtl.DatetimeLikeArrayMixin._add_delta(self, delta) From d799d8e46257b4e7d9fdae1ad9da203ea6c474e8 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Wed, 24 Oct 2018 22:15:28 -0700 Subject: [PATCH 20/30] Dummy commit to force CI --- pandas/tests/indexes/datetimes/test_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/indexes/datetimes/test_ops.py b/pandas/tests/indexes/datetimes/test_ops.py index c27207388fd6b..4db5001b196d8 100644 --- a/pandas/tests/indexes/datetimes/test_ops.py +++ b/pandas/tests/indexes/datetimes/test_ops.py @@ -33,7 +33,7 @@ def test_ops_properties(self): def test_ops_properties_basic(self): # sanity check that the behavior didn't change - # GH7206 + # GH#7206 for op in ['year', 'day', 'second', 'weekday']: pytest.raises(TypeError, lambda x: getattr(self.dt_series, op)) From 8cf614b692edf54f03f6fea2ae5a6713bfe7c464 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 25 Oct 2018 08:34:42 -0700 Subject: [PATCH 21/30] change default fill_value for maybe_mask_results --- pandas/core/arrays/datetimelike.py | 4 ++-- pandas/core/arrays/datetimes.py | 11 ++++++----- pandas/core/arrays/timedeltas.py | 10 ++++++---- pandas/core/indexes/timedeltas.py | 3 ++- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 526796acd1dbf..f38f3b9b2c151 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -221,7 +221,7 @@ def hasnans(self): """ return if I have any nans; enables various perf speedups """ return self._isnan.any() - def _maybe_mask_results(self, result, fill_value=None, convert=None): + def _maybe_mask_results(self, result, fill_value=iNaT, convert=None): """ Parameters ---------- @@ -384,7 +384,7 @@ def _add_delta_td(self, other): inc = delta_to_nanoseconds(other) new_values = checked_add_with_arr(self.asi8, inc, arr_mask=self._isnan).view('i8') - new_values = self._maybe_mask_results(new_values, fill_value=iNaT) + new_values = self._maybe_mask_results(new_values) return new_values.view('i8') def _add_delta_tdi(self, other): diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index cc15aad749459..dff3465d7c97e 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -75,11 +75,12 @@ def f(self): if field in self._object_ops: result = fields.get_date_name_field(values, field) - result = self._maybe_mask_results(result) + result = self._maybe_mask_results(result, fill_value=None) else: result = fields.get_date_field(values, field) - result = self._maybe_mask_results(result, convert='float64') + result = self._maybe_mask_results(result, fill_value=None, + convert='float64') return result @@ -481,7 +482,7 @@ def _sub_datetimelike_scalar(self, other): i8 = self.asi8 result = checked_add_with_arr(i8, -other.value, arr_mask=self._isnan) - result = self._maybe_mask_results(result, fill_value=iNaT) + result = self._maybe_mask_results(result) return result.view('timedelta64[ns]') @Appender((dtl.DatetimeLikeArrayMixin._add_delta.__doc__ or "").replace( @@ -869,7 +870,7 @@ def month_name(self, locale=None): result = fields.get_date_name_field(values, 'month_name', locale=locale) - result = self._maybe_mask_results(result) + result = self._maybe_mask_results(result, fill_value=None) return result def day_name(self, locale=None): @@ -905,7 +906,7 @@ def day_name(self, locale=None): result = fields.get_date_name_field(values, 'day_name', locale=locale) - result = self._maybe_mask_results(result) + result = self._maybe_mask_results(result, fill_value=None) return result @property diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index c81a214e29d0f..611ada7fc2106 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -47,7 +47,8 @@ def f(self): values = self.asi8 result = get_timedelta_field(values, alias) if self.hasnans: - result = self._maybe_mask_results(result, convert='float64') + result = self._maybe_mask_results(result, fill_value=None, + convert='float64') return result @@ -220,7 +221,7 @@ def _add_datetimelike_scalar(self, other): i8 = self.asi8 result = checked_add_with_arr(i8, other.value, arr_mask=self._isnan) - result = self._maybe_mask_results(result, fill_value=iNaT) + result = self._maybe_mask_results(result) return DatetimeArrayMixin(result, tz=other.tz) def _addsub_offset_array(self, other, op): @@ -256,7 +257,8 @@ def _evaluate_with_timedelta_like(self, other, op): result = op(left, right) else: result = op(left, np.float64(right)) - result = self._maybe_mask_results(result, convert='float64') + result = self._maybe_mask_results(result, fill_value=None, + convert='float64') return result return NotImplemented @@ -319,7 +321,7 @@ def total_seconds(self): Float64Index([0.0, 86400.0, 172800.0, 259200.00000000003, 345600.0], dtype='float64') """ - return self._maybe_mask_results(1e-9 * self.asi8) + return self._maybe_mask_results(1e-9 * self.asi8, fill_value=None) def to_pytimedelta(self): """ diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py index 1efa0a15d34d7..e5da21478d0a4 100644 --- a/pandas/core/indexes/timedeltas.py +++ b/pandas/core/indexes/timedeltas.py @@ -232,7 +232,8 @@ def astype(self, dtype, copy=True): # return an index (essentially this is division) result = self.values.astype(dtype, copy=copy) if self.hasnans: - values = self._maybe_mask_results(result, convert='float64') + values = self._maybe_mask_results(result, fill_value=None, + convert='float64') return Index(values, name=self.name) return Index(result.astype('i8'), name=self.name) return super(TimedeltaIndex, self).astype(dtype, copy=copy) From fb007cba966f8dc7b425e9f38a3c01b03e9b6d2f Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 25 Oct 2018 10:31:10 -0700 Subject: [PATCH 22/30] Make docstring extra explicit --- pandas/core/arrays/datetimelike.py | 2 +- pandas/core/arrays/datetimes.py | 3 ++- pandas/core/arrays/period.py | 3 ++- pandas/core/arrays/timedeltas.py | 5 +++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index f38f3b9b2c151..4e71ffcf581c9 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -352,7 +352,7 @@ def _add_offset(self, offset): def _add_delta(self, other): """ Add a timedelta-like, Tick or TimedeltaIndex-like object - to self, yielding another array of the same type + to self, yielding an int64 numpy array Parameters ---------- diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index dff3465d7c97e..9fa7b5e63f1a1 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -486,7 +486,8 @@ def _sub_datetimelike_scalar(self, other): return result.view('timedelta64[ns]') @Appender((dtl.DatetimeLikeArrayMixin._add_delta.__doc__ or "").replace( - "ndarray[int64]", "same type as self")) + "ndarray[int64]", "same type as self").replace( + "an int64 numpy array", "another array of the same type as self")) def _add_delta(self, delta): new_values = dtl.DatetimeLikeArrayMixin._add_delta(self, delta) return type(self)(new_values, tz=self.tz, freq='infer') diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 3445b35ca134b..3759451c7834f 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -393,7 +393,8 @@ def _add_delta_tdi(self, other): return self._addsub_int_array(delta, operator.add) @Appender((DatetimeLikeArrayMixin._add_delta.__doc__ or "").replace( - "ndarray[int64]", "same type as self")) + "ndarray[int64]", "same type as self").replace( + "an int64 numpy array", "another array of the same type as self")) def _add_delta(self, other): if not isinstance(self.freq, Tick): # We cannot add timedelta-like to non-tick PeriodArray diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index 611ada7fc2106..e11a1e260d0ee 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -4,7 +4,7 @@ import numpy as np from pandas._libs import tslibs -from pandas._libs.tslibs import Timedelta, Timestamp, NaT, iNaT +from pandas._libs.tslibs import Timedelta, Timestamp, NaT from pandas._libs.tslibs.fields import get_timedelta_field from pandas._libs.tslibs.timedeltas import array_to_timedelta64 @@ -191,7 +191,8 @@ def _add_offset(self, other): cls=type(self).__name__)) @Appender((dtl.DatetimeLikeArrayMixin._add_delta.__doc__ or "").replace( - "ndarray[int64]", "same type as self")) + "ndarray[int64]", "same type as self").replace( + "an int64 numpy array", "another array of the same type as self")) def _add_delta(self, delta): new_values = dtl.DatetimeLikeArrayMixin._add_delta(self, delta) return type(self)(new_values, freq='infer') From aecfef7d7a560b35eb1f5a05056fd6d160516c8b Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 25 Oct 2018 16:13:26 -0700 Subject: [PATCH 23/30] prettify docstrings --- pandas/core/arrays/datetimes.py | 18 ++++++++++++++---- pandas/core/arrays/timedeltas.py | 17 +++++++++++++---- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py index f4e3635699baf..83566d046f5cd 100644 --- a/pandas/core/arrays/datetimes.py +++ b/pandas/core/arrays/datetimes.py @@ -12,7 +12,7 @@ conversion, fields, timezones, resolution as libresolution) -from pandas.util._decorators import cache_readonly, Appender +from pandas.util._decorators import cache_readonly from pandas.errors import PerformanceWarning from pandas import compat @@ -485,10 +485,20 @@ def _sub_datetimelike_scalar(self, other): result = self._maybe_mask_results(result) return result.view('timedelta64[ns]') - @Appender((dtl.DatetimeLikeArrayMixin._add_delta.__doc__ or "").replace( - "ndarray[int64]", "same type as self").replace( - "an int64 numpy array", "another array of the same type as self")) def _add_delta(self, delta): + """ + Add a timedelta-like, Tick, or TimedeltaIndex-like object + to self, yielding a new DatetimeArray + + Parameters + ---------- + other : {timedelta, np.timedelta64, Tick, + TimedeltaIndex, ndarray[timedelta64]} + + Returns + ------- + result : DatetimeArray + """ new_values = dtl.DatetimeLikeArrayMixin._add_delta(self, delta) return type(self)(new_values, tz=self.tz, freq='infer') diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py index e11a1e260d0ee..397297c1b88d0 100644 --- a/pandas/core/arrays/timedeltas.py +++ b/pandas/core/arrays/timedeltas.py @@ -8,7 +8,6 @@ from pandas._libs.tslibs.fields import get_timedelta_field from pandas._libs.tslibs.timedeltas import array_to_timedelta64 -from pandas.util._decorators import Appender from pandas import compat from pandas.core.dtypes.common import ( @@ -190,10 +189,20 @@ def _add_offset(self, other): .format(typ=type(other).__name__, cls=type(self).__name__)) - @Appender((dtl.DatetimeLikeArrayMixin._add_delta.__doc__ or "").replace( - "ndarray[int64]", "same type as self").replace( - "an int64 numpy array", "another array of the same type as self")) def _add_delta(self, delta): + """ + Add a timedelta-like, Tick, or TimedeltaIndex-like object + to self, yielding a new TimedeltaArray + + Parameters + ---------- + other : {timedelta, np.timedelta64, Tick, + TimedeltaIndex, ndarray[timedelta64]} + + Returns + ------- + result : TimedeltaArray + """ new_values = dtl.DatetimeLikeArrayMixin._add_delta(self, delta) return type(self)(new_values, freq='infer') From a6eb01cc0f967ebb5fc243d71439a6c3ab6f907e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 25 Oct 2018 18:22:25 -0700 Subject: [PATCH 24/30] fixup super usage --- pandas/core/arrays/period.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 71a8ff6f920cb..9f593b35eef2d 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -739,7 +739,7 @@ def _add_offset(self, other): # Note: when calling parent class's _add_delta_td, it will call # delta_to_nanoseconds(delta). Because delta here is an integer, # delta_to_nanoseconds will return it unchanged. - result = super(PeriodArray, self)._add_delta_td(self, other.n) + result = super(PeriodArray, self)._add_delta_td(other.n) return type(self)(result, freq=self.freq) def _add_delta_td(self, other): @@ -781,7 +781,7 @@ def _add_delta(self, other): .format(cls=type(self).__name__, freqstr=self.freqstr)) - new_ordinals = super(PeriodArray, self)._add_delta(self, other) + new_ordinals = super(PeriodArray, self)._add_delta(other) return type(self)(new_ordinals, freq=self.freq) def _check_timedeltalike_freq_compat(self, other): From acf1f74c5ea9ace096154754b9b4e4fa604746f0 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 25 Oct 2018 18:27:13 -0700 Subject: [PATCH 25/30] flesh out TODO comment --- pandas/core/arrays/period.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 9f593b35eef2d..edc144e0508db 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -494,7 +494,8 @@ def _time_shift(self, n, freq=None): Frequency increment to shift by. """ if freq is not None: - raise NotImplementedError # TODO: ?? + # TODO: don't silently ignore kwarg + raise NotImplementedError values = self.asi8 + n * self.freq.n if self.hasnans: values[self._isnan] = iNaT From d306277163f724009cce355d0852b8c4fd05ee3e Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 25 Oct 2018 20:07:37 -0700 Subject: [PATCH 26/30] move test for moved method --- pandas/tests/arrays/test_period.py | 14 -------------- pandas/tests/indexes/period/test_period.py | 11 +++++++++++ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/pandas/tests/arrays/test_period.py b/pandas/tests/arrays/test_period.py index 780df579d2778..dcbb0d4048b0f 100644 --- a/pandas/tests/arrays/test_period.py +++ b/pandas/tests/arrays/test_period.py @@ -190,17 +190,3 @@ def tet_sub_period(): other = pd.Period("2000", freq="M") with tm.assert_raises_regex(IncompatibleFrequency, "freq"): arr - other - - -# ---------------------------------------------------------------------------- -# other - -def test_maybe_convert_timedelta(): - arr = period_array(['2000', '2001'], freq='D') - offset = pd.tseries.offsets.Day(2) - assert arr._maybe_convert_timedelta(offset) == 2 - assert arr._maybe_convert_timedelta(2) == 2 - - offset = pd.tseries.offsets.BusinessDay() - with tm.assert_raises_regex(ValueError, 'freq'): - arr._maybe_convert_timedelta(offset) diff --git a/pandas/tests/indexes/period/test_period.py b/pandas/tests/indexes/period/test_period.py index 405edba83dc7a..7ce51181d1b0c 100644 --- a/pandas/tests/indexes/period/test_period.py +++ b/pandas/tests/indexes/period/test_period.py @@ -557,3 +557,14 @@ def test_insert(self): for na in (np.nan, pd.NaT, None): result = period_range('2017Q1', periods=4, freq='Q').insert(1, na) tm.assert_index_equal(result, expected) + + +def test_maybe_convert_timedelta(): + pi = PeriodIndex(['2000', '2001'], freq='D') + offset = offsets.Day(2) + assert pi._maybe_convert_timedelta(offset) == 2 + assert pi._maybe_convert_timedelta(2) == 2 + + offset = BusinessDay() + with tm.assert_raises_regex(ValueError, 'freq'): + pi._maybe_convert_timedelta(offset) From f6e4073a378a8d85957d8810b932440505840dee Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 25 Oct 2018 21:57:31 -0700 Subject: [PATCH 27/30] Fixup name --- pandas/tests/indexes/period/test_period.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/indexes/period/test_period.py b/pandas/tests/indexes/period/test_period.py index 7ce51181d1b0c..efa193859e86a 100644 --- a/pandas/tests/indexes/period/test_period.py +++ b/pandas/tests/indexes/period/test_period.py @@ -565,6 +565,6 @@ def test_maybe_convert_timedelta(): assert pi._maybe_convert_timedelta(offset) == 2 assert pi._maybe_convert_timedelta(2) == 2 - offset = BusinessDay() + offset = offsets.BusinessDay() with tm.assert_raises_regex(ValueError, 'freq'): pi._maybe_convert_timedelta(offset) From 4e4b9edaeb3f611f919122f9b990c4162ec5e76f Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 27 Oct 2018 07:38:43 -0700 Subject: [PATCH 28/30] change NotImplementedError to TypeError --- pandas/core/arrays/period.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index edc144e0508db..48a7d0f952fd5 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -494,8 +494,9 @@ def _time_shift(self, n, freq=None): Frequency increment to shift by. """ if freq is not None: - # TODO: don't silently ignore kwarg - raise NotImplementedError + raise TypeError("`freq` argument is not supported for " + "{cls}._time_shift" + .format(cls=type(self).__name__)) values = self.asi8 + n * self.freq.n if self.hasnans: values[self._isnan] = iNaT From f35b3b68681f4453bdf53456769429f12e1c1af3 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 27 Oct 2018 07:54:32 -0700 Subject: [PATCH 29/30] Fix add_delta_tdi return type; docstrings --- pandas/core/arrays/period.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index 48a7d0f952fd5..feadf28150618 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -745,6 +745,15 @@ def _add_offset(self, other): return type(self)(result, freq=self.freq) def _add_delta_td(self, other): + """ + Parameters + ---------- + other : timedelta, Tick, np.timedelta64 + + Returns + ------- + result : ndarray[int64] + """ assert isinstance(self.freq, Tick) # checked by calling function assert isinstance(other, (timedelta, np.timedelta64, Tick)) @@ -757,10 +766,19 @@ def _add_delta_td(self, other): return ordinals def _add_delta_tdi(self, other): + """ + Parameters + ---------- + other : TimedeltaArray or ndarray[timedelta64] + + Returns + ------- + result : ndarray[int64] + """ assert isinstance(self.freq, Tick) # checked by calling function delta = self._check_timedeltalike_freq_compat(other) - return self._addsub_int_array(delta, operator.add) + return self._addsub_int_array(delta, operator.add).asi8 def _add_delta(self, other): """ From 0466b9c8efd2f38f61a79fdf90464ee71ae72750 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 27 Oct 2018 22:22:55 -0700 Subject: [PATCH 30/30] docstring edit, rename --- pandas/core/arrays/datetimelike.py | 5 +++-- pandas/core/arrays/period.py | 18 +++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py index 93325e80b7019..0247ce8dc6ac4 100644 --- a/pandas/core/arrays/datetimelike.py +++ b/pandas/core/arrays/datetimelike.py @@ -226,6 +226,7 @@ def _maybe_mask_results(self, result, fill_value=iNaT, convert=None): Parameters ---------- result : a ndarray + fill_value : object, default iNaT convert : string/dtype or None Returns @@ -369,14 +370,14 @@ def _add_delta(self, other): method (__add__ or __sub__), if necessary (i.e. for Indexes). """ if isinstance(other, (Tick, timedelta, np.timedelta64)): - new_values = self._add_delta_td(other) + new_values = self._add_timedeltalike_scalar(other) elif is_timedelta64_dtype(other): # ndarray[timedelta64] or TimedeltaArray/index new_values = self._add_delta_tdi(other) return new_values - def _add_delta_td(self, other): + def _add_timedeltalike_scalar(self, other): """ Add a delta of a timedeltalike return the i8 result view diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py index feadf28150618..31bcac2f4f529 100644 --- a/pandas/core/arrays/period.py +++ b/pandas/core/arrays/period.py @@ -738,13 +738,13 @@ def _add_offset(self, other): msg = DIFFERENT_FREQ_INDEX.format(self.freqstr, other.freqstr) raise IncompatibleFrequency(msg) - # Note: when calling parent class's _add_delta_td, it will call - # delta_to_nanoseconds(delta). Because delta here is an integer, - # delta_to_nanoseconds will return it unchanged. - result = super(PeriodArray, self)._add_delta_td(other.n) + # Note: when calling parent class's _add_timedeltalike_scalar, + # it will call delta_to_nanoseconds(delta). Because delta here + # is an integer, delta_to_nanoseconds will return it unchanged. + result = super(PeriodArray, self)._add_timedeltalike_scalar(other.n) return type(self)(result, freq=self.freq) - def _add_delta_td(self, other): + def _add_timedeltalike_scalar(self, other): """ Parameters ---------- @@ -759,10 +759,10 @@ def _add_delta_td(self, other): delta = self._check_timedeltalike_freq_compat(other) - # Note: when calling parent class's _add_delta_td, it will call - # delta_to_nanoseconds(delta). Because delta here is an integer, - # delta_to_nanoseconds will return it unchanged. - ordinals = super(PeriodArray, self)._add_delta_td(delta) + # Note: when calling parent class's _add_timedeltalike_scalar, + # it will call delta_to_nanoseconds(delta). Because delta here + # is an integer, delta_to_nanoseconds will return it unchanged. + ordinals = super(PeriodArray, self)._add_timedeltalike_scalar(delta) return ordinals def _add_delta_tdi(self, other):