Merge branch 'master' into na_option_handle

doc/source/whatsnew/v0.24.0.txt

@@ -281,6 +281,43 @@ that the dates have been converted to UTC
 .. ipython:: python
     pd.to_datetime(["2015-11-18 15:30:00+05:30", "2015-11-18 16:30:00+06:30"], utc=True)
 
+.. _whatsnew_0240.api_breaking.period_end_time:
+
+Time values in ``dt.end_time`` and ``to_timestamp(how='end')``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The time values in :class:`Period` and :class:`PeriodIndex` objects are now set
+to '23:59:59.999999999' when calling :attr:`Series.dt.end_time`, :attr:`Period.end_time`,
+:attr:`PeriodIndex.end_time`, :func:`Period.to_timestamp()` with ``how='end'``,
+or :func:`PeriodIndex.to_timestamp()` with ``how='end'`` (:issue:`17157`)
+
+Previous Behavior:
+
+.. code-block:: ipython
+
+   In [2]: p = pd.Period('2017-01-01', 'D')
+   In [3]: pi = pd.PeriodIndex([p])
+
+   In [4]: pd.Series(pi).dt.end_time[0]
+   Out[4]: Timestamp(2017-01-01 00:00:00)
+
+   In [5]: p.end_time
+   Out[5]: Timestamp(2017-01-01 23:59:59.999999999)
+
+Current Behavior:
+
+Calling :attr:`Series.dt.end_time` will now result in a time of '23:59:59.999999999' as
+is the case with :attr:`Period.end_time`, for example
+
+.. ipython:: python
+
+   p = pd.Period('2017-01-01', 'D')
+   pi = pd.PeriodIndex([p])
+
+   pd.Series(pi).dt.end_time[0]
+
+   p.end_time
+
 .. _whatsnew_0240.api.datetimelike.normalize:
 
 Tick DateOffset Normalize Restrictions
@@ -615,6 +652,7 @@ Reshaping
 - Bug in :meth:`Series.combine_first` with ``datetime64[ns, tz]`` dtype which would return tz-naive result (:issue:`21469`)
 - Bug in :meth:`Series.where` and :meth:`DataFrame.where` with ``datetime64[ns, tz]`` dtype (:issue:`21546`)
 - Bug in :meth:`Series.mask` and :meth:`DataFrame.mask` with ``list`` conditionals (:issue:`21891`)
+- Bug in :meth:`DataFrame.replace` raises RecursionError when converting OutOfBounds ``datetime64[ns, tz]`` (:issue:`20380`)
 - :func:`pandas.core.groupby.GroupBy.rank` now raises a ``ValueError`` when an invalid value is passed for argument ``na_option`` (:issue:`22124`)
 -
 

pandas/_libs/algos.pyx

@@ -129,7 +129,7 @@ def is_lexsorted(list list_of_arrays):
     for i in range(nlevels):
         arr = list_of_arrays[i]
         assert arr.dtype.name == 'int64'
-        vecs[i] = <int64_t*> arr.data
+        vecs[i] = <int64_t*> cnp.PyArray_DATA(arr)
 
     # Assume uniqueness??
     with nogil:

pandas/_libs/groupby.pyx

@@ -7,10 +7,12 @@ from cython cimport Py_ssize_t
 from libc.stdlib cimport malloc, free
 
 import numpy as np
+cimport numpy as cnp
 from numpy cimport (ndarray,
                     double_t,
                     int8_t, int16_t, int32_t, int64_t, uint8_t, uint16_t,
                     uint32_t, uint64_t, float32_t, float64_t)
+cnp.import_array()
 
 
 from util cimport numeric, get_nat
@@ -118,7 +120,7 @@ def group_median_float64(ndarray[float64_t, ndim=2] out,
     counts[:] = _counts[1:]
 
     data = np.empty((K, N), dtype=np.float64)
-    ptr = <float64_t*> data.data
+    ptr = <float64_t*> cnp.PyArray_DATA(data)
 
     take_2d_axis1_float64_float64(values.T, indexer, out=data)
 

pandas/_libs/index.pyx

@@ -37,7 +37,7 @@ cdef inline bint is_definitely_invalid_key(object val):
             return True
 
     # we have a _data, means we are a NDFrame
-    return (PySlice_Check(val) or cnp.PyArray_Check(val)
+    return (PySlice_Check(val) or util.is_array(val)
             or PyList_Check(val) or hasattr(val, '_data'))
 
 
@@ -104,7 +104,7 @@ cdef class IndexEngine:
             void* data_ptr
 
         loc = self.get_loc(key)
-        if PySlice_Check(loc) or cnp.PyArray_Check(loc):
+        if PySlice_Check(loc) or util.is_array(loc):
             return arr[loc]
         else:
             return get_value_at(arr, loc, tz=tz)
@@ -120,7 +120,7 @@ cdef class IndexEngine:
         loc = self.get_loc(key)
         value = convert_scalar(arr, value)
 
-        if PySlice_Check(loc) or cnp.PyArray_Check(loc):
+        if PySlice_Check(loc) or util.is_array(loc):
             arr[loc] = value
         else:
             util.set_value_at(arr, loc, value)

pandas/_libs/src/numpy_helper.h

@@ -16,8 +16,6 @@ The full license is in the LICENSE file, distributed with this software.
 #include "numpy/arrayscalars.h"
 
 
-PANDAS_INLINE npy_int64 get_nat(void) { return NPY_MIN_INT64; }
-
 PANDAS_INLINE int assign_value_1d(PyArrayObject* ap, Py_ssize_t _i,
                                   PyObject* v) {
     npy_intp i = (npy_intp)_i;
@@ -40,16 +38,10 @@ PANDAS_INLINE const char* get_c_string(PyObject* obj) {
 #endif
 }
 
-PANDAS_INLINE PyObject* char_to_string(const char* data) {
-#if PY_VERSION_HEX >= 0x03000000
-    return PyUnicode_FromString(data);
-#else
-    return PyString_FromString(data);
-#endif
-}
-
 void set_array_not_contiguous(PyArrayObject* ao) {
-    ao->flags &= ~(NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS);
+    // Numpy>=1.8-compliant equivalent to:
+    //  ao->flags &= ~(NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS);
+    PyArray_CLEARFLAGS(ao, (NPY_ARRAY_C_CONTIGUOUS | NPY_ARRAY_F_CONTIGUOUS));
 }
 
 #endif  // PANDAS__LIBS_SRC_NUMPY_HELPER_H_

pandas/_libs/tslibs/conversion.pyx

@@ -888,7 +888,7 @@ def tz_localize_to_utc(ndarray[int64_t] vals, object tz, object ambiguous=None,
 
     trans, deltas, typ = get_dst_info(tz)
 
-    tdata = <int64_t*> trans.data
+    tdata = <int64_t*> cnp.PyArray_DATA(trans)
     ntrans = len(trans)
 
     result_a = np.empty(n, dtype=np.int64)

pandas/_libs/tslibs/period.pyx

@@ -34,6 +34,7 @@ cdef extern from "../src/datetime/np_datetime.h":
 cimport util
 from util cimport is_period_object, is_string_object, INT32_MIN
 
+from pandas._libs.tslibs.timedeltas import Timedelta
 from timestamps import Timestamp
 from timezones cimport is_utc, is_tzlocal, get_dst_info
 from timedeltas cimport delta_to_nanoseconds
@@ -1221,6 +1222,10 @@ cdef class _Period(object):
             freq = self._maybe_convert_freq(freq)
         how = _validate_end_alias(how)
 
+        end = how == 'E'
+        if end:
+            return (self + 1).to_timestamp(how='start') - Timedelta(1, 'ns')
+
         if freq is None:
             base, mult = get_freq_code(self.freq)
             freq = get_to_timestamp_base(base)