diff --git a/pandas/core/internals/__init__.py b/pandas/core/internals/__init__.py index f0018928255e6..af1350f088b7a 100644 --- a/pandas/core/internals/__init__.py +++ b/pandas/core/internals/__init__.py @@ -9,12 +9,10 @@ ) from pandas.core.internals.blocks import ( # io.pytables, io.packers Block, - DatetimeBlock, DatetimeTZBlock, ExtensionBlock, NumericBlock, ObjectBlock, - TimeDeltaBlock, ) from pandas.core.internals.concat import concatenate_managers from pandas.core.internals.managers import ( @@ -28,11 +26,9 @@ "Block", "CategoricalBlock", "NumericBlock", - "DatetimeBlock", "DatetimeTZBlock", "ExtensionBlock", "ObjectBlock", - "TimeDeltaBlock", "make_block", "DataManager", "ArrayManager", diff --git a/pandas/core/internals/array_manager.py b/pandas/core/internals/array_manager.py index 367fe04678cd8..294d1fd078b08 100644 --- a/pandas/core/internals/array_manager.py +++ b/pandas/core/internals/array_manager.py @@ -471,7 +471,7 @@ def apply_with_block(self: T, f, align_keys=None, swap_axis=True, **kwargs) -> T # error: Item "ExtensionArray" of "Union[Any, ExtensionArray]" has no # attribute "tz" if hasattr(arr, "tz") and arr.tz is None: # type: ignore[union-attr] - # DatetimeArray needs to be converted to ndarray for DatetimeBlock + # DatetimeArray needs to be converted to ndarray for DatetimeLikeBlock # error: Item "ExtensionArray" of "Union[Any, ExtensionArray]" has no # attribute "_data" diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 2df1b62fb8221..d1d0db913f854 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -50,7 +50,6 @@ is_dtype_equal, is_extension_array_dtype, is_list_like, - is_object_dtype, is_sparse, pandas_dtype, ) @@ -207,13 +206,6 @@ def is_bool(self) -> bool: def external_values(self): return external_values(self.values) - @final - def internal_values(self): - """ - The array that Series._values returns (internal values). - """ - return self.values - @property def array_values(self) -> ExtensionArray: """ @@ -1771,7 +1763,8 @@ def get_values(self, dtype: Optional[DtypeObj] = None) -> np.ndarray: return object dtype as boxed values, such as Timestamps/Timedelta """ values = self.values - if is_object_dtype(dtype): + if dtype == _dtype_obj: + # DTA/TDA constructor and astype can handle 2D values = values.astype(object) # TODO(EA2D): reshape not needed with 2D EAs return np.asarray(values).reshape(self.shape) @@ -1821,7 +1814,7 @@ def diff(self, n: int, axis: int = 0) -> List[Block]: Returns ------- - A list with a new TimeDeltaBlock. + A list with a new Block. Notes ----- @@ -1869,19 +1862,16 @@ def delete(self, loc) -> None: pass -class DatetimeLikeBlockMixin(NDArrayBackedExtensionBlock): - """Mixin class for DatetimeBlock, DatetimeTZBlock, and TimedeltaBlock.""" - - values: Union[DatetimeArray, TimedeltaArray] +class DatetimeLikeBlock(NDArrayBackedExtensionBlock): + """Mixin class for DatetimeLikeBlock, DatetimeTZBlock.""" + __slots__ = () is_numeric = False - -class DatetimeBlock(DatetimeLikeBlockMixin): - __slots__ = () + values: Union[DatetimeArray, TimedeltaArray] -class DatetimeTZBlock(ExtensionBlock, DatetimeLikeBlockMixin): +class DatetimeTZBlock(ExtensionBlock, DatetimeLikeBlock): """ implement a datetime64 block with a tz attribute """ values: DatetimeArray @@ -1890,10 +1880,12 @@ class DatetimeTZBlock(ExtensionBlock, DatetimeLikeBlockMixin): is_extension = True is_numeric = False - diff = DatetimeBlock.diff - where = DatetimeBlock.where - putmask = DatetimeLikeBlockMixin.putmask - fillna = DatetimeLikeBlockMixin.fillna + diff = NDArrayBackedExtensionBlock.diff + where = NDArrayBackedExtensionBlock.where + putmask = NDArrayBackedExtensionBlock.putmask + fillna = NDArrayBackedExtensionBlock.fillna + + get_values = NDArrayBackedExtensionBlock.get_values # error: Incompatible types in assignment (expression has type # "Callable[[NDArrayBackedExtensionBlock], bool]", base class "ExtensionBlock" @@ -1901,10 +1893,6 @@ class DatetimeTZBlock(ExtensionBlock, DatetimeLikeBlockMixin): is_view = NDArrayBackedExtensionBlock.is_view # type: ignore[assignment] -class TimeDeltaBlock(DatetimeLikeBlockMixin): - __slots__ = () - - class ObjectBlock(Block): __slots__ = () is_object = True @@ -2022,10 +2010,8 @@ def get_block_type(values, dtype: Optional[Dtype] = None): # Note: need to be sure PandasArray is unwrapped before we get here cls = ExtensionBlock - elif kind == "M": - cls = DatetimeBlock - elif kind == "m": - cls = TimeDeltaBlock + elif kind in ["M", "m"]: + cls = DatetimeLikeBlock elif kind in ["f", "c", "i", "u", "b"]: cls = NumericBlock else: diff --git a/pandas/core/internals/concat.py b/pandas/core/internals/concat.py index a5c1f3985e70e..19c9b27db9f70 100644 --- a/pandas/core/internals/concat.py +++ b/pandas/core/internals/concat.py @@ -507,12 +507,16 @@ def _is_uniform_join_units(join_units: List[JoinUnit]) -> bool: _concatenate_join_units (which uses `concat_compat`). """ - # TODO: require dtype match in addition to same type? e.g. DatetimeTZBlock - # cannot necessarily join return ( # all blocks need to have the same type all(type(ju.block) is type(join_units[0].block) for ju in join_units) # noqa and + # e.g. DatetimeLikeBlock can be dt64 or td64, but these are not uniform + all( + is_dtype_equal(ju.block.dtype, join_units[0].block.dtype) + for ju in join_units + ) + and # no blocks that would get missing values (can lead to type upcasts) # unless we're an extension dtype. all(not ju.is_na or ju.block.is_extension for ju in join_units) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 3c8d942554575..b688f1b4fea5f 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -35,7 +35,6 @@ from pandas.core.dtypes.cast import infer_dtype_from_scalar from pandas.core.dtypes.common import ( - DT64NS_DTYPE, ensure_int64, is_dtype_equal, is_extension_array_dtype, @@ -1639,7 +1638,7 @@ def external_values(self): def internal_values(self): """The array that Series._values returns""" - return self._block.internal_values() + return self._block.values def array_values(self): """The array that Series.array returns""" @@ -1794,17 +1793,11 @@ def _form_blocks( ) blocks.extend(numeric_blocks) - if len(items_dict["TimeDeltaBlock"]): - timedelta_blocks = _multi_blockify( - items_dict["TimeDeltaBlock"], consolidate=consolidate + if len(items_dict["DatetimeLikeBlock"]): + dtlike_blocks = _multi_blockify( + items_dict["DatetimeLikeBlock"], consolidate=consolidate ) - blocks.extend(timedelta_blocks) - - if len(items_dict["DatetimeBlock"]): - datetime_blocks = _simple_blockify( - items_dict["DatetimeBlock"], DT64NS_DTYPE, consolidate=consolidate - ) - blocks.extend(datetime_blocks) + blocks.extend(dtlike_blocks) if len(items_dict["DatetimeTZBlock"]): dttz_blocks = [ diff --git a/pandas/tests/frame/methods/test_quantile.py b/pandas/tests/frame/methods/test_quantile.py index bc1d4605e985a..dbb5cb357de47 100644 --- a/pandas/tests/frame/methods/test_quantile.py +++ b/pandas/tests/frame/methods/test_quantile.py @@ -336,7 +336,7 @@ def test_quantile_box(self): ) tm.assert_frame_equal(res, exp) - # DatetimeBlock may be consolidated and contain NaT in different loc + # DatetimeLikeBlock may be consolidated and contain NaT in different loc df = DataFrame( { "A": [ diff --git a/pandas/tests/internals/test_api.py b/pandas/tests/internals/test_api.py index 60fbd2da70e79..0062d5aa34319 100644 --- a/pandas/tests/internals/test_api.py +++ b/pandas/tests/internals/test_api.py @@ -27,11 +27,9 @@ def test_namespace(): expected = [ "Block", "NumericBlock", - "DatetimeBlock", "DatetimeTZBlock", "ExtensionBlock", "ObjectBlock", - "TimeDeltaBlock", "make_block", "DataManager", "ArrayManager", diff --git a/pandas/tests/series/methods/test_dropna.py b/pandas/tests/series/methods/test_dropna.py index 1c7c52d228cfa..5bff7306fac33 100644 --- a/pandas/tests/series/methods/test_dropna.py +++ b/pandas/tests/series/methods/test_dropna.py @@ -70,7 +70,7 @@ def test_dropna_period_dtype(self): tm.assert_series_equal(result, expected) def test_datetime64_tz_dropna(self): - # DatetimeBlock + # DatetimeLikeBlock ser = Series( [ Timestamp("2011-01-01 10:00"), @@ -85,7 +85,7 @@ def test_datetime64_tz_dropna(self): ) tm.assert_series_equal(result, expected) - # DatetimeBlockTZ + # DatetimeTZBlock idx = DatetimeIndex( ["2011-01-01 10:00", NaT, "2011-01-03 10:00", NaT], tz="Asia/Tokyo" ) diff --git a/pandas/tests/series/methods/test_fillna.py b/pandas/tests/series/methods/test_fillna.py index cf6b357d0a418..51864df915f8c 100644 --- a/pandas/tests/series/methods/test_fillna.py +++ b/pandas/tests/series/methods/test_fillna.py @@ -334,7 +334,7 @@ def test_datetime64_fillna(self): @pytest.mark.parametrize("tz", ["US/Eastern", "Asia/Tokyo"]) def test_datetime64_tz_fillna(self, tz): - # DatetimeBlock + # DatetimeLikeBlock ser = Series( [ Timestamp("2011-01-01 10:00"), @@ -414,7 +414,7 @@ def test_datetime64_tz_fillna(self, tz): tm.assert_series_equal(expected, result) tm.assert_series_equal(isna(ser), null_loc) - # DatetimeBlockTZ + # DatetimeTZBlock idx = DatetimeIndex(["2011-01-01 10:00", NaT, "2011-01-03 10:00", NaT], tz=tz) ser = Series(idx) assert ser.dtype == f"datetime64[ns, {tz}]"