Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

REF: remove DatetimeBlock, TimeDeltaBlock #40614

Merged
merged 21 commits into from
Apr 2, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
fae2a97
REF: back DatetimeBlock, TimedeltaBlock by DTA/TDA
jbrockmendel Mar 16, 2021
c5e3dd9
Merge branch 'master' into ref-hybrid-4
jbrockmendel Mar 16, 2021
7a4d545
Merge branch 'master' into ref-hybrid-4
jbrockmendel Mar 16, 2021
c776390
fix pytables tests
jbrockmendel Mar 16, 2021
62b5806
Merge branch 'master' into ref-hybrid-4
jbrockmendel Mar 16, 2021
c3ce503
exception message shared AM/BM
jbrockmendel Mar 16, 2021
3b0e819
Merge branch 'master' into ref-hybrid-4
jbrockmendel Mar 19, 2021
868f436
Merge branch 'master' into ref-hybrid-4
jbrockmendel Mar 19, 2021
14b4087
Merge branch 'master' into ref-hybrid-4
jbrockmendel Mar 22, 2021
f076370
Merge branch 'master' into ref-hybrid-4
jbrockmendel Mar 22, 2021
d7f2718
nanargminmax with ndim=2 and mask.any()
jbrockmendel Mar 23, 2021
626c2a9
Merge branch 'master' into ref-hybrid-4
jbrockmendel Mar 23, 2021
3fbc3a5
Merge branch 'master' into ref-hybrid-4
jbrockmendel Mar 24, 2021
c7ff237
REF: remove DatetimeBlock, TimeDeltaBlock
jbrockmendel Mar 24, 2021
7d0167c
Merge branch 'master' into ref-hybrid-5
jbrockmendel Mar 27, 2021
0ed97e2
Merge branch 'master' into ref-hybrid-5
jbrockmendel Mar 30, 2021
1a7b015
Merge branch 'master' into ref-hybrid-5
jbrockmendel Mar 31, 2021
4da003b
Merge branch 'master' into ref-hybrid-5
jbrockmendel Mar 31, 2021
4aad27e
Merge branch 'master' into ref-hybrid-5
jbrockmendel Apr 2, 2021
3a5ceeb
Merge branch 'master' into ref-hybrid-5
jbrockmendel Apr 2, 2021
fb18444
mypy fixup
jbrockmendel Apr 2, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions pandas/core/internals/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,10 @@
)
from pandas.core.internals.blocks import ( # io.pytables, io.packers
Block,
DatetimeBlock,
DatetimeTZBlock,
ExtensionBlock,
NumericBlock,
ObjectBlock,
TimeDeltaBlock,
)
from pandas.core.internals.concat import concatenate_managers
from pandas.core.internals.managers import (
Expand All @@ -28,11 +26,9 @@
"Block",
"CategoricalBlock",
"NumericBlock",
"DatetimeBlock",
"DatetimeTZBlock",
"ExtensionBlock",
"ObjectBlock",
"TimeDeltaBlock",
"make_block",
"DataManager",
"ArrayManager",
Expand Down
2 changes: 1 addition & 1 deletion pandas/core/internals/array_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -471,7 +471,7 @@ def apply_with_block(self: T, f, align_keys=None, swap_axis=True, **kwargs) -> T
# error: Item "ExtensionArray" of "Union[Any, ExtensionArray]" has no
# attribute "tz"
if hasattr(arr, "tz") and arr.tz is None: # type: ignore[union-attr]
# DatetimeArray needs to be converted to ndarray for DatetimeBlock
# DatetimeArray needs to be converted to ndarray for DatetimeLikeBlock

# error: Item "ExtensionArray" of "Union[Any, ExtensionArray]" has no
# attribute "_data"
Expand Down
46 changes: 16 additions & 30 deletions pandas/core/internals/blocks.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@
is_dtype_equal,
is_extension_array_dtype,
is_list_like,
is_object_dtype,
is_sparse,
pandas_dtype,
)
Expand Down Expand Up @@ -207,13 +206,6 @@ def is_bool(self) -> bool:
def external_values(self):
return external_values(self.values)

@final
def internal_values(self):
"""
The array that Series._values returns (internal values).
"""
return self.values

@property
def array_values(self) -> ExtensionArray:
"""
Expand Down Expand Up @@ -1771,7 +1763,8 @@ def get_values(self, dtype: Optional[DtypeObj] = None) -> np.ndarray:
return object dtype as boxed values, such as Timestamps/Timedelta
"""
values = self.values
if is_object_dtype(dtype):
if dtype == _dtype_obj:
# DTA/TDA constructor and astype can handle 2D
values = values.astype(object)
# TODO(EA2D): reshape not needed with 2D EAs
return np.asarray(values).reshape(self.shape)
Expand Down Expand Up @@ -1821,7 +1814,7 @@ def diff(self, n: int, axis: int = 0) -> List[Block]:

Returns
-------
A list with a new TimeDeltaBlock.
A list with a new Block.

Notes
-----
Expand Down Expand Up @@ -1869,19 +1862,16 @@ def delete(self, loc) -> None:
pass


class DatetimeLikeBlockMixin(NDArrayBackedExtensionBlock):
"""Mixin class for DatetimeBlock, DatetimeTZBlock, and TimedeltaBlock."""

values: Union[DatetimeArray, TimedeltaArray]
class DatetimeLikeBlock(NDArrayBackedExtensionBlock):
"""Mixin class for DatetimeLikeBlock, DatetimeTZBlock."""

__slots__ = ()
is_numeric = False


class DatetimeBlock(DatetimeLikeBlockMixin):
__slots__ = ()
values: Union[DatetimeArray, TimedeltaArray]


class DatetimeTZBlock(ExtensionBlock, DatetimeLikeBlockMixin):
class DatetimeTZBlock(ExtensionBlock, DatetimeLikeBlock):
""" implement a datetime64 block with a tz attribute """

values: DatetimeArray
Expand All @@ -1890,21 +1880,19 @@ class DatetimeTZBlock(ExtensionBlock, DatetimeLikeBlockMixin):
is_extension = True
is_numeric = False

diff = DatetimeBlock.diff
where = DatetimeBlock.where
putmask = DatetimeLikeBlockMixin.putmask
fillna = DatetimeLikeBlockMixin.fillna
diff = NDArrayBackedExtensionBlock.diff
where = NDArrayBackedExtensionBlock.where
putmask = NDArrayBackedExtensionBlock.putmask
fillna = NDArrayBackedExtensionBlock.fillna

get_values = NDArrayBackedExtensionBlock.get_values

# error: Incompatible types in assignment (expression has type
# "Callable[[NDArrayBackedExtensionBlock], bool]", base class "ExtensionBlock"
# defined the type as "bool") [assignment]
is_view = NDArrayBackedExtensionBlock.is_view # type: ignore[assignment]


class TimeDeltaBlock(DatetimeLikeBlockMixin):
__slots__ = ()


class ObjectBlock(Block):
__slots__ = ()
is_object = True
Expand Down Expand Up @@ -2022,10 +2010,8 @@ def get_block_type(values, dtype: Optional[Dtype] = None):
# Note: need to be sure PandasArray is unwrapped before we get here
cls = ExtensionBlock

elif kind == "M":
cls = DatetimeBlock
elif kind == "m":
cls = TimeDeltaBlock
elif kind in ["M", "m"]:
cls = DatetimeLikeBlock
elif kind in ["f", "c", "i", "u", "b"]:
cls = NumericBlock
else:
Expand Down
8 changes: 6 additions & 2 deletions pandas/core/internals/concat.py
Original file line number Diff line number Diff line change
Expand Up @@ -507,12 +507,16 @@ def _is_uniform_join_units(join_units: List[JoinUnit]) -> bool:
_concatenate_join_units (which uses `concat_compat`).

"""
# TODO: require dtype match in addition to same type? e.g. DatetimeTZBlock
# cannot necessarily join
return (
# all blocks need to have the same type
all(type(ju.block) is type(join_units[0].block) for ju in join_units) # noqa
and
# e.g. DatetimeLikeBlock can be dt64 or td64, but these are not uniform
all(
is_dtype_equal(ju.block.dtype, join_units[0].block.dtype)
for ju in join_units
)
and
# no blocks that would get missing values (can lead to type upcasts)
# unless we're an extension dtype.
all(not ju.is_na or ju.block.is_extension for ju in join_units)
Expand Down
17 changes: 5 additions & 12 deletions pandas/core/internals/managers.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@

from pandas.core.dtypes.cast import infer_dtype_from_scalar
from pandas.core.dtypes.common import (
DT64NS_DTYPE,
ensure_int64,
is_dtype_equal,
is_extension_array_dtype,
Expand Down Expand Up @@ -1639,7 +1638,7 @@ def external_values(self):

def internal_values(self):
"""The array that Series._values returns"""
return self._block.internal_values()
return self._block.values

def array_values(self):
"""The array that Series.array returns"""
Expand Down Expand Up @@ -1794,17 +1793,11 @@ def _form_blocks(
)
blocks.extend(numeric_blocks)

if len(items_dict["TimeDeltaBlock"]):
timedelta_blocks = _multi_blockify(
items_dict["TimeDeltaBlock"], consolidate=consolidate
if len(items_dict["DatetimeLikeBlock"]):
dtlike_blocks = _multi_blockify(
items_dict["DatetimeLikeBlock"], consolidate=consolidate
)
blocks.extend(timedelta_blocks)

if len(items_dict["DatetimeBlock"]):
datetime_blocks = _simple_blockify(
items_dict["DatetimeBlock"], DT64NS_DTYPE, consolidate=consolidate
)
blocks.extend(datetime_blocks)
blocks.extend(dtlike_blocks)

if len(items_dict["DatetimeTZBlock"]):
dttz_blocks = [
Expand Down
2 changes: 1 addition & 1 deletion pandas/tests/frame/methods/test_quantile.py
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ def test_quantile_box(self):
)
tm.assert_frame_equal(res, exp)

# DatetimeBlock may be consolidated and contain NaT in different loc
# DatetimeLikeBlock may be consolidated and contain NaT in different loc
df = DataFrame(
{
"A": [
Expand Down
2 changes: 0 additions & 2 deletions pandas/tests/internals/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,9 @@ def test_namespace():
expected = [
"Block",
"NumericBlock",
"DatetimeBlock",
"DatetimeTZBlock",
"ExtensionBlock",
"ObjectBlock",
"TimeDeltaBlock",
"make_block",
"DataManager",
"ArrayManager",
Expand Down
4 changes: 2 additions & 2 deletions pandas/tests/series/methods/test_dropna.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def test_dropna_period_dtype(self):
tm.assert_series_equal(result, expected)

def test_datetime64_tz_dropna(self):
# DatetimeBlock
# DatetimeLikeBlock
ser = Series(
[
Timestamp("2011-01-01 10:00"),
Expand All @@ -85,7 +85,7 @@ def test_datetime64_tz_dropna(self):
)
tm.assert_series_equal(result, expected)

# DatetimeBlockTZ
# DatetimeTZBlock
idx = DatetimeIndex(
["2011-01-01 10:00", NaT, "2011-01-03 10:00", NaT], tz="Asia/Tokyo"
)
Expand Down
4 changes: 2 additions & 2 deletions pandas/tests/series/methods/test_fillna.py
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,7 @@ def test_datetime64_fillna(self):

@pytest.mark.parametrize("tz", ["US/Eastern", "Asia/Tokyo"])
def test_datetime64_tz_fillna(self, tz):
# DatetimeBlock
# DatetimeLikeBlock
ser = Series(
[
Timestamp("2011-01-01 10:00"),
Expand Down Expand Up @@ -414,7 +414,7 @@ def test_datetime64_tz_fillna(self, tz):
tm.assert_series_equal(expected, result)
tm.assert_series_equal(isna(ser), null_loc)

# DatetimeBlockTZ
# DatetimeTZBlock
idx = DatetimeIndex(["2011-01-01 10:00", NaT, "2011-01-03 10:00", NaT], tz=tz)
ser = Series(idx)
assert ser.dtype == f"datetime64[ns, {tz}]"
Expand Down