Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BUG: Handle all-NA blocks in concat #20382

Merged
merged 3 commits into from
Mar 18, 2018
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion pandas/core/internals.py
Original file line number Diff line number Diff line change
Expand Up @@ -5393,7 +5393,8 @@ def is_uniform_join_units(join_units):
# all blocks need to have the same type
all(type(ju.block) is type(join_units[0].block) for ju in join_units) and # noqa
# no blocks that would get missing values (can lead to type upcasts)
all(not ju.is_na for ju in join_units) and
# unless we're an extension dtype.
all(not ju.is_na or ju.block.is_extension for ju in join_units) and
# no blocks with indexers (as then the dimensions do not fit)
all(not ju.indexers for ju in join_units) and
# disregard Panels
Expand Down
15 changes: 15 additions & 0 deletions pandas/tests/extension/base/reshaping.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,21 @@ def test_concat(self, data, in_frame):
assert dtype == data.dtype
assert isinstance(result._data.blocks[0], ExtensionBlock)

@pytest.mark.parametrize('in_frame', [True])
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be [True, False]? If False isn't supported yet, might be nice to write a quick comment, as separate code paths for False exist in the test.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, was debugging that case. Thanks.

def test_concat_all_na_block(self, data_missing, in_frame):
valid_block = pd.Series(data_missing.take([1, 1]), index=[0, 1])
na_block = pd.Series(data_missing.take([0, 0]), index=[2, 3])
if in_frame:
valid_block = pd.DataFrame({"a": valid_block})
na_block = pd.DataFrame({"a": na_block})
result = pd.concat([valid_block, na_block])
if in_frame:
expected = pd.DataFrame({"a": data_missing.take([1, 1, 0, 0])})
self.assert_frame_equal(result, expected)
else:
expected = pd.DataFrame({"a": data_missing.take([1, 1, 0, 0])})
self.assert_series_equal(result, expected)

def test_align(self, data, na_value):
a = data[:3]
b = data[2:5]
Expand Down
35 changes: 27 additions & 8 deletions pandas/tests/extension/decimal/test_decimal.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,19 +35,38 @@ def na_value():
return decimal.Decimal("NaN")


class TestDtype(base.BaseDtypeTests):
class BaseDecimal(object):

def assert_series_equal(self, left, right, *args, **kwargs):

left_na = left.isna()
right_na = right.isna()

tm.assert_series_equal(left_na, right_na)
return tm.assert_series_equal(left[~left_na],
right[~right_na],
*args, **kwargs)

def assert_frame_equal(self, left, right, *args, **kwargs):
self.assert_series_equal(left.dtypes, right.dtypes)
for col in left.columns:
self.assert_series_equal(left[col], right[col],
*args, **kwargs)


class TestDtype(BaseDecimal, base.BaseDtypeTests):
pass


class TestInterface(base.BaseInterfaceTests):
class TestInterface(BaseDecimal, base.BaseInterfaceTests):
pass


class TestConstructors(base.BaseConstructorsTests):
class TestConstructors(BaseDecimal, base.BaseConstructorsTests):
pass


class TestReshaping(base.BaseReshapingTests):
class TestReshaping(BaseDecimal, base.BaseReshapingTests):

def test_align(self, data, na_value):
# Have to override since assert_series_equal doesn't
Expand Down Expand Up @@ -88,15 +107,15 @@ def test_align_frame(self, data, na_value):
assert e2.loc[0, 'A'].is_nan()


class TestGetitem(base.BaseGetitemTests):
class TestGetitem(BaseDecimal, base.BaseGetitemTests):
pass


class TestMissing(base.BaseMissingTests):
class TestMissing(BaseDecimal, base.BaseMissingTests):
pass


class TestMethods(base.BaseMethodsTests):
class TestMethods(BaseDecimal, base.BaseMethodsTests):
@pytest.mark.parametrize('dropna', [True, False])
@pytest.mark.xfail(reason="value_counts not implemented yet.")
def test_value_counts(self, all_data, dropna):
Expand All @@ -112,7 +131,7 @@ def test_value_counts(self, all_data, dropna):
tm.assert_series_equal(result, expected)


class TestCasting(base.BaseCastingTests):
class TestCasting(BaseDecimal, base.BaseCastingTests):
pass


Expand Down