Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix Series v Index bool ops #22173

Merged
merged 11 commits into from
Sep 23, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions doc/source/whatsnew/v0.24.0.txt
Original file line number Diff line number Diff line change
Expand Up @@ -809,5 +809,3 @@ Other
- :meth:`~pandas.io.formats.style.Styler.bar` now also supports tablewise application (in addition to rowwise and columnwise) with ``axis=None`` and setting clipping range with ``vmin`` and ``vmax`` (:issue:`21548` and :issue:`21526`). ``NaN`` values are also handled properly.
- Logical operations ``&, |, ^`` between :class:`Series` and :class:`Index` will no longer raise ``ValueError`` (:issue:`22092`)
-
-
-
58 changes: 33 additions & 25 deletions pandas/core/ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -1525,23 +1525,22 @@ def _bool_method_SERIES(cls, op, special):
Wrapper function for Series arithmetic operations, to avoid
code duplication.
"""
op_name = _get_op_name(op, special)

def na_op(x, y):
try:
result = op(x, y)
except TypeError:
if isinstance(y, list):
y = construct_1d_object_array_from_listlike(y)

if isinstance(y, (np.ndarray, ABCSeries, ABCIndexClass)):
if (is_bool_dtype(x.dtype) and is_bool_dtype(y.dtype)):
result = op(x, y) # when would this be hit?
else:
x = ensure_object(x)
y = ensure_object(y)
result = libops.vec_binop(x, y, op)
assert not isinstance(y, (list, ABCSeries, ABCIndexClass))
Copy link
Member

@gfyoung gfyoung Aug 3, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From an error reporting perspective, I might go for:

if isinstance(y, (list, ABCSeries, ABCIndexClass)):
   raise

A bare assert statement would unfortunately not provide much info to the end-user (admittedly, my proposed change assumes that the TypeError raised has some kind of message). At the very least, adding an error message on the assert would be useful.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This comment BTW applies to any of your other assert statements.

if isinstance(y, np.ndarray):
# bool-bool dtype operations should be OK, should not get here
assert not (is_bool_dtype(x) and is_bool_dtype(y))
x = ensure_object(x)
y = ensure_object(y)
result = libops.vec_binop(x, y, op)
else:
# let null fall thru
assert lib.is_scalar(y)
if not isna(y):
y = bool(y)
try:
Expand All @@ -1561,33 +1560,42 @@ def wrapper(self, other):
is_self_int_dtype = is_integer_dtype(self.dtype)

self, other = _align_method_SERIES(self, other, align_asobject=True)
res_name = get_op_result_name(self, other)

if isinstance(other, ABCDataFrame):
# Defer to DataFrame implementation; fail early
return NotImplemented

elif isinstance(other, ABCSeries):
name = get_op_result_name(self, other)
elif isinstance(other, (ABCSeries, ABCIndexClass)):
is_other_int_dtype = is_integer_dtype(other.dtype)
other = fill_int(other) if is_other_int_dtype else fill_bool(other)

filler = (fill_int if is_self_int_dtype and is_other_int_dtype
else fill_bool)

res_values = na_op(self.values, other.values)
unfilled = self._constructor(res_values,
index=self.index, name=name)
return filler(unfilled)
ovalues = other.values
finalizer = lambda x: x

else:
# scalars, list, tuple, np.array
filler = (fill_int if is_self_int_dtype and
is_integer_dtype(np.asarray(other)) else fill_bool)

res_values = na_op(self.values, other)
unfilled = self._constructor(res_values, index=self.index)
return filler(unfilled).__finalize__(self)
is_other_int_dtype = is_integer_dtype(np.asarray(other))
if is_list_like(other) and not isinstance(other, np.ndarray):
# TODO: Can we do this before the is_integer_dtype check?
# could the is_integer_dtype check be checking the wrong
# thing? e.g. other = [[0, 1], [2, 3], [4, 5]]?
other = construct_1d_object_array_from_listlike(other)

ovalues = other
finalizer = lambda x: x.__finalize__(self)

# For int vs int `^`, `|`, `&` are bitwise operators and return
# integer dtypes. Otherwise these are boolean ops
filler = (fill_int if is_self_int_dtype and is_other_int_dtype
else fill_bool)
res_values = na_op(self.values, ovalues)
unfilled = self._constructor(res_values,
index=self.index, name=res_name)
filled = filler(unfilled)
return finalizer(filled)

wrapper.__name__ = op_name
return wrapper


Expand Down
61 changes: 37 additions & 24 deletions pandas/tests/series/test_operators.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
NaT, date_range, timedelta_range, Categorical)
from pandas.core.indexes.datetimes import Timestamp
import pandas.core.nanops as nanops
from pandas.core import ops

from pandas.compat import range
from pandas import compat
Expand Down Expand Up @@ -425,30 +426,6 @@ def test_comparison_flex_alignment_fill(self):
exp = pd.Series([True, True, False, False], index=list('abcd'))
assert_series_equal(left.gt(right, fill_value=0), exp)

def test_logical_ops_with_index(self):
# GH22092
ser = Series([True, True, False, False])
idx1 = Index([True, False, True, False])
idx2 = Index([1, 0, 1, 0])

expected = Series([True, False, False, False])
result1 = ser & idx1
assert_series_equal(result1, expected)
result2 = ser & idx2
assert_series_equal(result2, expected)

expected = Series([True, True, True, False])
result1 = ser | idx1
assert_series_equal(result1, expected)
result2 = ser | idx2
assert_series_equal(result2, expected)

expected = Series([False, True, True, False])
result1 = ser ^ idx1
assert_series_equal(result1, expected)
result2 = ser ^ idx2
assert_series_equal(result2, expected)

def test_ne(self):
ts = Series([3, 4, 5, 6, 7], [3, 4, 5, 6, 7], dtype=float)
expected = [True, True, False, True, True]
Expand Down Expand Up @@ -627,6 +604,42 @@ def test_ops_datetimelike_align(self):
result = (dt2.to_frame() - dt.to_frame())[0]
assert_series_equal(result, expected)

@pytest.mark.parametrize('op', [
operator.and_,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pls add these as a fixture in conftest (follow-on PR)

operator.or_,
operator.xor,
pytest.param(ops.rand_,
marks=pytest.mark.xfail(reason="GH#22092 Index "
"implementation returns "
"Index",
raises=AssertionError,
strict=True)),
pytest.param(ops.ror_,
marks=pytest.mark.xfail(reason="GH#22092 Index "
"implementation raises",
raises=ValueError, strict=True)),
pytest.param(ops.rxor,
marks=pytest.mark.xfail(reason="GH#22092 Index "
"implementation raises",
raises=TypeError, strict=True))
])
def test_bool_ops_with_index(self, op):
# GH#22092, GH#19792
ser = Series([True, True, False, False])
idx1 = Index([True, False, True, False])
idx2 = Index([1, 0, 1, 0])

expected = Series([op(ser[n], idx1[n]) for n in range(len(ser))])

result = op(ser, idx1)
assert_series_equal(result, expected)

expected = Series([op(ser[n], idx2[n]) for n in range(len(ser))],
dtype=bool)

result = op(ser, idx2)
assert_series_equal(result, expected)

def test_operators_bitwise(self):
# GH 9016: support bitwise op for integer types
index = list('bca')
Expand Down