From b0ab1a48a834de78353e4ca9f4159909eaf3a6b8 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Thu, 2 Aug 2018 10:26:03 -0700 Subject: [PATCH 1/3] Fix Series bool ops with Index --- doc/source/whatsnew/v0.24.0.txt | 2 +- pandas/core/ops.py | 58 +++++++++++++++------------ pandas/tests/series/test_operators.py | 14 +++++++ 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt index 8a92db4c66fb5..23ca6259a1f22 100644 --- a/doc/source/whatsnew/v0.24.0.txt +++ b/doc/source/whatsnew/v0.24.0.txt @@ -674,6 +674,6 @@ Other - :meth: `~pandas.io.formats.style.Styler.background_gradient` now takes a ``text_color_threshold`` parameter to automatically lighten the text color based on the luminance of the background color. This improves readability with dark background colors without the need to limit the background colormap range. (:issue:`21258`) - Require at least 0.28.2 version of ``cython`` to support read-only memoryviews (:issue:`21688`) - :meth: `~pandas.io.formats.style.Styler.background_gradient` now also supports tablewise application (in addition to rowwise and columnwise) with ``axis=None`` (:issue:`15204`) -- +- Boolean operations ``&, |, ^`` between a :class:`Series` and an :class:`Index` will no longer raise ``TypeError`` (:issue:`19792`) - - diff --git a/pandas/core/ops.py b/pandas/core/ops.py index f7d863bba82a7..1aa40843dbac4 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -41,7 +41,7 @@ from pandas.core.dtypes.generic import ( ABCSeries, ABCDataFrame, ABCPanel, - ABCIndex, + ABCIndex, ABCIndexClass, ABCSparseSeries, ABCSparseArray) @@ -1448,23 +1448,22 @@ def _bool_method_SERIES(cls, op, special): Wrapper function for Series arithmetic operations, to avoid code duplication. """ + op_name = _get_op_name(op, special) def na_op(x, y): try: result = op(x, y) except TypeError: - if isinstance(y, list): - y = construct_1d_object_array_from_listlike(y) - - if isinstance(y, (np.ndarray, ABCSeries)): - if (is_bool_dtype(x.dtype) and is_bool_dtype(y.dtype)): - result = op(x, y) # when would this be hit? - else: - x = ensure_object(x) - y = ensure_object(y) - result = libops.vec_binop(x, y, op) + assert not isinstance(y, (list, ABCSeries, ABCIndexClass)) + if isinstance(y, np.ndarray): + # bool-bool dtype operations should be OK, should not get here + assert not (is_bool_dtype(x) and is_bool_dtype(y)) + x = ensure_object(x) + y = ensure_object(y) + result = libops.vec_binop(x, y, op) else: # let null fall thru + assert lib.is_scalar(y) if not isna(y): y = bool(y) try: @@ -1484,33 +1483,40 @@ def wrapper(self, other): is_self_int_dtype = is_integer_dtype(self.dtype) self, other = _align_method_SERIES(self, other, align_asobject=True) + res_name = get_op_result_name(self, other) if isinstance(other, ABCDataFrame): # Defer to DataFrame implementation; fail early return NotImplemented - elif isinstance(other, ABCSeries): - name = get_op_result_name(self, other) + elif isinstance(other, (ABCSeries, ABCIndexClass)): is_other_int_dtype = is_integer_dtype(other.dtype) other = fill_int(other) if is_other_int_dtype else fill_bool(other) - filler = (fill_int if is_self_int_dtype and is_other_int_dtype - else fill_bool) - - res_values = na_op(self.values, other.values) - unfilled = self._constructor(res_values, - index=self.index, name=name) - return filler(unfilled) + ovalues = other.values + finalizer = lambda x: x else: # scalars, list, tuple, np.array - filler = (fill_int if is_self_int_dtype and - is_integer_dtype(np.asarray(other)) else fill_bool) - - res_values = na_op(self.values, other) - unfilled = self._constructor(res_values, index=self.index) - return filler(unfilled).__finalize__(self) + is_other_int_dtype = is_integer_dtype(np.asarray(other)) + if is_list_like(other) and not isinstance(other, np.ndarray): + # TODO: Can we do this before the is_integer_dtype check? + # could the is_integer_dtype check be checking the wrong + # thing? e.g. other = [[0, 1], [2, 3], [4, 5]]? + other = construct_1d_object_array_from_listlike(other) + + ovalues = other + finalizer = lambda x: x.__finalize__(self) + + filler = (fill_int if is_self_int_dtype and is_other_int_dtype + else fill_bool) + res_values = na_op(self.values, ovalues) + unfilled = self._constructor(res_values, + index=self.index, name=res_name) + filled = filler(unfilled) + return finalizer(filled) + wrapper.__name__ = op_name return wrapper diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index df52b4cabc77c..7d61d9a7a5e22 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -1433,6 +1433,20 @@ def test_ops_datetimelike_align(self): result = (dt2.to_frame() - dt.to_frame())[0] assert_series_equal(result, expected) + def test_bool_ops_with_index(self): + # GH#19792 + # TODO: reversed ops still raises, GH#22092 + ser = Series([True, False, True]) + idx = pd.Index([False, True, True]) + + result = ser & idx + expected = Series([False, False, True]) + assert_series_equal(result, expected) + + result = ser | idx + expected = Series([True, True, True]) + assert_series_equal(result, expected) + def test_operators_bitwise(self): # GH 9016: support bitwise op for integer types index = list('bca') From 5f484822e32cfb3af8c89cbe307680f419c06142 Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Sat, 8 Sep 2018 15:19:23 -0700 Subject: [PATCH 2/3] comments, parametrize test --- pandas/core/ops.py | 2 ++ pandas/tests/series/test_operators.py | 25 ++++++++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/pandas/core/ops.py b/pandas/core/ops.py index ed8a00637297d..70fe7de0a973e 100644 --- a/pandas/core/ops.py +++ b/pandas/core/ops.py @@ -1585,6 +1585,8 @@ def wrapper(self, other): ovalues = other finalizer = lambda x: x.__finalize__(self) + # For int vs int `^`, `|`, `&` are bitwise operators and return + # integer dtypes. Otherwise these are boolean ops filler = (fill_int if is_self_int_dtype and is_other_int_dtype else fill_bool) res_values = na_op(self.values, ovalues) diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index c324ab2a56b47..fdb9fc62a058f 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -14,6 +14,7 @@ NaT, date_range, timedelta_range, Categorical) from pandas.core.indexes.datetimes import Timestamp import pandas.core.nanops as nanops +from pandas.core import ops from pandas.compat import range from pandas import compat @@ -603,18 +604,28 @@ def test_ops_datetimelike_align(self): result = (dt2.to_frame() - dt.to_frame())[0] assert_series_equal(result, expected) - def test_bool_ops_with_index(self): + @pytest.mark.parametrize('op', [ + operator.and_, + operator.or_, + pytest.param(ops.rand_, + marks=pytest.mark.xfail(reason="GH#22092 Index " + "implementation returns " + "Index", + raises=AssertionError, + strict=True)), + pytest.param(ops.ror_, + marks=pytest.mark.xfail(reason="GH#22092 Index " + "implementation raises", + raises=ValueError, strict=True)) + ]) + def test_bool_ops_with_index(self, op): # GH#19792 - # TODO: reversed ops still raises, GH#22092 ser = Series([True, False, True]) idx = pd.Index([False, True, True]) - result = ser & idx - expected = Series([False, False, True]) - assert_series_equal(result, expected) + expected = Series([op(ser[n], idx[n]) for n in range(len(ser))]) - result = ser | idx - expected = Series([True, True, True]) + result = op(ser, idx) assert_series_equal(result, expected) def test_operators_bitwise(self): From 17668a09aa9aafb4fd41a55f28190dd7b17b538b Mon Sep 17 00:00:00 2001 From: Brock Mendel Date: Tue, 18 Sep 2018 09:46:38 -0700 Subject: [PATCH 3/3] Merge tests --- pandas/tests/series/test_operators.py | 48 ++++++++++----------------- 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/pandas/tests/series/test_operators.py b/pandas/tests/series/test_operators.py index 02161ad25ab60..601e251d45b4b 100644 --- a/pandas/tests/series/test_operators.py +++ b/pandas/tests/series/test_operators.py @@ -426,30 +426,6 @@ def test_comparison_flex_alignment_fill(self): exp = pd.Series([True, True, False, False], index=list('abcd')) assert_series_equal(left.gt(right, fill_value=0), exp) - def test_logical_ops_with_index(self): - # GH22092 - ser = Series([True, True, False, False]) - idx1 = Index([True, False, True, False]) - idx2 = Index([1, 0, 1, 0]) - - expected = Series([True, False, False, False]) - result1 = ser & idx1 - assert_series_equal(result1, expected) - result2 = ser & idx2 - assert_series_equal(result2, expected) - - expected = Series([True, True, True, False]) - result1 = ser | idx1 - assert_series_equal(result1, expected) - result2 = ser | idx2 - assert_series_equal(result2, expected) - - expected = Series([False, True, True, False]) - result1 = ser ^ idx1 - assert_series_equal(result1, expected) - result2 = ser ^ idx2 - assert_series_equal(result2, expected) - def test_ne(self): ts = Series([3, 4, 5, 6, 7], [3, 4, 5, 6, 7], dtype=float) expected = [True, True, False, True, True] @@ -631,6 +607,7 @@ def test_ops_datetimelike_align(self): @pytest.mark.parametrize('op', [ operator.and_, operator.or_, + operator.xor, pytest.param(ops.rand_, marks=pytest.mark.xfail(reason="GH#22092 Index " "implementation returns " @@ -640,16 +617,27 @@ def test_ops_datetimelike_align(self): pytest.param(ops.ror_, marks=pytest.mark.xfail(reason="GH#22092 Index " "implementation raises", - raises=ValueError, strict=True)) + raises=ValueError, strict=True)), + pytest.param(ops.rxor, + marks=pytest.mark.xfail(reason="GH#22092 Index " + "implementation raises", + raises=TypeError, strict=True)) ]) def test_bool_ops_with_index(self, op): - # GH#19792 - ser = Series([True, False, True]) - idx = pd.Index([False, True, True]) + # GH#22092, GH#19792 + ser = Series([True, True, False, False]) + idx1 = Index([True, False, True, False]) + idx2 = Index([1, 0, 1, 0]) + + expected = Series([op(ser[n], idx1[n]) for n in range(len(ser))]) + + result = op(ser, idx1) + assert_series_equal(result, expected) - expected = Series([op(ser[n], idx[n]) for n in range(len(ser))]) + expected = Series([op(ser[n], idx2[n]) for n in range(len(ser))], + dtype=bool) - result = op(ser, idx) + result = op(ser, idx2) assert_series_equal(result, expected) def test_operators_bitwise(self):