diff --git a/doc/source/whatsnew/v1.0.4.rst b/doc/source/whatsnew/v1.0.4.rst index 1ce3ba5f088e7..294b2c3512623 100644 --- a/doc/source/whatsnew/v1.0.4.rst +++ b/doc/source/whatsnew/v1.0.4.rst @@ -15,6 +15,7 @@ including other versions of pandas. Fixed regressions ~~~~~~~~~~~~~~~~~ +- Bug where :meth:`Series.isna` and :meth:`DataFrame.isna` would raise for categorical dtype when ``pandas.options.mode.use_inf_as_na`` was set to ``True`` (:issue:`33594`) - Bug in :meth:`GroupBy.first` and :meth:`GroupBy.last` where None is not preserved in object dtype (:issue:`32800`) - Bug in DataFrame reductions using ``numeric_only=True`` and ExtensionArrays (:issue:`33256`). - Bug where :meth:`Categorical.replace` would replace with ``NaN`` whenever the new value and replacement value were equal (:issue:`33288`) diff --git a/pandas/core/dtypes/missing.py b/pandas/core/dtypes/missing.py index fb579f2f58a57..269f73153edfa 100644 --- a/pandas/core/dtypes/missing.py +++ b/pandas/core/dtypes/missing.py @@ -269,10 +269,18 @@ def _isna_ndarraylike(obj): def _isna_ndarraylike_old(obj): + is_extension = is_extension_array_dtype(obj) + values = getattr(obj, "values", obj) dtype = values.dtype - if is_string_dtype(dtype): + if is_extension: + if isinstance(obj, (ABCIndexClass, ABCSeries)): + values = obj._values + else: + values = obj + result = values.isna() | (values == -np.inf) | (values == np.inf) + elif is_string_dtype(dtype): # Working around NumPy ticket 1542 shape = values.shape diff --git a/pandas/tests/arrays/categorical/test_missing.py b/pandas/tests/arrays/categorical/test_missing.py index 8889f45a84237..19ab6d00ba10d 100644 --- a/pandas/tests/arrays/categorical/test_missing.py +++ b/pandas/tests/arrays/categorical/test_missing.py @@ -5,7 +5,8 @@ from pandas.core.dtypes.dtypes import CategoricalDtype -from pandas import Categorical, Index, Series, isna +import pandas as pd +from pandas import Categorical, DataFrame, Index, Series, isna import pandas._testing as tm @@ -82,3 +83,53 @@ def test_fillna_iterable_category(self, named): expected = Categorical([Point(0, 0), Point(0, 1), Point(0, 0)]) tm.assert_categorical_equal(result, expected) + + @pytest.mark.parametrize( + "values, expected", + [ + ([1, 2, 3], np.array([False, False, False])), + ([1, 2, np.nan], np.array([False, False, True])), + ([1, 2, np.inf], np.array([False, False, True])), + ([1, 2, pd.NA], np.array([False, False, True])), + ], + ) + def test_use_inf_as_na(self, values, expected): + # https://github.com/pandas-dev/pandas/issues/33594 + with pd.option_context("mode.use_inf_as_na", True): + cat = Categorical(values) + result = cat.isna() + tm.assert_numpy_array_equal(result, expected) + + result = Series(cat).isna() + expected = Series(expected) + tm.assert_series_equal(result, expected) + + result = DataFrame(cat).isna() + expected = DataFrame(expected) + tm.assert_frame_equal(result, expected) + + @pytest.mark.parametrize( + "values, expected", + [ + ([1, 2, 3], np.array([False, False, False])), + ([1, 2, np.nan], np.array([False, False, True])), + ([1, 2, np.inf], np.array([False, False, True])), + ([1, 2, pd.NA], np.array([False, False, True])), + ], + ) + def test_use_inf_as_na_outside_context(self, values, expected): + # https://github.com/pandas-dev/pandas/issues/33594 + # Using isna directly for Categorical will fail in general here + cat = Categorical(values) + + with pd.option_context("mode.use_inf_as_na", True): + result = pd.isna(cat) + tm.assert_numpy_array_equal(result, expected) + + result = pd.isna(Series(cat)) + expected = Series(expected) + tm.assert_series_equal(result, expected) + + result = pd.isna(DataFrame(cat)) + expected = DataFrame(expected) + tm.assert_frame_equal(result, expected)