REF: ignore_failures in BlockManager.reduce

doc/source/whatsnew/v1.2.0.rst

@@ -244,7 +244,7 @@ Timezones
 
 Numeric
 ^^^^^^^
--
+- Bug in :class:`DataFrame` reductions incorrectly ignoring ``ExtensionArray`` behaviors (:issue:`35881`)
 -
 
 Conversion

pandas/core/frame.py

@@ -8625,6 +8625,7 @@ def _reduce(
             cols = self.columns[~dtype_is_dt]
             self = self[cols]
 
+        any_object = self.dtypes.apply(is_object_dtype).any()
         # TODO: Make other agg func handle axis=None properly
         axis = self._get_axis_number(axis)
         labels = self._get_agg_axis(axis)
@@ -8654,7 +8655,13 @@ def _get_data(axis_matters: bool) -> DataFrame:
                 raise NotImplementedError(msg)
             return data
 
-        if numeric_only is not None:
+        if numeric_only is not None or (
+            numeric_only is None and axis == 0 and not any_object
+        ):
+            # For numeric_only non-None and axis non-None, we know
+            #  which blocks to use and no try/except is needed.
+            #  For numeric_only=None only the case with axis==0 and no object
+            #  dtypes are unambiguous can be handled with BlockManager.reduce
             df = self
             if numeric_only is True:
                 df = _get_data(axis_matters=True)
@@ -8663,6 +8670,7 @@ def _get_data(axis_matters: bool) -> DataFrame:
                 axis = 0
 
             out_dtype = "bool" if filter_type == "bool" else None
+            ignore_failures = numeric_only is None
 
             def blk_func(values):
                 if isinstance(values, ExtensionArray):
@@ -8672,12 +8680,15 @@ def blk_func(values):
 
             # After possibly _get_data and transposing, we are now in the
             #  simple case where we can use BlockManager.reduce
-            res = df._mgr.reduce(blk_func)
-            out = df._constructor(res,).iloc[0].rename(None)
+            res, indexer = df._mgr.reduce(blk_func, ignore_failures=ignore_failures)
+            out = df._constructor(res).iloc[0]
+
             if out_dtype is not None:
                 out = out.astype(out_dtype)
             if axis == 0 and is_object_dtype(out.dtype):
-                out[:] = coerce_to_dtypes(out.values, df.dtypes)
+                # GH#35865 careful to cast explicitly to object
+                nvs = coerce_to_dtypes(out.values, df.dtypes.iloc[np.sort(indexer)])
+                out[:] = np.array(nvs, dtype=object)
             return out
 
         assert numeric_only is None

pandas/core/internals/managers.py

@@ -2,6 +2,7 @@
 import itertools
 from typing import (
     Any,
+    Callable,
     DefaultDict,
     Dict,
     List,
@@ -324,18 +325,46 @@ def _verify_integrity(self) -> None:
                 f"tot_items: {tot_items}"
             )
 
-    def reduce(self: T, func) -> T:
+    def reduce(
+        self: T, func: Callable, ignore_failures: bool = False
+    ) -> Tuple[T, np.ndarray]:
+        """
+        Apply reduction function blockwise, returning a single-row BlockManager.
+
+        Parameters
+        ----------
+        func : reduction function
+        ignore_failures : bool, default False
+            Whether to drop blocks where func raises TypeError.
+
+        Returns
+        -------
+        BlockManager
+        np.ndarray
+            Indexer of mgr_locs that are retained.
+        """
         # If 2D, we assume that we're operating column-wise
         assert self.ndim == 2
 
         res_blocks: List[Block] = []
         for blk in self.blocks:
-            nbs = blk.reduce(func)
+            try:
+                nbs = blk.reduce(func)
+            except TypeError:
+                if ignore_failures:
+                    continue
+                raise
             res_blocks.extend(nbs)
 
-        index = Index([0])  # placeholder
-        new_mgr = BlockManager.from_blocks(res_blocks, [self.items, index])
-        return new_mgr
+        index = Index([None])  # placeholder
+        if res_blocks:
+            indexer = np.concatenate([blk.mgr_locs.as_array for blk in res_blocks])
+            new_mgr = self._combine(res_blocks, copy=False, index=index)
+        else:
+            indexer = []
+            new_mgr = type(self).from_blocks([], [Index([]), index])
+
+        return new_mgr, indexer
 
     def operate_blockwise(self, other: "BlockManager", array_op) -> "BlockManager":
         """
@@ -693,7 +722,9 @@ def get_numeric_data(self, copy: bool = False) -> "BlockManager":
         """
         return self._combine([b for b in self.blocks if b.is_numeric], copy)
 
-    def _combine(self: T, blocks: List[Block], copy: bool = True) -> T:
+    def _combine(
+        self: T, blocks: List[Block], copy: bool = True, index: Optional[Index] = None
+    ) -> T:
         """ return a new manager with the blocks """
         if len(blocks) == 0:
             return self.make_empty()
@@ -709,6 +740,8 @@ def _combine(self: T, blocks: List[Block], copy: bool = True) -> T:
             new_blocks.append(b)
 
         axes = list(self.axes)
+        if index is not None:
+            axes[-1] = index
         axes[0] = self.items.take(indexer)
 
         return type(self).from_blocks(new_blocks, axes)

pandas/tests/frame/test_analytics.py

@@ -1068,10 +1068,10 @@ def test_any_all_bool_only(self):
             pytest.param(np.any, {"A": pd.Series([0, 1], dtype="m8[ns]")}, True,),
             pytest.param(np.all, {"A": pd.Series([1, 2], dtype="m8[ns]")}, True,),
             pytest.param(np.any, {"A": pd.Series([1, 2], dtype="m8[ns]")}, True,),
-            (np.all, {"A": pd.Series([0, 1], dtype="category")}, False),
-            (np.any, {"A": pd.Series([0, 1], dtype="category")}, True),
+            (np.all, {"A": pd.Series([0, 1], dtype="category")}, True),
+            (np.any, {"A": pd.Series([0, 1], dtype="category")}, False),
             (np.all, {"A": pd.Series([1, 2], dtype="category")}, True),
-            (np.any, {"A": pd.Series([1, 2], dtype="category")}, True),
+            (np.any, {"A": pd.Series([1, 2], dtype="category")}, False),
             # Mix GH#21484
             pytest.param(
                 np.all,