Value error when trying to access a series having dtype: string and some elements set to None

[susmitpy]

System information

• OS Platform and Distribution: Mac OS Monterey 12.1
• Modin version (modin.__version__): 0.12.0
• Python version: 3.7.6
• Code we can use to reproduce:

Describe the problem

import pandas as pd
import modin.pandas as mpd

ps = pd.Series(["A","B","C"])
ms = mpd.Series(["A","B","C"])
ps = ps.astype("string")
ms = ms.astype("string")

print(ps.dtype)  # string
print(ms.dtype)  # string

ps[[0,1]] = None
ms[[0,1]] = None  # No error here

print(ps)  # <NA>, <NA>, "C" dtype: string
print(ms)  # ValueError: Must provide strings. (Error occurs when trying to access the series for any operation)

Source code / logs

ray::apply_func() (pid=2632, ip=127.0.0.1)
  File "/Users/susmitvengurlekar/.virtualenvs/backend_xboost/lib/python3.7/site-packages/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition.py", line 427, in apply_func
    result = func(partition.copy(), *args, **kwargs)
  File "/Users/susmitvengurlekar/.virtualenvs/backend_xboost/lib/python3.7/site-packages/modin/core/storage_formats/pandas/query_compiler.py", line 3080, in iloc_mut
    partition.iloc[row_internal_indices, col_internal_indices] = item
  File "/Users/susmitvengurlekar/.virtualenvs/backend_xboost/lib/python3.7/site-packages/pandas/core/indexing.py", line 723, in __setitem__
    iloc._setitem_with_indexer(indexer, value, self.name)
  File "/Users/susmitvengurlekar/.virtualenvs/backend_xboost/lib/python3.7/site-packages/pandas/core/indexing.py", line 1732, in _setitem_with_indexer
    self._setitem_single_block(indexer, value, name)
  File "/Users/susmitvengurlekar/.virtualenvs/backend_xboost/lib/python3.7/site-packages/pandas/core/indexing.py", line 1968, in _setitem_single_block
    self.obj._mgr = self.obj._mgr.setitem(indexer=indexer, value=value)
  File "/Users/susmitvengurlekar/.virtualenvs/backend_xboost/lib/python3.7/site-packages/pandas/core/internals/managers.py", line 355, in setitem
    return self.apply("setitem", indexer=indexer, value=value)
  File "/Users/susmitvengurlekar/.virtualenvs/backend_xboost/lib/python3.7/site-packages/pandas/core/internals/managers.py", line 327, in apply
    applied = getattr(b, f)(**kwargs)
  File "/Users/susmitvengurlekar/.virtualenvs/backend_xboost/lib/python3.7/site-packages/pandas/core/internals/blocks.py", line 1491, in setitem
    self.values[indexer] = value
  File "/Users/susmitvengurlekar/.virtualenvs/backend_xboost/lib/python3.7/site-packages/pandas/core/arrays/string_.py", line 413, in __setitem__
    raise ValueError("Must provide strings.")
ValueError: Must provide strings.

If nulls are already present and the type is changed it works without any issues

ms = mpd.Series(["A","B","C",None])
ms = ms.astype("string")
print(ms)  # elements are printed and the dtype is string

Workaround:
Convert dtype to object before setting values to None

[mvashishtha]

@susmitpy Thanks for reporting this issue. I can reproduce it on my computer.

The root cause appears to be a bug in pandas. When I run your example, I found that it comes from this line, where we try to use iloc to set part of a dataframe to [[None], [None]]. Here's the code I'm using to reproduce the pandas bug:

import pandas as pd
s = pd.DataFrame([["A"], ["B"], ["C"]], dtype="string")
s.iloc[[0, 1], :] = [None, None]

On the latest pandas source code (version 1.5.0.dev0+122.gb4e578db85) I get a NotImplementedError:

Stack trace

---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
Input In [9], in <module>
----> 1 s.iloc[[0, 1], :] = [None, None]

File ~/pandas/pandas/core/indexing.py:731, in _LocationIndexer.__setitem__(self, key, value)
    728 self._has_valid_setitem_indexer(key)
    730 iloc = self if self.name == "iloc" else self.obj.iloc
--> 731 iloc._setitem_with_indexer(indexer, value, self.name)

File ~/pandas/pandas/core/indexing.py:1704, in _iLocIndexer._setitem_with_indexer(self, indexer, value, name)
   1702     self._setitem_with_indexer_split_path(indexer, value, name)
   1703 else:
-> 1704     self._setitem_single_block(indexer, value, name)

File ~/pandas/pandas/core/indexing.py:1952, in _iLocIndexer._setitem_single_block(self, indexer, value, name)
   1949 self.obj._check_is_chained_assignment_possible()
   1951 # actually do the set
-> 1952 self.obj._mgr = self.obj._mgr.setitem(indexer=indexer, value=value)
   1953 self.obj._maybe_update_cacher(clear=True, inplace=True)

File ~/pandas/pandas/core/internals/managers.py:337, in BaseBlockManager.setitem(self, indexer, value)
    331 def setitem(self: T, indexer, value) -> T:
    332     """
    333     Set values with indexer.
    334
    335     For SingleBlockManager, this backs s[indexer] = value
    336     """
--> 337     return self.apply("setitem", indexer=indexer, value=value)

File ~/pandas/pandas/core/internals/managers.py:304, in BaseBlockManager.apply(self, f, align_keys, ignore_failures, **kwargs)
    302         applied = b.apply(f, **kwargs)
    303     else:
--> 304         applied = getattr(b, f)(**kwargs)
    305 except (TypeError, NotImplementedError):
    306     if not ignore_failures:

File ~/pandas/pandas/core/internals/blocks.py:1394, in EABackedBlock.setitem(self, indexer, value)
   1391     nb = self.coerce_to_target_dtype(value)
   1392     return nb.setitem(indexer, value)
-> 1394 indexer = self._unwrap_setitem_indexer(indexer)
   1395 value = self._maybe_squeeze_arg(value)
   1397 values = self.values

File ~/pandas/pandas/core/internals/blocks.py:1652, in ExtensionBlock._unwrap_setitem_indexer(self, indexer)
   1650             indexer = indexer[0]
   1651         else:
-> 1652             raise NotImplementedError(
   1653                 "This should not be reached. Please report a bug at "
   1654                 "github.com/pandas-dev/pandas/"
   1655             )
   1656 return indexer

NotImplementedError: This should not be reached. Please report a bug at github.com/pandas-dev/pandas/

On the latest pandas release candidate (version 1.4.0rc0) I get a ValueError:

Stack trace

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Input In [1], in <module>
      1 import pandas as pd
      3 s = pd.DataFrame([["A"], ["B"], ["C"]], dtype="string")
----> 4 s.iloc[[0, 1], :] = [None, None]

File /usr/local/lib/python3.9/site-packages/pandas/core/indexing.py:715, in _LocationIndexer.__setitem__(self, key, value)
    712 self._has_valid_setitem_indexer(key)
    714 iloc = self if self.name == "iloc" else self.obj.iloc
--> 715 iloc._setitem_with_indexer(indexer, value, self.name)

File /usr/local/lib/python3.9/site-packages/pandas/core/indexing.py:1689, in _iLocIndexer._setitem_with_indexer(self, indexer, value, name)
   1687     self._setitem_with_indexer_split_path(indexer, value, name)
   1688 else:
-> 1689     self._setitem_single_block(indexer, value, name)

File /usr/local/lib/python3.9/site-packages/pandas/core/indexing.py:1937, in _iLocIndexer._setitem_single_block(self, indexer, value, name)
   1934 self.obj._check_is_chained_assignment_possible()
   1936 # actually do the set
-> 1937 self.obj._mgr = self.obj._mgr.setitem(indexer=indexer, value=value)
   1938 self.obj._maybe_update_cacher(clear=True, inplace=True)

File /usr/local/lib/python3.9/site-packages/pandas/core/internals/managers.py:337, in BaseBlockManager.setitem(self, indexer, value)
    331 def setitem(self: T, indexer, value) -> T:
    332     """
    333     Set values with indexer.
    334
    335     For SingleBlockManager, this backs s[indexer] = value
    336     """
--> 337     return self.apply("setitem", indexer=indexer, value=value)

File /usr/local/lib/python3.9/site-packages/pandas/core/internals/managers.py:304, in BaseBlockManager.apply(self, f, align_keys, ignore_failures, **kwargs)
    302         applied = b.apply(f, **kwargs)
    303     else:
--> 304         applied = getattr(b, f)(**kwargs)
    305 except (TypeError, NotImplementedError):
    306     if not ignore_failures:

File /usr/local/lib/python3.9/site-packages/pandas/core/internals/blocks.py:1595, in ExtensionBlock.setitem(self, indexer, value)
   1592     value = value._ixs(0, axis=1)._values
   1594 check_setitem_lengths(indexer, value, self.values)
-> 1595 self.values[indexer] = value
   1596 return self

File /usr/local/lib/python3.9/site-packages/pandas/core/arrays/string_.py:412, in StringArray.__setitem__(self, key, value)
    410         value = np.asarray(value, dtype=object)
    411     if len(value) and not lib.is_string_array(value, skipna=True):
--> 412         raise ValueError("Must provide strings.")
    414 super().__setitem__(key, value)

ValueError: Must provide strings.

On the version of pandas that Modin is using (version 1.3.5) I get a ValueError:

Stack trace

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Input In [1], in <module>
      1 import pandas as pd
      3 s = pd.DataFrame([["A"], ["B"], ["C"]], dtype="string")
----> 4 s.iloc[[0, 1], :] = [None, None]

File /usr/local/lib/python3.9/site-packages/pandas/core/indexing.py:723, in _LocationIndexer.__setitem__(self, key, value)
    720 self._has_valid_setitem_indexer(key)
    722 iloc = self if self.name == "iloc" else self.obj.iloc
--> 723 iloc._setitem_with_indexer(indexer, value, self.name)

File /usr/local/lib/python3.9/site-packages/pandas/core/indexing.py:1732, in _iLocIndexer._setitem_with_indexer(self, indexer, value, name)
   1730     self._setitem_with_indexer_split_path(indexer, value, name)
   1731 else:
-> 1732     self._setitem_single_block(indexer, value, name)

File /usr/local/lib/python3.9/site-packages/pandas/core/indexing.py:1968, in _iLocIndexer._setitem_single_block(self, indexer, value, name)
   1965 self.obj._check_is_chained_assignment_possible()
   1967 # actually do the set
-> 1968 self.obj._mgr = self.obj._mgr.setitem(indexer=indexer, value=value)
   1969 self.obj._maybe_update_cacher(clear=True)

File /usr/local/lib/python3.9/site-packages/pandas/core/internals/managers.py:355, in BaseBlockManager.setitem(self, indexer, value)
    354 def setitem(self: T, indexer, value) -> T:
--> 355     return self.apply("setitem", indexer=indexer, value=value)

File /usr/local/lib/python3.9/site-packages/pandas/core/internals/managers.py:327, in BaseBlockManager.apply(self, f, align_keys, ignore_failures, **kwargs)
    325         applied = b.apply(f, **kwargs)
    326     else:
--> 327         applied = getattr(b, f)(**kwargs)
    328 except (TypeError, NotImplementedError):
    329     if not ignore_failures:

File /usr/local/lib/python3.9/site-packages/pandas/core/internals/blocks.py:1491, in ExtensionBlock.setitem(self, indexer, value)
   1488     indexer = indexer[0]
   1490 check_setitem_lengths(indexer, value, self.values)
-> 1491 self.values[indexer] = value
   1492 return self

File /usr/local/lib/python3.9/site-packages/pandas/core/arrays/string_.py:413, in StringArray.__setitem__(self, key, value)
    411         value = np.asarray(value, dtype=object)
    412     if len(value) and not lib.is_string_array(value, skipna=True):
--> 413         raise ValueError("Must provide strings.")
    415 super().__setitem__(key, value)

ValueError: Must provide strings.

@modin-project/modin-core I have filed pandas-dev/pandas#45469 for the pandas bug. Meanwhile, should we create a fix that works around the pandas bug?

[mvashishtha]

I'm working on this now. Just to further clarify, the bug seems to only apply to dtype string, and it appears when assigning the scalars None or np.NaN. A simple reproducer is:

import modin.pandas as pd
df = pd.DataFrame([["A", "B"], ["C", "D"]], dtype="string")
df.iloc[0, 0] = None

But it also gets triggered by iloc in another form:

import modin.pandas as pd
df = pd.DataFrame([["A", "B"], ["C", "D"]], dtype="string")
df.iloc[[0, 1], [0, 1]] = None

and the bug applies to string Series as well.

and the corresponding