From 629dfecff4da14567438f25eb66ef50a2e907611 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Wed, 7 Apr 2021 09:55:10 -0500 Subject: [PATCH 1/3] Pandas 1.3.0 compatibility --- partd/compatibility.py | 5 +++++ partd/pandas.py | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/partd/compatibility.py b/partd/compatibility.py index 20a217a..aef2966 100644 --- a/partd/compatibility.py +++ b/partd/compatibility.py @@ -1,7 +1,10 @@ from __future__ import absolute_import +from distutils.version import LooseVersion import sys +import pandas as pd + if sys.version_info[0] == 3: from io import StringIO unicode = str @@ -12,3 +15,5 @@ unicode = unicode import cPickle as pickle from Queue import Queue, Empty + +PANDAS_VERSION = LooseVersion(pd.__version__) diff --git a/partd/pandas.py b/partd/pandas.py index db3a208..20b794b 100644 --- a/partd/pandas.py +++ b/partd/pandas.py @@ -8,7 +8,7 @@ from . import numpy as pnp from .core import Interface -from .compatibility import pickle +from .compatibility import pickle, PANDAS_VERSION from .encode import Encode from .utils import extend, framesplit, frame @@ -20,6 +20,18 @@ def is_extension_array_dtype(dtype): return False +if PANDAS_VERSION >= "1.3.0": + # Some `ExtensionArray`s can have a `.dtype` which is not a `ExtensionDtype` + # (e.g. they can be backed by a NumPy dtype). For these cases we check + # whether the instance is a `ExtensionArray`. + # https://github.com/dask/partd/issues/48 + from pandas.api.extensions import ExtensionArray + def is_extension_array(x): + return isinstance(x, ExtensionArray) +else: + def is_extension_array(x): + return False + dumps = partial(pickle.dumps, protocol=pickle.HIGHEST_PROTOCOL) @@ -129,7 +141,7 @@ def block_to_header_bytes(block): elif is_datetime64tz_dtype(block): extension = ('datetime64_tz_type', (block.values.tzinfo,)) values = values.view('i8') - elif is_extension_array_dtype(block.dtype): + elif is_extension_array_dtype(block.dtype) or is_extension_array(values): extension = ("other", ()) else: extension = ('numpy_type', ()) From b6dd754d5af3b8a0b4ce1cadca3df3035742e902 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Wed, 7 Apr 2021 10:21:42 -0500 Subject: [PATCH 2/3] Move pandas import to pandas.py module --- partd/compatibility.py | 4 ---- partd/pandas.py | 4 +++- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/partd/compatibility.py b/partd/compatibility.py index aef2966..c74cd91 100644 --- a/partd/compatibility.py +++ b/partd/compatibility.py @@ -1,10 +1,7 @@ from __future__ import absolute_import -from distutils.version import LooseVersion import sys -import pandas as pd - if sys.version_info[0] == 3: from io import StringIO unicode = str @@ -16,4 +13,3 @@ import cPickle as pickle from Queue import Queue, Empty -PANDAS_VERSION = LooseVersion(pd.__version__) diff --git a/partd/pandas.py b/partd/pandas.py index 20b794b..54ca355 100644 --- a/partd/pandas.py +++ b/partd/pandas.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +from distutils.version import LooseVersion from functools import partial import numpy as np @@ -8,10 +9,11 @@ from . import numpy as pnp from .core import Interface -from .compatibility import pickle, PANDAS_VERSION +from .compatibility import pickle from .encode import Encode from .utils import extend, framesplit, frame +PANDAS_VERSION = LooseVersion(pd.__version__) try: # pandas >= 0.24.0 From 2471238e43c9533e763d77101ce40ca3b97da0a8 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Wed, 7 Apr 2021 10:43:49 -0500 Subject: [PATCH 3/3] Use is_extension_array more broadly --- partd/pandas.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/partd/pandas.py b/partd/pandas.py index 54ca355..6a6da92 100644 --- a/partd/pandas.py +++ b/partd/pandas.py @@ -1,6 +1,5 @@ from __future__ import absolute_import -from distutils.version import LooseVersion from functools import partial import numpy as np @@ -13,8 +12,6 @@ from .encode import Encode from .utils import extend, framesplit, frame -PANDAS_VERSION = LooseVersion(pd.__version__) - try: # pandas >= 0.24.0 from pandas.api.types import is_extension_array_dtype @@ -22,7 +19,7 @@ def is_extension_array_dtype(dtype): return False -if PANDAS_VERSION >= "1.3.0": +try: # Some `ExtensionArray`s can have a `.dtype` which is not a `ExtensionDtype` # (e.g. they can be backed by a NumPy dtype). For these cases we check # whether the instance is a `ExtensionArray`. @@ -30,7 +27,7 @@ def is_extension_array_dtype(dtype): from pandas.api.extensions import ExtensionArray def is_extension_array(x): return isinstance(x, ExtensionArray) -else: +except ImportError: def is_extension_array(x): return False