From 662eea24882d4eb12f63c86478074894123ba5e3 Mon Sep 17 00:00:00 2001 From: David Krych Date: Thu, 14 Jun 2018 11:32:12 -0400 Subject: [PATCH 1/7] BUG: Fix Index construction when given empty generator (#21470). --- pandas/core/indexes/base.py | 17 ++++++++++------- pandas/tests/indexes/test_base.py | 10 +++++++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 4b32e5d4f5654..caca5929138b2 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -430,15 +430,18 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None, elif data is None or is_scalar(data): cls._scalar_data_error(data) else: - if tupleize_cols and is_list_like(data) and data: + if tupleize_cols and is_list_like(data): + # GH21470: convert iterable to list before determining if empty if is_iterator(data): data = list(data) - # we must be all tuples, otherwise don't construct - # 10697 - if all(isinstance(e, tuple) for e in data): - from .multi import MultiIndex - return MultiIndex.from_tuples( - data, names=name or kwargs.get('names')) + + if data: + # we must be all tuples, otherwise don't construct + # 10697 + if all(isinstance(e, tuple) for e in data): + from .multi import MultiIndex + return MultiIndex.from_tuples( + data, names=name or kwargs.get('names')) # other iterable of some kind subarr = com._asarray_tuplesafe(data, dtype=object) return Index(subarr, dtype=dtype, copy=copy, name=name, **kwargs) diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index b8bd218ec25ab..b9dbfcfb8311d 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -445,7 +445,7 @@ def test_constructor_dtypes_timedelta(self, attr, klass): result = klass(list(values), dtype=dtype) tm.assert_index_equal(result, index) - def test_constructor_empty_gen(self): + def test_constructor_empty_list(self): skip_index_keys = ["repeats", "periodIndex", "rangeIndex", "tuples"] for key, index in self.generate_index_types(skip_index_keys): @@ -453,6 +453,14 @@ def test_constructor_empty_gen(self): assert isinstance(empty, index.__class__) assert not len(empty) + def test_constructor_empty_gen(self): + skip_index_keys = ["repeats", "periodIndex", "rangeIndex", + "tuples"] + for key, index in self.generate_index_types(skip_index_keys): + empty = index.__class__(x for x in []) + assert isinstance(empty, index.__class__) + assert not len(empty) + @pytest.mark.parametrize("empty,klass", [ (PeriodIndex([], freq='B'), PeriodIndex), (RangeIndex(step=1), pd.RangeIndex), From e4707452ec7fff9fabee2a6fbca49a33be045500 Mon Sep 17 00:00:00 2001 From: David Krych Date: Thu, 14 Jun 2018 12:45:51 -0400 Subject: [PATCH 2/7] Fix trailing whitespace --- pandas/core/indexes/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index caca5929138b2..5f1726736ce4b 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -431,7 +431,7 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None, cls._scalar_data_error(data) else: if tupleize_cols and is_list_like(data): - # GH21470: convert iterable to list before determining if empty + # GH21470: convert iterable to list before determining if empty if is_iterator(data): data = list(data) From bf81721474142c488523f50f875232761764783b Mon Sep 17 00:00:00 2001 From: David Krych Date: Thu, 14 Jun 2018 13:24:18 -0400 Subject: [PATCH 3/7] Don't nest if statements. --- pandas/core/indexes/base.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 5f1726736ce4b..62614aac178f6 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -435,13 +435,12 @@ def __new__(cls, data=None, dtype=None, copy=False, name=None, if is_iterator(data): data = list(data) - if data: + if data and all(isinstance(e, tuple) for e in data): # we must be all tuples, otherwise don't construct # 10697 - if all(isinstance(e, tuple) for e in data): - from .multi import MultiIndex - return MultiIndex.from_tuples( - data, names=name or kwargs.get('names')) + from .multi import MultiIndex + return MultiIndex.from_tuples( + data, names=name or kwargs.get('names')) # other iterable of some kind subarr = com._asarray_tuplesafe(data, dtype=object) return Index(subarr, dtype=dtype, copy=copy, name=name, **kwargs) From b0d9d2a365eabc63fa41e98dd4ee4f4b0f36a520 Mon Sep 17 00:00:00 2001 From: David Krych Date: Thu, 14 Jun 2018 16:07:59 -0400 Subject: [PATCH 4/7] Fix CategoricalIndex constructor with iterator; test empty iterators. --- pandas/core/arrays/categorical.py | 4 ++-- pandas/tests/indexes/test_base.py | 27 +++++++++++---------------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index d466198b648ef..200000a5313e6 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -28,7 +28,7 @@ is_categorical, is_categorical_dtype, is_list_like, is_sequence, - is_scalar, + is_scalar, is_iterator, is_dict_like) from pandas.core.algorithms import factorize, take_1d, unique1d, take @@ -2483,7 +2483,7 @@ def _convert_to_list_like(list_like): if isinstance(list_like, list): return list_like if (is_sequence(list_like) or isinstance(list_like, tuple) or - isinstance(list_like, types.GeneratorType)): + is_iterator(list_like)): return list(list_like) elif is_scalar(list_like): return [list_like] diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py index b9dbfcfb8311d..1d8a958c3413f 100644 --- a/pandas/tests/indexes/test_base.py +++ b/pandas/tests/indexes/test_base.py @@ -445,29 +445,24 @@ def test_constructor_dtypes_timedelta(self, attr, klass): result = klass(list(values), dtype=dtype) tm.assert_index_equal(result, index) - def test_constructor_empty_list(self): - skip_index_keys = ["repeats", "periodIndex", "rangeIndex", - "tuples"] - for key, index in self.generate_index_types(skip_index_keys): - empty = index.__class__([]) - assert isinstance(empty, index.__class__) - assert not len(empty) - - def test_constructor_empty_gen(self): - skip_index_keys = ["repeats", "periodIndex", "rangeIndex", - "tuples"] - for key, index in self.generate_index_types(skip_index_keys): - empty = index.__class__(x for x in []) - assert isinstance(empty, index.__class__) - assert not len(empty) + @pytest.mark.parametrize("value", [[], iter([]), (x for x in [])]) + @pytest.mark.parametrize("klass", + [Index, Float64Index, Int64Index, UInt64Index, + CategoricalIndex, DatetimeIndex, TimedeltaIndex]) + def test_constructor_empty(self, value, klass): + empty = klass(value) + assert isinstance(empty, klass) + assert not len(empty) @pytest.mark.parametrize("empty,klass", [ (PeriodIndex([], freq='B'), PeriodIndex), + (PeriodIndex(iter([]), freq='B'), PeriodIndex), + (PeriodIndex((x for x in []), freq='B'), PeriodIndex), (RangeIndex(step=1), pd.RangeIndex), (MultiIndex(levels=[[1, 2], ['blue', 'red']], labels=[[], []]), MultiIndex) ]) - def test_constructor_empty(self, empty, klass): + def test_constructor_empty_special(self, empty, klass): assert isinstance(empty, klass) assert not len(empty) From 254676c2617a2bb76830248d5d7faade8c005c85 Mon Sep 17 00:00:00 2001 From: David Krych Date: Fri, 15 Jun 2018 09:49:33 -0400 Subject: [PATCH 5/7] Remove unused import. --- pandas/core/arrays/categorical.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py index 200000a5313e6..e22b0d626a218 100644 --- a/pandas/core/arrays/categorical.py +++ b/pandas/core/arrays/categorical.py @@ -3,7 +3,6 @@ import numpy as np from warnings import warn import textwrap -import types from pandas import compat from pandas.compat import u, lzip From ad2c0cb911423ff805a8cfa74eb0e1d913a6476c Mon Sep 17 00:00:00 2001 From: David Krych Date: Fri, 15 Jun 2018 14:16:40 -0400 Subject: [PATCH 6/7] Add whatsnew. --- doc/source/whatsnew/v0.23.2.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/source/whatsnew/v0.23.2.txt b/doc/source/whatsnew/v0.23.2.txt index 79a4c3da2ffa4..0a7852b63977e 100644 --- a/doc/source/whatsnew/v0.23.2.txt +++ b/doc/source/whatsnew/v0.23.2.txt @@ -47,8 +47,7 @@ Bug Fixes **Conversion** - -- +- Bug in constructing :class:`Index` with an iterator or generator (:issue:`21470`) - **Indexing** From 650f810f34781197d3e84f8db1e786a77f2e4ff8 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Mon, 18 Jun 2018 18:41:33 -0400 Subject: [PATCH 7/7] doc --- doc/source/whatsnew/v0.23.2.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.23.2.txt b/doc/source/whatsnew/v0.23.2.txt index 764d942371ea9..94669c5b02410 100644 --- a/doc/source/whatsnew/v0.23.2.txt +++ b/doc/source/whatsnew/v0.23.2.txt @@ -48,11 +48,11 @@ Bug Fixes **Timedelta** -- Bug in constructing :class:`Index` with an iterator or generator (:issue:`21470`) - Bug in :class:`Timedelta` where non-zero timedeltas shorter than 1 microsecond were considered False (:issue:`21484`) **Conversion** +- Bug in constructing :class:`Index` with an iterator or generator (:issue:`21470`) - Bug in :meth:`Series.nlargest` for signed and unsigned integer dtypes when the minimum value is present (:issue:`21426`)