diff --git a/pandas/core/common.py b/pandas/core/common.py index e0dc420bc53f8c..b0fe169c99f29c 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -233,6 +233,20 @@ def _any_none(*args): return False +def _all_none(*args): + for arg in args: + if arg is not None: + return False + return True + + +def _any_not_none(*args): + for arg in args: + if arg is not None: + return True + return False + + def _all_not_none(*args): for arg in args: if arg is None: @@ -459,13 +473,6 @@ def _apply_if_callable(maybe_callable, obj, **kwargs): return maybe_callable -def _all_none(*args): - for arg in args: - if arg is not None: - return False - return True - - def _where_compat(mask, arr1, arr2): if arr1.dtype == _NS_DTYPE and arr2.dtype == _NS_DTYPE: new_vals = np.where(mask, arr1.view('i8'), arr2.view('i8')) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index c7ae9bbee90130..8da2cb49edfe69 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -29,10 +29,9 @@ from pandas.core.dtypes.missing import isna, notna from pandas.core.dtypes.generic import ABCSeries, ABCPanel, ABCDataFrame -from pandas.core.common import (_values_from_object, - _maybe_box_datetimelike, - SettingWithCopyError, SettingWithCopyWarning, - AbstractMethodError) +from pandas.core.common import (_count_not_none, _maybe_box_datetimelike, + _values_from_object, AbstractMethodError, + SettingWithCopyError, SettingWithCopyWarning) from pandas.core.base import PandasObject, SelectionMixin from pandas.core.index import (Index, MultiIndex, _ensure_index, @@ -3126,7 +3125,7 @@ def filter(self, items=None, like=None, regex=None, axis=None): """ import re - nkw = sum([x is not None for x in [items, like, regex]]) + nkw = _count_not_none(items, like, regex) if nkw > 1: raise TypeError('Keyword arguments `items`, `like`, or `regex` ' 'are mutually exclusive') diff --git a/pandas/core/groupby.py b/pandas/core/groupby.py index 9518f17e5f4f17..87b9e55624ac87 100644 --- a/pandas/core/groupby.py +++ b/pandas/core/groupby.py @@ -39,7 +39,8 @@ from pandas.core.dtypes.missing import isna, notna, _maybe_fill from pandas.core.common import (_values_from_object, AbstractMethodError, - _default_index) + _default_index, _not_none, _get_callable_name, + _asarray_tuplesafe) from pandas.core.base import (PandasObject, SelectionMixin, GroupByError, DataError, SpecificationError) @@ -60,7 +61,6 @@ from pandas.util._validators import validate_kwargs import pandas.core.algorithms as algorithms -import pandas.core.common as com from pandas.core.config import option_context from pandas.plotting._core import boxplot_frame_groupby @@ -877,10 +877,9 @@ def _concat_objects(self, keys, values, not_indexed_same=False): def reset_identity(values): # reset the identities of the components # of the values to prevent aliasing - for v in values: - if v is not None: - ax = v._get_axis(self.axis) - ax._reset_identity() + for v in _not_none(*values): + ax = v._get_axis(self.axis) + ax._reset_identity() return values if not not_indexed_same: @@ -1806,7 +1805,7 @@ def apply(self, f, data, axis=0): group_keys = self._get_group_keys() # oh boy - f_name = com._get_callable_name(f) + f_name = _get_callable_name(f) if (f_name not in _plotting_methods and hasattr(splitter, 'fast_apply') and axis == 0): try: @@ -2533,7 +2532,7 @@ def __init__(self, index, grouper=None, obj=None, name=None, level=None, self.grouper = self.obj[self.name] elif isinstance(self.grouper, (list, tuple)): - self.grouper = com._asarray_tuplesafe(self.grouper) + self.grouper = _asarray_tuplesafe(self.grouper) # a passed Categorical elif is_categorical_dtype(self.grouper): @@ -2739,7 +2738,7 @@ def _get_grouper(obj, key=None, axis=0, level=None, sort=True, if not any_callable and not all_in_columns_index and \ not any_arraylike and not any_groupers and \ match_axis_length and level is None: - keys = [com._asarray_tuplesafe(keys)] + keys = [_asarray_tuplesafe(keys)] if isinstance(level, (tuple, list)): if key is None: @@ -3028,7 +3027,7 @@ def _aggregate_multiple_funcs(self, arg, _level): columns.append(f) else: # protect against callables without names - columns.append(com._get_callable_name(f)) + columns.append(_get_callable_name(f)) arg = lzip(columns, arg) results = {} @@ -3686,14 +3685,13 @@ def _wrap_applied_output(self, keys, values, not_indexed_same=False): key_names = self.grouper.names # GH12824. - def first_non_None_value(values): + def first_not_none(values): try: - v = next(v for v in values if v is not None) + return next(_not_none(*values)) except StopIteration: return None - return v - v = first_non_None_value(values) + v = first_not_none(values) if v is None: # GH9684. If all values are None, then this will throw an error. @@ -3726,7 +3724,7 @@ def first_non_None_value(values): key_index = None # make Nones an empty object - v = first_non_None_value(values) + v = first_not_none(values) if v is None: return DataFrame() elif isinstance(v, NDFrame): diff --git a/pandas/core/indexes/api.py b/pandas/core/indexes/api.py index d20a0b0a2c73df..08cda8a06ba64f 100644 --- a/pandas/core/indexes/api.py +++ b/pandas/core/indexes/api.py @@ -123,7 +123,7 @@ def _get_consensus_names(indexes): # find the non-none names, need to tupleify to make # the set hashable, then reverse on return consensus_names = set([tuple(i.names) for i in indexes - if any(n is not None for n in i.names)]) + if com._any_not_none(*i.names)]) if len(consensus_names) == 1: return list(list(consensus_names)[0]) return [None] * indexes[0].nlevels diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py index 0a55559750d7c8..79edf05318b5df 100644 --- a/pandas/core/indexes/base.py +++ b/pandas/core/indexes/base.py @@ -42,16 +42,15 @@ needs_i8_conversion, is_iterator, is_list_like, is_scalar) -from pandas.core.common import (is_bool_indexer, - _values_from_object, - _asarray_tuplesafe) +from pandas.core.common import (is_bool_indexer, _values_from_object, + _asarray_tuplesafe, _not_none, + _index_labels_to_array) from pandas.core.base import PandasObject, IndexOpsMixin import pandas.core.base as base from pandas.util._decorators import ( Appender, Substitution, cache_readonly, deprecate_kwarg) from pandas.core.indexes.frozen import FrozenList -import pandas.core.common as com import pandas.core.dtypes.concat as _concat import pandas.core.missing as missing import pandas.core.algorithms as algos @@ -3145,8 +3144,8 @@ def _join_multi(self, other, how, return_indexers=True): other_is_mi = isinstance(other, MultiIndex) # figure out join names - self_names = [n for n in self.names if n is not None] - other_names = [n for n in other.names if n is not None] + self_names = _not_none(*self.names) + other_names = _not_none(*other.names) overlap = list(set(self_names) & set(other_names)) # need at least 1 in common, but not more than 1 @@ -3691,7 +3690,7 @@ def drop(self, labels, errors='raise'): ------- dropped : Index """ - labels = com._index_labels_to_array(labels) + labels = _index_labels_to_array(labels) indexer = self.get_indexer(labels) mask = indexer == -1 if mask.any(): diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py index 4b6e31133ba4b0..b075f03f1d8c7f 100644 --- a/pandas/core/indexes/multi.py +++ b/pandas/core/indexes/multi.py @@ -21,7 +21,8 @@ is_scalar) from pandas.core.dtypes.missing import isna, array_equivalent from pandas.errors import PerformanceWarning, UnsortedIndexError -from pandas.core.common import (_values_from_object, +from pandas.core.common import (_any_not_none, + _values_from_object, is_bool_indexer, is_null_slice, is_true_slices) @@ -509,7 +510,7 @@ def _format_attrs(self): max_seq_items=False)), ('labels', ibase.default_pprint(self._labels, max_seq_items=False))] - if not all(name is None for name in self.names): + if _any_not_none(*self.names): attrs.append(('names', ibase.default_pprint(self.names))) if self.sortorder is not None: attrs.append(('sortorder', ibase.default_pprint(self.sortorder))) diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py index 9f7bac641ae08d..b2e55d4826670e 100644 --- a/pandas/core/indexes/range.py +++ b/pandas/core/indexes/range.py @@ -12,6 +12,7 @@ from pandas import compat from pandas.compat import lrange, range from pandas.compat.numpy import function as nv +from pandas.core.common import _all_none from pandas.core.indexes.base import Index, _index_shared_docs from pandas.util._decorators import Appender, cache_readonly import pandas.core.dtypes.concat as _concat @@ -83,7 +84,7 @@ def _ensure_int(value, field): return new_value - if start is None and stop is None and step is None: + if _all_none(start, stop, step): msg = "RangeIndex(...) must be called with integers" raise TypeError(msg) elif start is None: diff --git a/pandas/core/panel.py b/pandas/core/panel.py index 14fba9560cae25..5c7efb678796fa 100644 --- a/pandas/core/panel.py +++ b/pandas/core/panel.py @@ -15,13 +15,13 @@ is_string_like, is_scalar) from pandas.core.dtypes.missing import notna -import pandas.core.common as com import pandas.core.ops as ops import pandas.core.missing as missing from pandas import compat from pandas.compat import (map, zip, range, u, OrderedDict) from pandas.compat.numpy import function as nv -from pandas.core.common import _try_sort, _default_index +from pandas.core.common import (_try_sort, _default_index, _all_not_none, + _any_not_none, _apply_if_callable) from pandas.core.frame import DataFrame from pandas.core.generic import NDFrame, _shared_docs from pandas.core.index import (Index, MultiIndex, _ensure_index, @@ -165,7 +165,7 @@ def _init_data(self, data, copy, dtype, **kwargs): axes = None if isinstance(data, BlockManager): - if any(x is not None for x in passed_axes): + if _any_not_none(*passed_axes): axes = [x if x is not None else y for x, y in zip(passed_axes, data.axes)] mgr = data @@ -177,7 +177,7 @@ def _init_data(self, data, copy, dtype, **kwargs): mgr = self._init_matrix(data, passed_axes, dtype=dtype, copy=copy) copy = False dtype = None - elif is_scalar(data) and all(x is not None for x in passed_axes): + elif is_scalar(data) and _all_not_none(*passed_axes): values = cast_scalar_to_array([len(x) for x in passed_axes], data, dtype=dtype) mgr = self._init_matrix(values, passed_axes, dtype=values.dtype, @@ -278,7 +278,7 @@ def from_dict(cls, data, intersect=False, orient='items', dtype=None): return cls(**d) def __getitem__(self, key): - key = com._apply_if_callable(key, self) + key = _apply_if_callable(key, self) if isinstance(self._info_axis, MultiIndex): return self._getitem_multilevel(key) @@ -593,7 +593,7 @@ def _box_item_values(self, key, values): return self._constructor_sliced(values, **d) def __setitem__(self, key, value): - key = com._apply_if_callable(key, self) + key = _apply_if_callable(key, self) shape = tuple(self.shape) if isinstance(value, self._constructor_sliced): value = value.reindex( @@ -615,7 +615,9 @@ def __setitem__(self, key, value): def _unpickle_panel_compat(self, state): # pragma: no cover "Unpickle the panel" - _unpickle = com._unpickle_array + from pandas.io.pickle import _unpickle_array + + _unpickle = _unpickle_array vals, items, major, minor = state items = _unpickle(items) diff --git a/pandas/core/reshape/concat.py b/pandas/core/reshape/concat.py index 4040c651366174..4c3c662c87373c 100644 --- a/pandas/core/reshape/concat.py +++ b/pandas/core/reshape/concat.py @@ -241,7 +241,7 @@ def __init__(self, objs, axis=0, join='outer', join_axes=None, raise ValueError('No objects to concatenate') if keys is None: - objs = [obj for obj in objs if obj is not None] + objs = list(com._not_none(*objs)) else: # #1649 clean_keys = [] diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py index 6bb6988a7442a1..e409090e76944b 100644 --- a/pandas/core/reshape/merge.py +++ b/pandas/core/reshape/merge.py @@ -1550,4 +1550,4 @@ def _should_fill(lname, rname): def _any(x): - return x is not None and len(x) > 0 and any([y is not None for y in x]) + return x is not None and com._any_not_none(*x) diff --git a/pandas/core/series.py b/pandas/core/series.py index 49b6a6651367b8..a06a5c108a0e86 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -45,7 +45,8 @@ SettingWithCopyError, _maybe_box_datetimelike, _dict_compat, - standardize_mapping) + standardize_mapping, + _any_none) from pandas.core.index import (Index, MultiIndex, InvalidIndexError, Float64Index, _ensure_index) from pandas.core.indexing import check_bool_indexer, maybe_convert_indices @@ -713,7 +714,7 @@ def _get_with(self, key): def _get_values_tuple(self, key): # mpl hackaround - if any(k is None for k in key): + if _any_none(*key): return self._get_values(key) if not isinstance(self.index, MultiIndex): diff --git a/pandas/core/window.py b/pandas/core/window.py index 869296503225d0..7caa257c8ca66e 100644 --- a/pandas/core/window.py +++ b/pandas/core/window.py @@ -32,7 +32,7 @@ from pandas.core.base import (PandasObject, SelectionMixin, GroupByMixin) -import pandas.core.common as com +from pandas.core.common import _asarray_tuplesafe, _count_not_none import pandas._libs.window as _window from pandas import compat @@ -535,7 +535,7 @@ def _prep_window(self, **kwargs): window = self._get_window() if isinstance(window, (list, tuple, np.ndarray)): - return com._asarray_tuplesafe(window).astype(float) + return _asarray_tuplesafe(window).astype(float) elif is_integer(window): import scipy.signal as sig @@ -1972,8 +1972,7 @@ def dataframe_from_int_dict(data, frame_template): def _get_center_of_mass(com, span, halflife, alpha): - valid_count = len([x for x in [com, span, halflife, alpha] - if x is not None]) + valid_count = _count_not_none(com, span, halflife, alpha) if valid_count > 1: raise ValueError("com, span, halflife, and alpha " "are mutually exclusive") diff --git a/pandas/io/formats/excel.py b/pandas/io/formats/excel.py index 9e888c38edaa7d..af24537cabf90a 100644 --- a/pandas/io/formats/excel.py +++ b/pandas/io/formats/excel.py @@ -10,6 +10,7 @@ from pandas.compat import reduce from pandas.io.formats.css import CSSResolver, CSSWarning from pandas.io.formats.printing import pprint_thing +from pandas.core.common import _any_not_none from pandas.core.dtypes.common import is_float import pandas._libs.lib as lib from pandas import Index, MultiIndex, PeriodIndex @@ -548,8 +549,7 @@ def _format_hierarchical_rows(self): self.rowcounter += 1 # if index labels are not empty go ahead and dump - if (any(x is not None for x in index_labels) and - self.header is not False): + if _any_not_none(*index_labels) and self.header is not False: for cidx, name in enumerate(index_labels): yield ExcelCell(self.rowcounter - 1, cidx, name, diff --git a/pandas/io/formats/format.py b/pandas/io/formats/format.py index e8ea0714b1dda1..e6b2a115195001 100644 --- a/pandas/io/formats/format.py +++ b/pandas/io/formats/format.py @@ -26,6 +26,7 @@ is_list_like) from pandas.core.dtypes.generic import ABCSparseArray from pandas.core.base import PandasObject +from pandas.core.common import _any_not_none, sentinel_factory from pandas.core.index import Index, MultiIndex, _ensure_index from pandas import compat from pandas.compat import (StringIO, lzip, range, map, zip, u, @@ -36,7 +37,6 @@ _stringify_path) from pandas.io.formats.printing import adjoin, justify, pprint_thing from pandas.io.formats.common import get_level_lengths -import pandas.core.common as com import pandas._libs.lib as lib from pandas._libs.tslib import (iNaT, Timestamp, Timedelta, format_array_from_datetime) @@ -1257,7 +1257,7 @@ def _column_header(): if self.fmt.sparsify: # GH3547 - sentinel = com.sentinel_factory() + sentinel = sentinel_factory() else: sentinel = None levels = self.columns.format(sparsify=sentinel, adjoin=False, @@ -1426,7 +1426,7 @@ def _write_hierarchical_rows(self, fmt_values, indent): if self.fmt.sparsify: # GH3547 - sentinel = com.sentinel_factory() + sentinel = sentinel_factory() levels = frame.index.format(sparsify=sentinel, adjoin=False, names=False) @@ -2344,7 +2344,7 @@ def single_row_table(row): # pragma: no cover def _has_names(index): if isinstance(index, MultiIndex): - return any([x is not None for x in index.names]) + return _any_not_none(*index.names) else: return index.name is not None diff --git a/pandas/io/formats/style.py b/pandas/io/formats/style.py index d7677e3642c26e..2e87b3b925eddf 100644 --- a/pandas/io/formats/style.py +++ b/pandas/io/formats/style.py @@ -27,7 +27,7 @@ from pandas.compat import range from pandas.core.config import get_option from pandas.core.generic import _shared_docs -import pandas.core.common as com +from pandas.core.common import _any_not_none, sentinel_factory from pandas.core.indexing import _maybe_numeric_slice, _non_reducing_slice from pandas.util._decorators import Appender try: @@ -259,8 +259,7 @@ def format_attr(pair): row_es.append(es) head.append(row_es) - if self.data.index.names and not all(x is None - for x in self.data.index.names): + if self.data.index.names and _any_not_none(*self.data.index.names): index_header_row = [] for c, name in enumerate(self.data.index.names): @@ -1204,7 +1203,7 @@ def _get_level_lengths(index): Result is a dictionary of (level, inital_position): span """ - sentinel = com.sentinel_factory() + sentinel = sentinel_factory() levels = index.format(sparsify=sentinel, adjoin=False, names=False) if index.nlevels == 1: diff --git a/pandas/io/json/table_schema.py b/pandas/io/json/table_schema.py index c3865afa9c0c05..9cec5b3d6ba498 100644 --- a/pandas/io/json/table_schema.py +++ b/pandas/io/json/table_schema.py @@ -3,6 +3,7 @@ http://specs.frictionlessdata.io/json-table-schema/ """ +from pandas.core.common import _all_not_none from pandas.core.dtypes.common import ( is_integer_dtype, is_timedelta64_dtype, is_numeric_dtype, is_bool_dtype, is_datetime64_dtype, is_datetime64tz_dtype, @@ -61,7 +62,7 @@ def as_json_table_type(x): def set_default_names(data): """Sets index names to 'index' for regular, or 'level_x' for Multi""" - if all(name is not None for name in data.index.names): + if _all_not_none(*data.index.names): return data data = data.copy() diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index ca1b4d031d3ced..ae00bbb488d5be 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -34,7 +34,7 @@ from pandas.core.base import StringMixin from pandas.io.formats.printing import adjoin, pprint_thing from pandas.errors import PerformanceWarning -from pandas.core.common import _asarray_tuplesafe +from pandas.core.common import _asarray_tuplesafe, _all_none from pandas.core.algorithms import match, unique from pandas.core.categorical import Categorical, _factorize_from_iterables from pandas.core.internals import (BlockManager, make_block, @@ -905,7 +905,7 @@ def remove(self, key, where=None, start=None, stop=None): raise KeyError('No object named %s in the file' % key) # remove the node - if where is None and start is None and stop is None: + if _all_none(where, start, stop): s.group._f_remove(recursive=True) # delete from the table @@ -2363,7 +2363,7 @@ def delete(self, where=None, start=None, stop=None, **kwargs): support fully deleting the node in its entirety (only) - where specification must be None """ - if where is None and start is None and stop is None: + if _all_none(where, start, stop): self._handle.remove_node(self.group, recursive=True) return None diff --git a/pandas/plotting/_core.py b/pandas/plotting/_core.py index 211d9777e7515d..16712f0626307f 100644 --- a/pandas/plotting/_core.py +++ b/pandas/plotting/_core.py @@ -20,7 +20,7 @@ is_iterator) from pandas.core.dtypes.generic import ABCSeries -from pandas.core.common import AbstractMethodError, _try_sort +from pandas.core.common import AbstractMethodError, _try_sort, _any_not_none from pandas.core.generic import _shared_docs, _shared_doc_kwargs from pandas.core.index import Index, MultiIndex @@ -607,7 +607,7 @@ def _plot(cls, ax, x, y, style=None, is_errorbar=False, **kwds): def _get_index_name(self): if isinstance(self.data.index, MultiIndex): name = self.data.index.names - if any(x is not None for x in name): + if _any_not_none(*name): name = ','.join([pprint_thing(x) for x in name]) else: name = None @@ -955,7 +955,7 @@ def _make_plot(self): it = self._iter_data() stacking_id = self._get_stacking_id() - is_errorbar = any(e is not None for e in self.errors.values()) + is_errorbar = _any_not_none(*self.errors.values()) colors = self._get_colors() for i, (label, y) in enumerate(it): diff --git a/pandas/tests/frame/test_to_csv.py b/pandas/tests/frame/test_to_csv.py index 6a4b1686a31e25..f18e199860ec64 100644 --- a/pandas/tests/frame/test_to_csv.py +++ b/pandas/tests/frame/test_to_csv.py @@ -9,6 +9,7 @@ import numpy as np from pandas.compat import (lmap, range, lrange, StringIO, u) +from pandas.core.common import _all_none from pandas.errors import ParserError from pandas import (DataFrame, Index, Series, MultiIndex, Timestamp, date_range, read_csv, compat, to_datetime) @@ -548,7 +549,7 @@ def _make_frame(names=None): df = _make_frame(True) df.to_csv(path, tupleize_cols=False, index=False) result = read_csv(path, header=[0, 1], tupleize_cols=False) - assert all([x is None for x in result.columns.names]) + assert _all_none(*result.columns.names) result.columns.names = df.columns.names assert_frame_equal(df, result) diff --git a/pandas/tests/util/test_util.py b/pandas/tests/util/test_util.py index ffc9703abff41d..659ce36de6babf 100644 --- a/pandas/tests/util/test_util.py +++ b/pandas/tests/util/test_util.py @@ -8,6 +8,7 @@ import pytest from pandas.compat import intern +from pandas.core.common import _all_none from pandas.util._move import move_into_mutable_buffer, BadMove, stolenbuf from pandas.util._decorators import deprecate_kwarg, make_signature from pandas.util._validators import (validate_args, validate_kwargs, @@ -437,7 +438,7 @@ def test_set_locale(self): pytest.skip("Only a single locale found, no point in " "trying to test setting another locale") - if all(x is None for x in self.current_locale): + if _all_none(*self.current_locale): # Not sure why, but on some travis runs with pytest, # getlocale() returned (None, None). pytest.skip("Current locale is not set.") diff --git a/pandas/util/testing.py b/pandas/util/testing.py index 202c9473eea12e..718106fd45974d 100644 --- a/pandas/util/testing.py +++ b/pandas/util/testing.py @@ -33,6 +33,7 @@ is_list_like) from pandas.io.formats.printing import pprint_thing from pandas.core.algorithms import take_1d +from pandas.core.common import _all_not_none import pandas.compat as compat from pandas.compat import ( @@ -594,7 +595,7 @@ def set_locale(new_locale, lc_var=locale.LC_ALL): except ValueError: yield new_locale else: - if all(lc is not None for lc in normalized_locale): + if _all_not_none(*normalized_locale): yield '.'.join(normalized_locale) else: yield new_locale