Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TYP: io.json._json, util._decorators #36903

Merged
merged 12 commits into from
Oct 10, 2020
42 changes: 30 additions & 12 deletions pandas/io/json/_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from io import BytesIO, StringIO
from itertools import islice
import os
from typing import IO, Any, Callable, List, Optional, Type
from typing import IO, Any, Callable, List, Optional, Tuple, Type

import numpy as np

Expand Down Expand Up @@ -777,8 +777,8 @@ def read(self):
obj = self._get_object_parser(lines_json)
else:
data = ensure_str(self.data)
data = data.split("\n")
obj = self._get_object_parser(self._combine_lines(data))
data_lines = data.split("\n")
obj = self._get_object_parser(self._combine_lines(data_lines))
else:
obj = self._get_object_parser(self.data)
self.close()
Expand Down Expand Up @@ -848,6 +848,8 @@ def __next__(self):


class Parser:
_split_keys: Tuple[str, ...]
_default_orient: str
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can do it here or as follow-on, but we should do the same for Writer and remove the ignore there.


_STAMP_UNITS = ("s", "ms", "us", "ns")
_MIN_STAMPS = {
Expand All @@ -873,6 +875,7 @@ def __init__(

if orient is None:
orient = self._default_orient

self.orient = orient

self.dtype = dtype
Expand Down Expand Up @@ -902,8 +905,8 @@ def check_keys_split(self, decoded):
"""
bad_keys = set(decoded.keys()).difference(set(self._split_keys))
if bad_keys:
bad_keys = ", ".join(bad_keys)
raise ValueError(f"JSON data had unexpected key(s): {bad_keys}")
bad_keys_joined = ", ".join(bad_keys)
raise ValueError(f"JSON data had unexpected key(s): {bad_keys_joined}")

def parse(self):

Expand All @@ -922,14 +925,22 @@ def parse(self):
self._try_convert_types()
return self.obj

def _parse_numpy(self):
raise AbstractMethodError(self)

def _parse_no_numpy(self):
raise AbstractMethodError(self)

def _convert_axes(self):
"""
Try to convert axes.
"""
for axis_name in self.obj._AXIS_ORDERS:
obj = self.obj
assert obj is not None # for mypy
for axis_name in obj._AXIS_ORDERS:
new_axis, result = self._try_convert_data(
name=axis_name,
data=self.obj._get_axis(axis_name),
data=obj._get_axis(axis_name),
use_dtypes=False,
convert_dates=True,
)
Expand Down Expand Up @@ -1083,7 +1094,11 @@ def _parse_numpy(self):
self.check_keys_split(decoded)
self.obj = create_series_with_explicit_dtype(**decoded)
elif self.orient in ["columns", "index"]:
self.obj = create_series_with_explicit_dtype(*data, dtype_if_empty=object)
# error: "create_series_with_explicit_dtype"
# gets multiple values for keyword argument "dtype_if_empty
self.obj = create_series_with_explicit_dtype(
*data, dtype_if_empty=object
) # type:ignore[misc]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the default for dtype_if_empty is object, so do we need to pass dtype_if_empty.

if data has more than six items, this would indeed raise.

>>> def func(a=None, b=None, c=object):
...     print(c)
...
>>> data = [1, 2]
>>> func(*data, c=20)
20
>>>
>>> data = [1, 2, 3]
>>> func(*data, c=20)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: func() got multiple values for argument 'c'
>>>

do we know what data contains?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not off the top of my head. @WillAyd any idea?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure. Assuming this path comes from the numpy keyword we've deprecated that anyway, so not worth investing a ton of time in #30636

else:
self.obj = create_series_with_explicit_dtype(data, dtype_if_empty=object)

Expand Down Expand Up @@ -1175,9 +1190,12 @@ def _process_converter(self, f, filt=None):
if filt is None:
filt = lambda col, c: True

obj = self.obj
assert obj is not None # for mypy

needs_new_obj = False
new_obj = dict()
for i, (col, c) in enumerate(self.obj.items()):
for i, (col, c) in enumerate(obj.items()):
if filt(col, c):
new_data, result = f(col, c)
if result:
Expand All @@ -1188,9 +1206,9 @@ def _process_converter(self, f, filt=None):
if needs_new_obj:

# possibly handle dup columns
new_obj = DataFrame(new_obj, index=self.obj.index)
new_obj.columns = self.obj.columns
self.obj = new_obj
new_frame = DataFrame(new_obj, index=obj.index)
new_frame.columns = obj.columns
self.obj = new_frame

def _try_convert_types(self):
if self.obj is None:
Expand Down
4 changes: 2 additions & 2 deletions pandas/plotting/_matplotlib/converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import datetime as pydt
from datetime import datetime, timedelta, tzinfo
import functools
from typing import Any, List, Optional, Tuple
from typing import Any, Dict, List, Optional, Tuple

from dateutil.relativedelta import relativedelta
import matplotlib.dates as dates
Expand Down Expand Up @@ -1002,7 +1002,7 @@ def __init__(
self.format = None
self.freq = freq
self.locs: List[Any] = [] # unused, for matplotlib compat
self.formatdict = None
self.formatdict: Optional[Dict[Any, Any]] = None
self.isminor = minor_locator
self.isdynamic = dynamic_mode
self.offset = 0
Expand Down
3 changes: 2 additions & 1 deletion pandas/util/_decorators.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,8 @@ def decorate(func):
allow_args = allowed_args
else:
spec = inspect.getfullargspec(func)
allow_args = spec.args[: -len(spec.defaults)]
# mypy doesn't know that spec.defaults is Sized
allow_args = spec.args[: -len(spec.defaults)] # type:ignore[arg-type]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the mypy error is pandas\util\_decorators.py:282: error: Argument 1 to "len" has incompatible type "Optional[Tuple[Any, ...]]"; expected "Sized" [arg-type]

(side note: when ignoring I prefer to see the mypy error as a comment, I think it helps when browsing the code for type: ignores to remove as well as during review)

indeed spec.defaults could be None.

>>> def func(a, b, c):
...     pass
...
>>>
>>> inspect.getfullargspec(func)
FullArgSpec(args=['a', 'b', 'c'], varargs=None, varkw=None, defaults=None, kwonlyargs=[], kwonlydefaults=None, annotations={})
>>>

does allowed_args ensure defaults is not None here?

if so, would a assert spec.defaults is not None be more appropriate that an ignore? (with a comment as to why is cannot be None)


@wraps(func)
def wrapper(*args, **kwargs):
Expand Down
6 changes: 0 additions & 6 deletions setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -259,9 +259,6 @@ check_untyped_defs=False
[mypy-pandas.io.html]
check_untyped_defs=False

[mypy-pandas.io.json._json]
check_untyped_defs=False

[mypy-pandas.io.parsers]
check_untyped_defs=False

Expand All @@ -285,6 +282,3 @@ check_untyped_defs=False

[mypy-pandas.plotting._misc]
check_untyped_defs=False

[mypy-pandas.util._decorators]
check_untyped_defs=False