Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TYP: io.json._json, util._decorators #36903

Merged
merged 12 commits into from
Oct 10, 2020
47 changes: 33 additions & 14 deletions pandas/io/json/_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from io import BytesIO, StringIO
from itertools import islice
import os
from typing import IO, Any, Callable, List, Optional, Type
from typing import IO, Any, Callable, List, Optional, Tuple, Type

import numpy as np

Expand Down Expand Up @@ -111,6 +111,8 @@ def to_json(


class Writer:
_default_orient: str

def __init__(
self,
obj,
Expand All @@ -126,8 +128,7 @@ def __init__(
self.obj = obj

if orient is None:
# error: "Writer" has no attribute "_default_orient"
orient = self._default_orient # type: ignore[attr-defined]
orient = self._default_orient

self.orient = orient
self.date_format = date_format
Expand Down Expand Up @@ -777,8 +778,8 @@ def read(self):
obj = self._get_object_parser(lines_json)
else:
data = ensure_str(self.data)
data = data.split("\n")
obj = self._get_object_parser(self._combine_lines(data))
data_lines = data.split("\n")
obj = self._get_object_parser(self._combine_lines(data_lines))
else:
obj = self._get_object_parser(self.data)
self.close()
Expand Down Expand Up @@ -848,6 +849,8 @@ def __next__(self):


class Parser:
_split_keys: Tuple[str, ...]
_default_orient: str
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can do it here or as follow-on, but we should do the same for Writer and remove the ignore there.


_STAMP_UNITS = ("s", "ms", "us", "ns")
_MIN_STAMPS = {
Expand All @@ -873,6 +876,7 @@ def __init__(

if orient is None:
orient = self._default_orient

self.orient = orient

self.dtype = dtype
Expand Down Expand Up @@ -902,8 +906,8 @@ def check_keys_split(self, decoded):
"""
bad_keys = set(decoded.keys()).difference(set(self._split_keys))
if bad_keys:
bad_keys = ", ".join(bad_keys)
raise ValueError(f"JSON data had unexpected key(s): {bad_keys}")
bad_keys_joined = ", ".join(bad_keys)
raise ValueError(f"JSON data had unexpected key(s): {bad_keys_joined}")

def parse(self):

Expand All @@ -922,14 +926,22 @@ def parse(self):
self._try_convert_types()
return self.obj

def _parse_numpy(self):
raise AbstractMethodError(self)

def _parse_no_numpy(self):
raise AbstractMethodError(self)

def _convert_axes(self):
"""
Try to convert axes.
"""
for axis_name in self.obj._AXIS_ORDERS:
obj = self.obj
assert obj is not None # for mypy
for axis_name in obj._AXIS_ORDERS:
new_axis, result = self._try_convert_data(
name=axis_name,
data=self.obj._get_axis(axis_name),
data=obj._get_axis(axis_name),
use_dtypes=False,
convert_dates=True,
)
Expand Down Expand Up @@ -1083,7 +1095,11 @@ def _parse_numpy(self):
self.check_keys_split(decoded)
self.obj = create_series_with_explicit_dtype(**decoded)
elif self.orient in ["columns", "index"]:
self.obj = create_series_with_explicit_dtype(*data, dtype_if_empty=object)
# error: "create_series_with_explicit_dtype"
# gets multiple values for keyword argument "dtype_if_empty
self.obj = create_series_with_explicit_dtype(
*data, dtype_if_empty=object
) # type:ignore[misc]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the default for dtype_if_empty is object, so do we need to pass dtype_if_empty.

if data has more than six items, this would indeed raise.

>>> def func(a=None, b=None, c=object):
...     print(c)
...
>>> data = [1, 2]
>>> func(*data, c=20)
20
>>>
>>> data = [1, 2, 3]
>>> func(*data, c=20)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: func() got multiple values for argument 'c'
>>>

do we know what data contains?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not off the top of my head. @WillAyd any idea?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure. Assuming this path comes from the numpy keyword we've deprecated that anyway, so not worth investing a ton of time in #30636

else:
self.obj = create_series_with_explicit_dtype(data, dtype_if_empty=object)

Expand Down Expand Up @@ -1175,9 +1191,12 @@ def _process_converter(self, f, filt=None):
if filt is None:
filt = lambda col, c: True

obj = self.obj
assert obj is not None # for mypy

needs_new_obj = False
new_obj = dict()
for i, (col, c) in enumerate(self.obj.items()):
for i, (col, c) in enumerate(obj.items()):
if filt(col, c):
new_data, result = f(col, c)
if result:
Expand All @@ -1188,9 +1207,9 @@ def _process_converter(self, f, filt=None):
if needs_new_obj:

# possibly handle dup columns
new_obj = DataFrame(new_obj, index=self.obj.index)
new_obj.columns = self.obj.columns
self.obj = new_obj
new_frame = DataFrame(new_obj, index=obj.index)
new_frame.columns = obj.columns
self.obj = new_frame

def _try_convert_types(self):
if self.obj is None:
Expand Down
4 changes: 2 additions & 2 deletions pandas/plotting/_matplotlib/converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import datetime as pydt
from datetime import datetime, timedelta, tzinfo
import functools
from typing import Any, List, Optional, Tuple
from typing import Any, Dict, List, Optional, Tuple

from dateutil.relativedelta import relativedelta
import matplotlib.dates as dates
Expand Down Expand Up @@ -1002,7 +1002,7 @@ def __init__(
self.format = None
self.freq = freq
self.locs: List[Any] = [] # unused, for matplotlib compat
self.formatdict = None
self.formatdict: Optional[Dict[Any, Any]] = None
self.isminor = minor_locator
self.isdynamic = dynamic_mode
self.offset = 0
Expand Down
3 changes: 3 additions & 0 deletions pandas/util/_decorators.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,9 @@ def decorate(func):
allow_args = allowed_args
else:
spec = inspect.getfullargspec(func)

# We must have some defaults if we are deprecating default-less
assert spec.defaults is not None # for mypy
allow_args = spec.args[: -len(spec.defaults)]

@wraps(func)
Expand Down
6 changes: 0 additions & 6 deletions setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -259,9 +259,6 @@ check_untyped_defs=False
[mypy-pandas.io.html]
check_untyped_defs=False

[mypy-pandas.io.json._json]
check_untyped_defs=False

[mypy-pandas.io.parsers]
check_untyped_defs=False

Expand All @@ -285,6 +282,3 @@ check_untyped_defs=False

[mypy-pandas.plotting._misc]
check_untyped_defs=False

[mypy-pandas.util._decorators]
check_untyped_defs=False