diff --git a/README.md b/README.md index d94e1b4a..de42db1b 100644 --- a/README.md +++ b/README.md @@ -28,26 +28,27 @@ a trustworthy source. ``dill`` is part of ``pathos``, a python framework for heterogeneous computing. ``dill`` is in active development, so any user feedback, bug reports, comments, -or suggestions are highly appreciated. A list of issues is located at https://github.com/uqfoundation/dill/issues, with a legacy list maintained at https://uqfoundation.github.io/project/pathos/query. +or suggestions are highly appreciated. A list of issues is located at +https://github.com/uqfoundation/dill/issues, with a legacy list maintained at +https://uqfoundation.github.io/project/pathos/query. Major Features -------------- ``dill`` can pickle the following standard types: -* none, type, bool, int, long, float, complex, str, unicode, +* none, type, bool, int, float, complex, bytes, str, * tuple, list, dict, file, buffer, builtin, -* both old and new style classes, -* instances of old and new style classes, +* python classes, namedtuples, dataclasses, metaclasses, +* instances of classes, * set, frozenset, array, functions, exceptions ``dill`` can also pickle more 'exotic' standard types: -* functions with yields, nested functions, lambdas +* functions with yields, nested functions, lambdas, * cell, method, unboundmethod, module, code, methodwrapper, -* dictproxy, methoddescriptor, getsetdescriptor, memberdescriptor, -* wrapperdescriptor, xrange, slice, -* notimplemented, ellipsis, quit +* methoddescriptor, getsetdescriptor, memberdescriptor, wrapperdescriptor, +* dictproxy, slice, notimplemented, ellipsis, quit ``dill`` cannot yet pickle these standard types: @@ -133,7 +134,7 @@ There are a number of options to control serialization which are provided as keyword arguments to several ``dill`` functions: * with *protocol*, the pickle protocol level can be set. This uses the - same value as the ``pickle`` module, *HIGHEST_PROTOCOL* or *DEFAULT_PROTOCOL*. + same value as the ``pickle`` module, *DEFAULT_PROTOCOL*. * with *byref=True*, ``dill`` to behave a lot more like pickle with certain objects (like modules) pickled by reference as opposed to attempting to pickle the object itself. diff --git a/dill/__diff.py b/dill/__diff.py index 3ff65763..60268a6d 100644 --- a/dill/__diff.py +++ b/dill/__diff.py @@ -10,18 +10,15 @@ Module to show if an object has changed since it was memorised """ +import builtins import os import sys import types try: import numpy HAS_NUMPY = True -except: - HAS_NUMPY = False -try: - import builtins except ImportError: - import __builtin__ as builtins + HAS_NUMPY = False # pypy doesn't use reference counting getrefcount = getattr(sys, 'getrefcount', lambda x:0) @@ -44,10 +41,7 @@ def get_attrs(obj): if type(obj) in builtins_types \ or type(obj) is type and obj in builtins_types: return - try: - return obj.__dict__ - except: - return + return getattr(obj, '__dict__', None) def get_seq(obj, cache={str: False, frozenset: False, list: True, set: True, diff --git a/dill/__init__.py b/dill/__init__.py index 39ac4df2..80281811 100644 --- a/dill/__init__.py +++ b/dill/__init__.py @@ -26,7 +26,7 @@ ``dill`` provides the user the same interface as the ``pickle`` module, and also includes some additional features. In addition to pickling python objects, ``dill`` provides the ability to save the state of an interpreter -session in a single command. Hence, it would be feasable to save an +session in a single command. Hence, it would be feasible to save an interpreter session, close the interpreter, ship the pickled file to another computer, open a new interpreter, unpickle the session and thus continue from the 'saved' state of the original interpreter @@ -42,7 +42,9 @@ ``dill`` is part of ``pathos``, a python framework for heterogeneous computing. ``dill`` is in active development, so any user feedback, bug reports, comments, -or suggestions are highly appreciated. A list of issues is located at https://github.com/uqfoundation/dill/issues, with a legacy list maintained at https://uqfoundation.github.io/project/pathos/query. +or suggestions are highly appreciated. A list of issues is located at +https://github.com/uqfoundation/dill/issues, with a legacy list maintained at +https://uqfoundation.github.io/project/pathos/query. Major Features @@ -50,19 +52,18 @@ ``dill`` can pickle the following standard types: - - none, type, bool, int, long, float, complex, str, unicode, + - none, type, bool, int, float, complex, bytes, str, - tuple, list, dict, file, buffer, builtin, - - both old and new style classes, - - instances of old and new style classes, + - python classes, namedtuples, dataclasses, metaclasses, + - instances of classes, - set, frozenset, array, functions, exceptions ``dill`` can also pickle more 'exotic' standard types: - functions with yields, nested functions, lambdas, - cell, method, unboundmethod, module, code, methodwrapper, - - dictproxy, methoddescriptor, getsetdescriptor, memberdescriptor, - - wrapperdescriptor, xrange, slice, - - notimplemented, ellipsis, quit + - methoddescriptor, getsetdescriptor, memberdescriptor, wrapperdescriptor, + - dictproxy, slice, notimplemented, ellipsis, quit ``dill`` cannot yet pickle these standard types: @@ -148,7 +149,7 @@ as keyword arguments to several ``dill`` functions: * with *protocol*, the pickle protocol level can be set. This uses the - same value as the ``pickle`` module, *HIGHEST_PROTOCOL* or *DEFAULT_PROTOCOL*. + same value as the ``pickle`` module, *DEFAULT_PROTOCOL*. * with *byref=True*, ``dill`` to behave a lot more like pickle with certain objects (like modules) pickled by reference as opposed to attempting to pickle the object itself. @@ -300,23 +301,9 @@ # make sure "trace" is turned off detect.trace(False) -try: - from importlib import reload -except ImportError: - try: - from imp import reload - except ImportError: - pass - -# put the objects in order, if possible -try: - from collections import OrderedDict as odict -except ImportError: - try: - from ordereddict import OrderedDict as odict - except ImportError: - odict = dict -objects = odict() +from importlib import reload + +objects = {} # local import of dill._objects #from . import _objects #objects.update(_objects.succeeds) @@ -377,7 +364,6 @@ def extend(use_dill=True): return extend() -del odict def license(): diff --git a/dill/_dill.py b/dill/_dill.py index 7660cb07..c718c4f2 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -31,52 +31,27 @@ import sys diff = None _use_diff = False -PY3 = (sys.hexversion >= 0x3000000) -# OLDER: 3.0 <= x < 3.4 *OR* x < 2.7.10 #NOTE: guessing relevant versions -OLDER = (PY3 and sys.hexversion < 0x3040000) or (sys.hexversion < 0x2070ab1) -OLD33 = (sys.hexversion < 0x3030000) -OLD37 = (sys.hexversion < 0x3070000) OLD38 = (sys.hexversion < 0x3080000) OLD39 = (sys.hexversion < 0x3090000) OLD310 = (sys.hexversion < 0x30a0000) -PY34 = (0x3040000 <= sys.hexversion < 0x3050000) -if PY3: #XXX: get types from .objtypes ? - import builtins as __builtin__ - from pickle import _Pickler as StockPickler, Unpickler as StockUnpickler - from _thread import LockType - if (sys.hexversion >= 0x30200f0): - from _thread import RLock as RLockType - else: - from threading import _RLock as RLockType - #from io import IOBase - from types import CodeType, FunctionType, MethodType, GeneratorType, \ - TracebackType, FrameType, ModuleType, BuiltinMethodType - BufferType = memoryview #XXX: unregistered - ClassType = type # no 'old-style' classes - EllipsisType = type(Ellipsis) - #FileType = IOBase - NotImplementedType = type(NotImplemented) - SliceType = slice - TypeType = type # 'new-style' classes #XXX: unregistered - XRangeType = range - if OLD33: - DictProxyType = type(object.__dict__) - else: - from types import MappingProxyType as DictProxyType -else: - import __builtin__ - from pickle import Pickler as StockPickler, Unpickler as StockUnpickler - from thread import LockType - from threading import _RLock as RLockType - from types import CodeType, FunctionType, ClassType, MethodType, \ - GeneratorType, DictProxyType, XRangeType, SliceType, TracebackType, \ - NotImplementedType, EllipsisType, FrameType, ModuleType, \ - BufferType, BuiltinMethodType, TypeType -from pickle import HIGHEST_PROTOCOL, PickleError, PicklingError, UnpicklingError -try: - from pickle import DEFAULT_PROTOCOL -except ImportError: - DEFAULT_PROTOCOL = HIGHEST_PROTOCOL +#XXX: get types from .objtypes ? +import builtins as __builtin__ +from pickle import _Pickler as StockPickler, Unpickler as StockUnpickler +from _thread import LockType +from _thread import RLock as RLockType +#from io import IOBase +from types import CodeType, FunctionType, MethodType, GeneratorType, \ + TracebackType, FrameType, ModuleType, BuiltinMethodType +BufferType = memoryview #XXX: unregistered +ClassType = type # no 'old-style' classes +EllipsisType = type(Ellipsis) +#FileType = IOBase +NotImplementedType = type(NotImplemented) +SliceType = slice +TypeType = type # 'new-style' classes #XXX: unregistered +XRangeType = range +from types import MappingProxyType as DictProxyType +from pickle import DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, PickleError, PicklingError, UnpicklingError import __main__ as _main_module import marshal import gc @@ -85,20 +60,9 @@ from collections import OrderedDict from functools import partial from operator import itemgetter, attrgetter -# new in python3.3 -if sys.hexversion < 0x03030000: - FileNotFoundError = IOError -if PY3 and sys.hexversion < 0x03040000: - GENERATOR_FAIL = True -else: GENERATOR_FAIL = False -if PY3: - import importlib.machinery - EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES) -else: - import imp - EXTENSION_SUFFIXES = tuple(suffix - for (suffix, _, s_type) in imp.get_suffixes() - if s_type == imp.C_EXTENSION) +GENERATOR_FAIL = False +import importlib.machinery +EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES) try: import ctypes HAS_CTYPES = True @@ -107,28 +71,15 @@ except ImportError: HAS_CTYPES = False IS_PYPY = False -IS_PYPY2 = IS_PYPY and not PY3 NumpyUfuncType = None NumpyDType = None NumpyArrayType = None try: - if OLDER: - raise AttributeError('find_spec not found') - import importlib if not importlib.machinery.PathFinder().find_spec('numpy'): raise ImportError("No module named 'numpy'") NumpyUfuncType = True NumpyDType = True NumpyArrayType = True -except AttributeError: - try: - import imp - imp.find_module('numpy') - NumpyUfuncType = True - NumpyDType = True - NumpyArrayType = True - except ImportError: - pass except ImportError: pass def __hook__(): @@ -193,10 +144,7 @@ def numpydtype(obj): return False MethodWrapperType #XXX: unused # make sure to add these 'hand-built' types to _typemap -if PY3: - CellType = type((lambda x: lambda y: x)(0).__closure__[0]) -else: - CellType = type((lambda x: lambda y: x)(0).func_closure[0]) +CellType = type((lambda x: lambda y: x)(0).__closure__[0]) PartialType = type(partial(int, base=2)) SuperType = type(super(Exception, TypeError())) ItemGetterType = type(itemgetter(0)) @@ -204,7 +152,7 @@ def numpydtype(obj): return False try: from functools import _lru_cache_wrapper as LRUCacheType -except: +except ImportError: LRUCacheType = None if not isinstance(LRUCacheType, type): @@ -230,20 +178,11 @@ def get_file_type(*args, **kwargs): PyBufferedWriterType = get_file_type('wb', buffering=-1, open=_open) except ImportError: PyTextWrapperType = PyBufferedRandomType = PyBufferedReaderType = PyBufferedWriterType = None -try: - from cStringIO import StringIO, InputType, OutputType -except ImportError: - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO - InputType = OutputType = None -if not IS_PYPY2: - from socket import socket as SocketType - try: #FIXME: additionally calls ForkingPickler.register several times - from multiprocessing.reduction import _reduce_socket as reduce_socket - except ImportError: - from multiprocessing.reduction import reduce_socket +from io import BytesIO as StringIO +InputType = OutputType = None +from socket import socket as SocketType +#FIXME: additionally calls ForkingPickler.register several times +from multiprocessing.reduction import _reduce_socket as reduce_socket try: __IPYTHON__ is True # is ipython ExitType = None # IPython.core.autocall.ExitAutocall @@ -384,8 +323,7 @@ def _module_map(): from collections import defaultdict, namedtuple modmap = namedtuple('Modmap', ['by_name', 'by_id', 'top_level']) modmap = modmap(defaultdict(list), defaultdict(list), {}) - items = 'items' if PY3 else 'iteritems' - for modname, module in getattr(sys.modules, items)(): + for modname, module in sys.modules.items(): if not isinstance(module, ModuleType): continue if '.' not in modname: @@ -414,8 +352,7 @@ def _stash_modules(main_module): imported_as = [] imported_top_level = [] # keep separeted for backwards compatibility original = {} - items = 'items' if PY3 else 'iteritems' - for name, obj in getattr(main_module.__dict__, items)(): + for name, obj in main_module.__dict__.items(): if obj is main_module: original[name] = newmod # self-reference continue @@ -674,22 +611,17 @@ def use_diff(on=True): if _use_diff and diff is None: try: from . import diff as d - except: + except ImportError: import diff as d diff = d def _create_typemap(): import types - if PY3: - d = dict(list(__builtin__.__dict__.items()) + \ - list(types.__dict__.items())).items() - builtin = 'builtins' - else: - d = types.__dict__.iteritems() - builtin = '__builtin__' + d = dict(list(__builtin__.__dict__.items()) + \ + list(types.__dict__.items())).items() for key, value in d: - if getattr(value, '__module__', None) == builtin \ - and type(value) is type: + if getattr(value, '__module__', None) == 'builtins' \ + and type(value) is type: yield key, value return _reverse_typemap = dict(_create_typemap()) @@ -719,22 +651,15 @@ def _create_typemap(): 'PyTextWrapperType': PyTextWrapperType, } -if PY3: - _incedental_reverse_typemap.update({ - "DictKeysType": type({}.keys()), - "DictValuesType": type({}.values()), - "DictItemsType": type({}.items()), +_incedental_reverse_typemap.update({ + "DictKeysType": type({}.keys()), + "DictValuesType": type({}.values()), + "DictItemsType": type({}.items()), - "OdictKeysType": type(x.keys()), - "OdictValuesType": type(x.values()), - "OdictItemsType": type(x.items()), - }) -else: - _incedental_reverse_typemap.update({ - "DictKeysType": type({}.viewkeys()), - "DictValuesType": type({}.viewvalues()), - "DictItemsType": type({}.viewitems()), - }) + "OdictKeysType": type(x.keys()), + "OdictValuesType": type(x.values()), + "OdictItemsType": type(x.items()), +}) if ExitType: _incedental_reverse_typemap['ExitType'] = ExitType @@ -761,7 +686,7 @@ def _create_typemap(): try: import winreg _incedental_reverse_typemap["HKEYType"] = winreg.HKEYType -except: +except ImportError: pass _reverse_typemap.update(_incedental_reverse_typemap) @@ -769,10 +694,7 @@ def _create_typemap(): del x -if PY3: - _typemap = dict((v, k) for k, v in _reverse_typemap.items()) -else: - _typemap = dict((v, k) for k, v in _reverse_typemap.iteritems()) +_typemap = dict((v, k) for k, v in _reverse_typemap.items()) def _unmarshal(string): return marshal.loads(string) @@ -805,7 +727,7 @@ def _create_code(*args): args = args[1:] else: # from < 3.10 (or pre-LNOTAB storage) LNOTAB = b'' - if PY3 and hasattr(args[-3], 'encode'): #NOTE: from PY2 fails (optcode) + if hasattr(args[-3], 'encode'): #NOTE: from PY2 fails (optcode) args = list(args) if len(args) == 20: # from 3.11a # obj.co_argcount, obj.co_posonlyargcount, @@ -933,7 +855,7 @@ def _create_code(*args): return CodeType(*args) elif len(args) == 15: return CodeType(args[0], 0, *args[1:]) # from 3.7 return CodeType(args[0], 0, 0, *args[1:]) # from 2.7 - elif hasattr(CodeType, 'co_kwonlyargcount'): # python 3.7 + else: # python 3.7 # obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, # obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, # obj.co_names, obj.co_varnames, obj.co_filename, @@ -953,24 +875,6 @@ def _create_code(*args): return CodeType(args[0], *argz) elif len(args) == 15: return CodeType(*args) return CodeType(args[0], 0, *args[1:]) # from 2.7 - # obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, - # obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, - # obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, - # obj.co_freevars, obj.co_cellvars - if len(args) == 20: # from 3.11a - args = args[:1] + args[3:12] + args[13:14] + (LNOTAB,) + args[18:] - return CodeType(*args) - elif len(args) == 18: # from 3.11 - args = args[:1] + args[3:12] + args[13:14] + (LNOTAB,) + args[16:] - return CodeType(*args) - elif len(args) == 16: # from 3.10 or from 3.8 - if LNOTAB: # here and above uses stored LNOTAB - argz = args[3:-3] + (LNOTAB,) + args[-2:] - else: - argz = args[3:] - return CodeType(args[0], *argz) - elif len(args) == 15: return CodeType(args[0], *args[2:]) # from 3.7 - return CodeType(*args) def _create_ftype(ftypeobj, func, args, kwds): if kwds is None: @@ -1010,12 +914,9 @@ def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdat import tempfile f = tempfile.TemporaryFile(mode) else: - # treat x mode as w mode - if "x" in mode and sys.hexversion < 0x03030000: - raise ValueError("invalid mode: '%s'" % mode) try: exists = os.path.exists(name) - except: + except Exception: exists = False if not exists: if strictio: @@ -1044,6 +945,7 @@ def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdat elif name == '': # file did not exist import tempfile f = tempfile.TemporaryFile(mode) + # treat x mode as w mode elif fmode == CONTENTS_FMODE \ and ("w" in mode or "x" in mode): # stop truncation when opening @@ -1054,28 +956,9 @@ def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdat flags |= os.O_WRONLY f = os.fdopen(os.open(name, flags), mode) # set name to the correct value - if PY3: - r = getattr(f, "buffer", f) - r = getattr(r, "raw", r) - r.name = name - else: - if not HAS_CTYPES: - raise ImportError("No module named 'ctypes'") - class FILE(ctypes.Structure): - _fields_ = [("refcount", ctypes.c_long), - ("type_obj", ctypes.py_object), - ("file_pointer", ctypes.c_voidp), - ("name", ctypes.py_object)] - - class PyObject(ctypes.Structure): - _fields_ = [ - ("ob_refcnt", ctypes.c_int), - ("ob_type", ctypes.py_object) - ] - #FIXME: CONTENTS_FMODE fails for pypy due to issue #1233 - # https://bitbucket.org/pypy/pypy/issues/1233 - ctypes.cast(id(f), ctypes.POINTER(FILE)).contents.name = name - ctypes.cast(id(name), ctypes.POINTER(PyObject)).contents.ob_refcnt += 1 + r = getattr(f, "buffer", f) + r = getattr(r, "raw", r) + r.name = name assert f.name == name else: f = open(name, mode) @@ -1136,7 +1019,7 @@ def __ror__(self, a): # mapping referenced by the proxy. It may work for other implementations, # but is not guaranteed. MAPPING_PROXY_TRICK = __d is (DictProxyType(__d) | _dictproxy_helper_instance) -except: +except Exception: MAPPING_PROXY_TRICK = False del __d @@ -1147,26 +1030,15 @@ def __ror__(self, a): _CELL_REF = None _CELL_EMPTY = Sentinel('_CELL_EMPTY') -if PY3: - def _create_cell(contents=None): - if contents is not _CELL_EMPTY: - value = contents - return (lambda: value).__closure__[0] - -else: - def _create_cell(contents=None): - if contents is not _CELL_EMPTY: - value = contents - return (lambda: value).func_closure[0] - +def _create_cell(contents=None): + if contents is not _CELL_EMPTY: + value = contents + return (lambda: value).__closure__[0] def _create_weakref(obj, *args): from weakref import ref if obj is None: # it's dead - if PY3: - from collections import UserDict - else: - from UserDict import UserDict + from collections import UserDict return ref(UserDict(), *args) return ref(obj, *args) @@ -1174,10 +1046,7 @@ def _create_weakproxy(obj, callable=False, *args): from weakref import proxy if obj is None: # it's dead if callable: return proxy(lambda x:x, *args) - if PY3: - from collections import UserDict - else: - from UserDict import UserDict + from collections import UserDict return proxy(UserDict(), *args) return proxy(obj, *args) @@ -1198,37 +1067,24 @@ def _create_dtypemeta(scalar_type): return NumpyDType return type(NumpyDType(scalar_type)) -if OLD37: - def _create_namedtuple(name, fieldnames, modulename, defaults=None): - class_ = _import_module(modulename + '.' + name, safe=True) - if class_ is not None: - return class_ - import collections - t = collections.namedtuple(name, fieldnames) - t.__module__ = modulename - return t -else: - def _create_namedtuple(name, fieldnames, modulename, defaults=None): - class_ = _import_module(modulename + '.' + name, safe=True) - if class_ is not None: - return class_ - import collections - t = collections.namedtuple(name, fieldnames, defaults=defaults, module=modulename) - return t +def _create_namedtuple(name, fieldnames, modulename, defaults=None): + class_ = _import_module(modulename + '.' + name, safe=True) + if class_ is not None: + return class_ + import collections + t = collections.namedtuple(name, fieldnames, defaults=defaults, module=modulename) + return t def _create_capsule(pointer, name, context, destructor): attr_found = False try: # based on https://github.com/python/cpython/blob/f4095e53ab708d95e019c909d5928502775ba68f/Objects/capsule.c#L209-L231 - if PY3: - uname = name.decode('utf8') - else: - uname = name + uname = name.decode('utf8') for i in range(1, uname.count('.')+1): names = uname.rsplit('.', i) try: module = __import__(names[0]) - except: + except ImportError: pass obj = module for attr in names[1:]: @@ -1236,7 +1092,7 @@ def _create_capsule(pointer, name, context, destructor): capsule = obj attr_found = True break - except: + except Exception: pass if attr_found: @@ -1254,14 +1110,14 @@ def _getattr(objclass, name, repr_str): try: #XXX: works only for __builtin__ ? attr = repr_str.split("'")[3] return eval(attr+'.__dict__["'+name+'"]') - except: + except Exception: try: attr = objclass.__dict__ if type(attr) is DictProxyType: attr = attr[name] else: attr = getattr(objclass,name) - except: + except (AttributeError, KeyError): attr = getattr(objclass,name) return attr @@ -1314,7 +1170,7 @@ def _locate_function(obj, pickler=None): try: found, _ = _getattribute(module, obj.__qualname__) return found is obj - except: + except AttributeError: return False else: found = _import_module(module_name + '.' + obj.__name__, safe=True) @@ -1370,10 +1226,7 @@ def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO else: pickler.save_reduce(*reduction) # pop None created by calling preprocessing step off stack - if PY3: - pickler.write(bytes('0', 'UTF-8')) - else: - pickler.write('0') + pickler.write(bytes('0', 'UTF-8')) #@register(CodeType) #def save_code(pickler, obj): @@ -1389,62 +1242,54 @@ def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO @register(CodeType) def save_code(pickler, obj): logger.trace(pickler, "Co: %s", obj) - if PY3: - if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args) - args = ( - obj.co_lnotab, # for < python 3.10 [not counted in args] - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, - obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable, - obj.co_columntable, obj.co_exceptiontable, obj.co_freevars, - obj.co_cellvars - ) - elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args) - args = ( - obj.co_lnotab, # for < python 3.10 [not counted in args] - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, - obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, - obj.co_freevars, obj.co_cellvars - ) - elif hasattr(obj, "co_linetable"): # python 3.10 (16 args) - args = ( - obj.co_lnotab, # for < python 3.10 [not counted in args] - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_linetable, obj.co_freevars, - obj.co_cellvars - ) - elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args) - args = ( - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, - obj.co_cellvars - ) - else: # python 3.7 (15 args) - args = ( - obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, - obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, - obj.co_names, obj.co_varnames, obj.co_filename, - obj.co_name, obj.co_firstlineno, obj.co_lnotab, - obj.co_freevars, obj.co_cellvars - ) - else: # python 2.7 (14 args) + if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args) args = ( - obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, - obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, - obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, + obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable, + obj.co_columntable, obj.co_exceptiontable, obj.co_freevars, + obj.co_cellvars + ) + elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, + obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, obj.co_freevars, obj.co_cellvars - ) + ) + elif hasattr(obj, "co_linetable"): # python 3.10 (16 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_linetable, obj.co_freevars, + obj.co_cellvars + ) + elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args) + args = ( + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, + obj.co_cellvars + ) + else: # python 3.7 (15 args) + args = ( + obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, + obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, + obj.co_names, obj.co_varnames, obj.co_filename, + obj.co_name, obj.co_firstlineno, obj.co_lnotab, + obj.co_freevars, obj.co_cellvars + ) pickler.save_reduce(_create_code, args, obj=obj) logger.trace(pickler, "# Co") @@ -1459,26 +1304,17 @@ def save_module_dict(pickler, obj): if is_dill(pickler, child=False) and obj == pickler._main.__dict__ and \ not (pickler._session and pickler._first_pass): logger.trace(pickler, "D1: %s", _repr_dict(obj)) # obj - if PY3: - pickler.write(bytes('c__builtin__\n__main__\n', 'UTF-8')) - else: - pickler.write('c__builtin__\n__main__\n') + pickler.write(bytes('c__builtin__\n__main__\n', 'UTF-8')) logger.trace(pickler, "# D1") elif (not is_dill(pickler, child=False)) and (obj == _main_module.__dict__): logger.trace(pickler, "D3: %s", _repr_dict(obj)) # obj - if PY3: - pickler.write(bytes('c__main__\n__dict__\n', 'UTF-8')) - else: - pickler.write('c__main__\n__dict__\n') #XXX: works in general? + pickler.write(bytes('c__main__\n__dict__\n', 'UTF-8')) #XXX: works in general? logger.trace(pickler, "# D3") elif '__name__' in obj and obj != _main_module.__dict__ \ - and type(obj['__name__']) is str \ - and obj is getattr(_import_module(obj['__name__'],True), '__dict__', None): + and type(obj['__name__']) is str \ + and obj is getattr(_import_module(obj['__name__'],True), '__dict__', None): logger.trace(pickler, "D4: %s", _repr_dict(obj)) # obj - if PY3: - pickler.write(bytes('c%s\n__dict__\n' % obj['__name__'], 'UTF-8')) - else: - pickler.write('c%s\n__dict__\n' % obj['__name__']) + pickler.write(bytes('c%s\n__dict__\n' % obj['__name__'], 'UTF-8')) logger.trace(pickler, "# D4") else: logger.trace(pickler, "D2: %s", _repr_dict(obj)) # obj @@ -1572,38 +1408,17 @@ def save_rlock(pickler, obj): logger.trace(pickler, "RL: %s", obj) r = obj.__repr__() # don't use _release_save as it unlocks the lock count = int(r.split('count=')[1].split()[0].rstrip('>')) - owner = int(r.split('owner=')[1].split()[0]) if PY3 else getattr(obj, '_RLock__owner') + owner = int(r.split('owner=')[1].split()[0]) pickler.save_reduce(_create_rlock, (count,owner,), obj=obj) logger.trace(pickler, "# RL") return -if not IS_PYPY2: - #@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL - def save_socket(pickler, obj): - logger.trace(pickler, "So: %s", obj) - pickler.save_reduce(*reduce_socket(obj)) - logger.trace(pickler, "# So") - return - -if sys.hexversion <= 0x3050000: - @register(ItemGetterType) - def save_itemgetter(pickler, obj): - logger.trace(pickler, "Ig: %s", obj) - helper = _itemgetter_helper() - obj(helper) - pickler.save_reduce(type(obj), tuple(helper.items), obj=obj) - logger.trace(pickler, "# Ig") - return - - @register(AttrGetterType) - def save_attrgetter(pickler, obj): - logger.trace(pickler, "Ag: %s", obj) - attrs = [] - helper = _attrgetter_helper(attrs) - obj(helper) - pickler.save_reduce(type(obj), tuple(attrs), obj=obj) - logger.trace(pickler, "# Ag") - return +#@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL +def save_socket(pickler, obj): + logger.trace(pickler, "So: %s", obj) + pickler.save_reduce(*reduce_socket(obj)) + logger.trace(pickler, "# So") + return def _save_file(pickler, obj, open_): if obj.closed: @@ -1683,15 +1498,6 @@ def save_stringo(pickler, obj): logger.trace(pickler, "# Io") return -if 0x2050000 <= sys.hexversion < 0x3010000: - @register(PartialType) - def save_functor(pickler, obj): - logger.trace(pickler, "Fu: %s", obj) - pickler.save_reduce(_create_ftype, (type(obj), obj.func, obj.args, - obj.keywords), obj=obj) - logger.trace(pickler, "# Fu") - return - if LRUCacheType is not None: from functools import lru_cache @register(LRUCacheType) @@ -1718,32 +1524,6 @@ def save_super(pickler, obj): logger.trace(pickler, "# Su") return -if OLDER or not PY3: - @register(BuiltinMethodType) - def save_builtin_method(pickler, obj): - if obj.__self__ is not None: - if obj.__self__ is __builtin__: - module = 'builtins' if PY3 else '__builtin__' - _t = "B1" - logger.trace(pickler, "%s: %s", _t, obj) - else: - module = obj.__self__ - _t = "B3" - logger.trace(pickler, "%s: %s", _t, obj) - if is_dill(pickler, child=True): - _recurse = pickler._recurse - pickler._recurse = False - pickler.save_reduce(_get_attr, (module, obj.__name__), obj=obj) - if is_dill(pickler, child=True): - pickler._recurse = _recurse - logger.trace(pickler, "# %s", _t) - else: - logger.trace(pickler, "B2: %s", obj) - name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) - StockPickler.save_global(pickler, obj, name=name) - logger.trace(pickler, "# B2") - return - if IS_PYPY: @register(MethodType) def save_instancemethod0(pickler, obj): @@ -1751,50 +1531,39 @@ def save_instancemethod0(pickler, obj): if code is not None and type(code) is not CodeType \ and getattr(obj.__self__, obj.__name__) == obj: # Some PyPy builtin functions have no module name - logger.trace(pickler, "Me2: %s" % obj) + logger.trace(pickler, "Me2: %s", obj) # TODO: verify that this works for all PyPy builtin methods pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) logger.trace(pickler, "# Me2") return - logger.trace(pickler, "Me1: %s" % obj) + logger.trace(pickler, "Me1: %s", obj) pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) logger.trace(pickler, "# Me1") return else: @register(MethodType) def save_instancemethod0(pickler, obj): - logger.trace(pickler, "Me1: %s" % obj) + logger.trace(pickler, "Me1: %s", obj) pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) logger.trace(pickler, "# Me1") return -if sys.hexversion >= 0x20500f0: - if not IS_PYPY: - @register(MemberDescriptorType) - @register(GetSetDescriptorType) - @register(MethodDescriptorType) - @register(WrapperDescriptorType) - @register(ClassMethodDescriptorType) - def save_wrapper_descriptor(pickler, obj): - logger.trace(pickler, "Wr: %s", obj) - pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, - obj.__repr__()), obj=obj) - logger.trace(pickler, "# Wr") - return - else: - @register(MemberDescriptorType) - @register(GetSetDescriptorType) - def save_wrapper_descriptor(pickler, obj): - logger.trace(pickler, "Wr: %s", obj) - pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, - obj.__repr__()), obj=obj) - logger.trace(pickler, "# Wr") - return - -elif not IS_PYPY: +if not IS_PYPY: + @register(MemberDescriptorType) + @register(GetSetDescriptorType) @register(MethodDescriptorType) @register(WrapperDescriptorType) + @register(ClassMethodDescriptorType) + def save_wrapper_descriptor(pickler, obj): + logger.trace(pickler, "Wr: %s", obj) + pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, + obj.__repr__()), obj=obj) + logger.trace(pickler, "# Wr") + return +else: + @register(MemberDescriptorType) + @register(GetSetDescriptorType) def save_wrapper_descriptor(pickler, obj): logger.trace(pickler, "Wr: %s", obj) pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, @@ -1806,7 +1575,7 @@ def save_wrapper_descriptor(pickler, obj): def save_cell(pickler, obj): try: f = obj.cell_contents - except: + except ValueError: # cell is empty logger.trace(pickler, "Ce3: %s", obj) # _shims._CELL_EMPTY is defined in _shims.py to support PyPy 2.7. # It unpickles to a sentinel object _dill._CELL_EMPTY, also created in @@ -1822,10 +1591,7 @@ def save_cell(pickler, obj): # The result of this function call will be None pickler.save_reduce(_shims._delattr, (obj, 'cell_contents')) # pop None created by calling _delattr off stack - if PY3: - pickler.write(bytes('0', 'UTF-8')) - else: - pickler.write('0') + pickler.write(bytes('0', 'UTF-8')) logger.trace(pickler, "# Ce3") return if is_dill(pickler, child=True): @@ -1909,10 +1675,7 @@ def _locate_object(address, module=None): for obj in special: if address == id(obj): return obj if module: - if PY3: - objects = iter(module.__dict__.values()) - else: - objects = module.__dict__.itervalues() + objects = iter(module.__dict__.values()) else: objects = iter(gc.get_objects()) for obj in objects: if address == id(obj): return obj @@ -1988,7 +1751,7 @@ def save_module(pickler, obj): pickler.save_reduce(_import_module, (obj.__name__,), obj=obj, state=_main_dict) logger.trace(pickler, "# M1") - elif PY3 and obj.__name__ == "dill._dill": + elif obj.__name__ == "dill._dill": logger.trace(pickler, "M2: %s", obj) pickler.save_global(obj, name="_dill") logger.trace(pickler, "# M2") @@ -2002,7 +1765,7 @@ def save_module(pickler, obj): @register(TypeType) def save_type(pickler, obj, postproc_list=None): if obj in _typemap: - logger.trace(pickler, "T1: %s" % obj) + logger.trace(pickler, "T1: %s", obj) # if obj in _incedental_types: # warnings.warn('Type %r may only exist on this implementation of Python and cannot be unpickled in other implementations.' % (obj,), PicklingWarning) pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj) @@ -2010,7 +1773,7 @@ def save_type(pickler, obj, postproc_list=None): elif obj.__bases__ == (tuple,) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]): # special case: namedtuples logger.trace(pickler, "T6: %s", obj) - if OLD37 or (not obj._field_defaults): + if not obj._field_defaults: pickler.save_reduce(_create_namedtuple, (obj.__name__, obj._fields, obj.__module__), obj=obj) else: defaults = [obj._field_defaults[field] for field in obj._fields if field in obj._field_defaults] @@ -2022,10 +1785,7 @@ def save_type(pickler, obj, postproc_list=None): elif obj is type(None): logger.trace(pickler, "T7: %s", obj) #XXX: pickler.save_reduce(type, (None,), obj=obj) - if PY3: - pickler.write(bytes('c__builtin__\nNoneType\n', 'UTF-8')) - else: - pickler.write('c__builtin__\nNoneType\n') + pickler.write(bytes('c__builtin__\nNoneType\n', 'UTF-8')) logger.trace(pickler, "# T7") elif obj is NotImplementedType: logger.trace(pickler, "T7: %s", obj) @@ -2056,7 +1816,7 @@ def save_type(pickler, obj, postproc_list=None): #print ("%s\n%s" % (obj.__bases__, obj.__dict__)) for name in _dict.get("__slots__", []): del _dict[name] - if PY3 and obj_name != obj.__name__: + if obj_name != obj.__name__: if postproc_list is None: postproc_list = [] postproc_list.append((setattr, (obj, '__qualname__', obj_name))) @@ -2077,12 +1837,6 @@ def save_type(pickler, obj, postproc_list=None): logger.trace(pickler, "# T4") return -# Error in PyPy 2.7 when adding ABC support -if IS_PYPY2: - @register(FrameType) - def save_frame(pickler, obj): - raise PicklingError('Cannot pickle a Python stack frame') - @register(property) def save_property(pickler, obj): logger.trace(pickler, "Pr: %s", obj) @@ -2094,24 +1848,15 @@ def save_property(pickler, obj): @register(classmethod) def save_classmethod(pickler, obj): logger.trace(pickler, "Cm: %s", obj) - im_func = '__func__' if PY3 else 'im_func' - try: - orig_func = getattr(obj, im_func) - except AttributeError: # Python 2.6 - orig_func = obj.__get__(None, object) - if isinstance(obj, classmethod): - orig_func = getattr(orig_func, im_func) # Unbind - - # if PY3: - # if type(obj.__dict__) is dict: - # if obj.__dict__: - # state = obj.__dict__ - # else: - # state = None + orig_func = obj.__func__ + + # if type(obj.__dict__) is dict: + # if obj.__dict__: + # state = obj.__dict__ # else: - # state = (None, {'__dict__', obj.__dict__}) + # state = None # else: - # state = None + # state = (None, {'__dict__', obj.__dict__}) pickler.save_reduce(type(obj), (orig_func,), obj=obj) logger.trace(pickler, "# Cm") @@ -2131,16 +1876,16 @@ def save_function(pickler, obj): found, _ = _getattribute(module, obj.__qualname__) if getattr(found, '__func__', None) is obj: _pypy_builtin = True - except: + except AttributeError: pass if _pypy_builtin: - logger.trace(pickler, "F3: %s" % obj) + logger.trace(pickler, "F3: %s", obj) pickler.save_reduce(getattr, (found, '__func__'), obj=obj) logger.trace(pickler, "# F3") return - logger.trace(pickler, "F1: %s" % obj) + logger.trace(pickler, "F1: %s", obj) _recurse = getattr(pickler, '_recurse', None) _postproc = getattr(pickler, '_postproc', None) _main_modified = getattr(pickler, '_main_modified', None) @@ -2157,7 +1902,7 @@ def save_function(pickler, obj): # is created to correctly handle recursion. globs = {'__name__': obj.__module__} else: - globs_copy = obj.__globals__ if PY3 else obj.func_globals + globs_copy = obj.__globals__ # If the globals is the __dict__ from the module being saved as a # session, substitute it by the dictionary being actually saved. @@ -2175,10 +1920,7 @@ def save_function(pickler, obj): # In the case that the globals are copied, we need to ensure that # the globals dictionary is updated when all objects in the # dictionary are already created. - if PY3: - glob_ids = {id(g) for g in globs_copy.values()} - else: - glob_ids = {id(g) for g in globs_copy.itervalues()} + glob_ids = {id(g) for g in globs_copy.values()} for stack_element in _postproc: if stack_element in glob_ids: _postproc[stack_element].append((_setitems, (globs, globs_copy))) @@ -2186,42 +1928,28 @@ def save_function(pickler, obj): else: postproc_list.append((_setitems, (globs, globs_copy))) - if PY3: - closure = obj.__closure__ - state_dict = {} - for fattrname in ('__doc__', '__kwdefaults__', '__annotations__'): - fattr = getattr(obj, fattrname, None) - if fattr is not None: - state_dict[fattrname] = fattr - if obj.__qualname__ != obj.__name__: - state_dict['__qualname__'] = obj.__qualname__ - if '__name__' not in globs or obj.__module__ != globs['__name__']: - state_dict['__module__'] = obj.__module__ - - state = obj.__dict__ - if type(state) is not dict: - state_dict['__dict__'] = state - state = None - if state_dict: - state = state, state_dict - - _save_with_postproc(pickler, (_create_function, ( - obj.__code__, globs, obj.__name__, obj.__defaults__, - closure - ), state), obj=obj, postproc_list=postproc_list) - else: - closure = obj.func_closure - if obj.__doc__ is not None: - postproc_list.append((setattr, (obj, '__doc__', obj.__doc__))) - if '__name__' not in globs or obj.__module__ != globs['__name__']: - postproc_list.append((setattr, (obj, '__module__', obj.__module__))) - if obj.__dict__: - postproc_list.append((setattr, (obj, '__dict__', obj.__dict__))) - - _save_with_postproc(pickler, (_create_function, ( - obj.func_code, globs, obj.func_name, obj.func_defaults, + closure = obj.__closure__ + state_dict = {} + for fattrname in ('__doc__', '__kwdefaults__', '__annotations__'): + fattr = getattr(obj, fattrname, None) + if fattr is not None: + state_dict[fattrname] = fattr + if obj.__qualname__ != obj.__name__: + state_dict['__qualname__'] = obj.__qualname__ + if '__name__' not in globs or obj.__module__ != globs['__name__']: + state_dict['__module__'] = obj.__module__ + + state = obj.__dict__ + if type(state) is not dict: + state_dict['__dict__'] = state + state = None + if state_dict: + state = state, state_dict + + _save_with_postproc(pickler, (_create_function, ( + obj.__code__, globs, obj.__name__, obj.__defaults__, closure - )), obj=obj, postproc_list=postproc_list) + ), state), obj=obj, postproc_list=postproc_list) # Lift closure cell update to earliest function (#458) if _postproc: @@ -2237,10 +1965,7 @@ def save_function(pickler, obj): # Change the value of the cell pickler.save_reduce(*possible_postproc) # pop None created by calling preprocessing step off stack - if PY3: - pickler.write(bytes('0', 'UTF-8')) - else: - pickler.write('0') + pickler.write(bytes('0', 'UTF-8')) logger.trace(pickler, "# F1") else: @@ -2372,10 +2097,7 @@ def check(obj, *args, **kwds): # unpickle = "dill.loads(%s, ignore=%s)"%(repr(_obj), repr(ignore)) # cmd = [python, "-c", "import dill; print(%s)"%unpickle] # msg = "SUCCESS" if not subprocess.call(cmd) else "LOAD FAILED" - if verbose is None: - msg = "%s -c import dill; dill.loads(%s)" % (python, repr(_obj)) - else: - msg = "%s -c import dill; print(dill.loads(%s))" % (python, repr(_obj)) + msg = "%s -c import dill; print(dill.loads(%s))" % (python, repr(_obj)) msg = "SUCCESS" if not subprocess.call(msg.split(None,2)) else "LOAD FAILED" if verbose: print(msg) @@ -2384,7 +2106,7 @@ def check(obj, *args, **kwds): # use to protect against missing attributes def is_dill(pickler, child=None): "check the dill-ness of your pickler" - if (child is False) or PY34 or (not hasattr(pickler.__class__, 'mro')): + if child is False or not hasattr(pickler.__class__, 'mro'): return 'dill' in pickler.__module__ return Pickler in pickler.__class__.mro() @@ -2394,9 +2116,8 @@ def _extend(): for t,func in Pickler.dispatch.items(): try: StockPickler.dispatch[t] = func - except: #TypeError, PicklingError, UnpicklingError + except Exception: #TypeError, PicklingError, UnpicklingError logger.trace(pickler, "skip: %s", t) - else: pass return del diff, _use_diff, use_diff diff --git a/dill/_objects.py b/dill/_objects.py index 954a3c04..8efee787 100644 --- a/dill/_objects.py +++ b/dill/_objects.py @@ -15,22 +15,9 @@ # helper imports import warnings; warnings.filterwarnings("ignore", category=DeprecationWarning) import sys -PY3 = (hex(sys.hexversion) >= '0x30000f0') -if PY3: - import queue as Queue - import dbm as anydbm -else: - import Queue - import anydbm - import sets # deprecated/removed - import mutex # removed -try: - from cStringIO import StringIO # has StringI and StringO types -except ImportError: # only has StringIO type - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO +import queue as Queue +import dbm as anydbm +from io import BytesIO as StringIO import re import array import collections @@ -41,6 +28,7 @@ import weakref import pprint import decimal +import numbers import functools import itertools import operator @@ -56,6 +44,7 @@ import hmac import os import logging +import logging.handlers import optparse #import __hello__ import threading @@ -64,8 +53,7 @@ try: import bz2 import sqlite3 - if PY3: import dbm.ndbm as dbm - else: import dbm + import dbm.ndbm as dbm HAS_ALL = True except ImportError: # Ubuntu HAS_ALL = False @@ -112,7 +100,7 @@ class _newclass2(object): def _function(x): yield x def _function2(): try: raise - except: + except Exception: from sys import exc_info e, er, tb = exc_info() return er, tb @@ -123,20 +111,12 @@ class _Struct(ctypes.Structure): _filedescrip, _tempfile = tempfile.mkstemp('r') # deleted in cleanup _tmpf = tempfile.TemporaryFile('w') -# put the objects in order, if possible -try: - from collections import OrderedDict as odict -except ImportError: - try: - from ordereddict import OrderedDict as odict - except ImportError: - odict = dict # objects used by dill for type declaration -registered = d = odict() +registered = d = {} # objects dill fails to pickle -failures = x = odict() +failures = x = {} # all other type objects -succeeds = a = odict() +succeeds = a = {} # types module (part of CH 8) a['BooleanType'] = bool(1) @@ -157,12 +137,8 @@ class _Struct(ctypes.Structure): a['StringType'] = _str = str(1) a['TupleType'] = _tuple = () a['TypeType'] = type -if PY3: - a['LongType'] = _int - a['UnicodeType'] = _str -else: - a['LongType'] = long(1) - a['UnicodeType'] = unicode(1) +a['LongType'] = _int +a['UnicodeType'] = _str # built-in constants (CH 4) a['CopyrightType'] = copyright # built-in types (CH 5) @@ -181,17 +157,13 @@ class _Struct(ctypes.Structure): a['TZInfoType'] = datetime.tzinfo() a['DateTimeType'] = datetime.datetime.today() a['CalendarType'] = calendar.Calendar() -if not PY3: - a['SetsType'] = sets.Set() - a['ImmutableSetType'] = sets.ImmutableSet() - a['MutexType'] = mutex.mutex() # numeric and mathematical types (CH 9) a['DecimalType'] = decimal.Decimal(1) a['CountType'] = itertools.count(0) # data compression and archiving (CH 12) a['TarInfoType'] = tarfile.TarInfo() # generic operating system services (CH 15) -a['LoggerType'] = logging.getLogger() +a['LoggerType'] = _logger = logging.getLogger() a['FormatterType'] = logging.Formatter() # pickle ok a['FilterType'] = logging.Filter() # pickle ok a['LogRecordType'] = logging.makeLogRecord(_dict) # pickle ok @@ -223,40 +195,33 @@ class _Struct(ctypes.Structure): #NOTE: ctypes.c_int._objects is memberdescriptor for object's __dict__ #NOTE: base class of all ctypes data types is non-public _CData -try: # python 2.6 - import fractions - import number - import io - from io import StringIO as TextIO - # built-in functions (CH 2) - a['ByteArrayType'] = bytearray([1]) - # numeric and mathematical types (CH 9) - a['FractionType'] = fractions.Fraction() - a['NumberType'] = numbers.Number() - # generic operating system services (CH 15) - a['IOBaseType'] = io.IOBase() - a['RawIOBaseType'] = io.RawIOBase() - a['TextIOBaseType'] = io.TextIOBase() - a['BufferedIOBaseType'] = io.BufferedIOBase() - a['UnicodeIOType'] = TextIO() # the new StringIO - a['LoggingAdapterType'] = logging.LoggingAdapter(_logger,_dict) # pickle ok - if HAS_CTYPES: - a['CBoolType'] = ctypes.c_bool(1) - a['CLongDoubleType'] = ctypes.c_longdouble() -except ImportError: - pass -try: # python 2.7 - import argparse - # data types (CH 8) - a['OrderedDictType'] = collections.OrderedDict(_dict) - a['CounterType'] = collections.Counter(_dict) - if HAS_CTYPES: - a['CSSizeTType'] = ctypes.c_ssize_t() - # generic operating system services (CH 15) - a['NullHandlerType'] = logging.NullHandler() # pickle ok # new 2.7 - a['ArgParseFileType'] = argparse.FileType() # pickle ok -except (AttributeError, ImportError): - pass +import fractions +import io +from io import StringIO as TextIO +# built-in functions (CH 2) +a['ByteArrayType'] = bytearray([1]) +# numeric and mathematical types (CH 9) +a['FractionType'] = fractions.Fraction() +a['NumberType'] = numbers.Number() +# generic operating system services (CH 15) +a['IOBaseType'] = io.IOBase() +a['RawIOBaseType'] = io.RawIOBase() +a['TextIOBaseType'] = io.TextIOBase() +a['BufferedIOBaseType'] = io.BufferedIOBase() +a['UnicodeIOType'] = TextIO() # the new StringIO +a['LoggerAdapterType'] = logging.LoggerAdapter(_logger,_dict) # pickle ok +if HAS_CTYPES: + a['CBoolType'] = ctypes.c_bool(1) + a['CLongDoubleType'] = ctypes.c_longdouble() +import argparse +# data types (CH 8) +a['OrderedDictType'] = collections.OrderedDict(_dict) +a['CounterType'] = collections.Counter(_dict) +if HAS_CTYPES: + a['CSSizeTType'] = ctypes.c_ssize_t() +# generic operating system services (CH 15) +a['NullHandlerType'] = logging.NullHandler() # pickle ok # new 2.7 +a['ArgParseFileType'] = argparse.FileType() # pickle ok # -- pickle fails on all below here ----------------------------------------- # types module (part of CH 8) @@ -288,31 +253,21 @@ class _Struct(ctypes.Structure): except ImportError: pass # other (concrete) object types -if PY3: - d['CellType'] = (_lambda)(0).__closure__[0] - a['XRangeType'] = _xrange = range(1) -else: - d['CellType'] = (_lambda)(0).func_closure[0] - a['XRangeType'] = _xrange = xrange(1) +d['CellType'] = (_lambda)(0).__closure__[0] +a['XRangeType'] = _xrange = range(1) a['MethodDescriptorType'] = type.__dict__['mro'] a['WrapperDescriptorType'] = type.__repr__ #a['WrapperDescriptorType2'] = type.__dict__['__module__']#XXX: GetSetDescriptor -a['ClassMethodDescriptorType'] = type.__dict__['__prepare__' if PY3 else 'mro'] +a['ClassMethodDescriptorType'] = type.__dict__['__prepare__'] # built-in functions (CH 2) -if PY3 or IS_PYPY: - _methodwrap = (1).__lt__ -else: - _methodwrap = (1).__cmp__ +_methodwrap = (1).__lt__ a['MethodWrapperType'] = _methodwrap a['StaticMethodType'] = staticmethod(_method) a['ClassMethodType'] = classmethod(_method) a['PropertyType'] = property() d['SuperType'] = super(Exception, _exception) # string services (CH 7) -if PY3: - _in = _bytes -else: - _in = _str +_in = _bytes a['InputType'] = _cstrI = StringIO(_in) a['OutputType'] = _cstrO = StringIO() # data types (CH 8) @@ -327,16 +282,12 @@ class _Struct(ctypes.Structure): a['QueueType'] = Queue.Queue() # numeric and mathematical types (CH 9) d['PartialType'] = functools.partial(int,base=2) -if PY3: - a['IzipType'] = zip('0','1') -else: - a['IzipType'] = itertools.izip('0','1') +a['IzipType'] = zip('0','1') a['ChainType'] = itertools.chain('0','1') d['ItemGetterType'] = operator.itemgetter(0) d['AttrGetterType'] = operator.attrgetter('__repr__') # file and directory access (CH 10) -if PY3: _fileW = _cstrO -else: _fileW = _tmpf +_fileW = _cstrO # data persistence (CH 11) if HAS_ALL: x['ConnectionType'] = _conn = sqlite3.connect(':memory:') @@ -344,8 +295,7 @@ class _Struct(ctypes.Structure): a['ShelveType'] = shelve.Shelf({}) # data compression and archiving (CH 12) if HAS_ALL: - if (hex(sys.hexversion) < '0x2070ef0') or PY3: - x['BZ2FileType'] = bz2.BZ2File(os.devnull) + x['BZ2FileType'] = bz2.BZ2File(os.devnull) x['BZ2CompressorType'] = bz2.BZ2Compressor() x['BZ2DecompressorType'] = bz2.BZ2Decompressor() #x['ZipFileType'] = _zip = zipfile.ZipFile(os.devnull,'w') @@ -362,17 +312,10 @@ class _Struct(ctypes.Structure): a['NamedLoggerType'] = _logger = logging.getLogger(__name__) #a['FrozenModuleType'] = __hello__ #FIXME: prints "Hello world..." # interprocess communication (CH 17) -if PY3: - x['SocketType'] = _socket = socket.socket() - x['SocketPairType'] = socket.socketpair()[0] -else: - x['SocketType'] = _socket = socket.socket() - x['SocketPairType'] = _socket._sock +x['SocketType'] = _socket = socket.socket() +x['SocketPairType'] = socket.socketpair()[0] # python runtime services (CH 27) -if PY3: - a['GeneratorContextManagerType'] = contextlib.contextmanager(max)([1]) -else: - a['GeneratorContextManagerType'] = contextlib.GeneratorContextManager(max) +a['GeneratorContextManagerType'] = contextlib.contextmanager(max)([1]) try: # ipython __IPYTHON__ is True # is ipython @@ -389,27 +332,21 @@ class _Struct(ctypes.Structure): a['NumpyInt32Type'] = _numpy_int32 except ImportError: pass -try: # python 2.6 - # numeric and mathematical types (CH 9) - a['ProductType'] = itertools.product('0','1') - # generic operating system services (CH 15) - a['FileHandlerType'] = logging.FileHandler(os.devnull) - a['RotatingFileHandlerType'] = logging.handlers.RotatingFileHandler(os.devnull) - a['SocketHandlerType'] = logging.handlers.SocketHandler('localhost',514) - a['MemoryHandlerType'] = logging.handlers.MemoryHandler(1) -except AttributeError: - pass -try: # python 2.7 - # data types (CH 8) - a['WeakSetType'] = weakref.WeakSet() # 2.7 -# # generic operating system services (CH 15) [errors when dill is imported] -# a['ArgumentParserType'] = _parser = argparse.ArgumentParser('PROG') -# a['NamespaceType'] = _parser.parse_args() # pickle ok -# a['SubParsersActionType'] = _parser.add_subparsers() -# a['MutuallyExclusiveGroupType'] = _parser.add_mutually_exclusive_group() -# a['ArgumentGroupType'] = _parser.add_argument_group() -except AttributeError: - pass +# numeric and mathematical types (CH 9) +a['ProductType'] = itertools.product('0','1') +# generic operating system services (CH 15) +a['FileHandlerType'] = logging.FileHandler(os.devnull) +a['RotatingFileHandlerType'] = logging.handlers.RotatingFileHandler(os.devnull) +a['SocketHandlerType'] = logging.handlers.SocketHandler('localhost',514) +a['MemoryHandlerType'] = logging.handlers.MemoryHandler(1) +# data types (CH 8) +a['WeakSetType'] = weakref.WeakSet() # 2.7 +# generic operating system services (CH 15) [errors when dill is imported] +#a['ArgumentParserType'] = _parser = argparse.ArgumentParser('PROG') +#a['NamespaceType'] = _parser.parse_args() # pickle ok +#a['SubParsersActionType'] = _parser.add_subparsers() +#a['MutuallyExclusiveGroupType'] = _parser.add_mutually_exclusive_group() +#a['ArgumentGroupType'] = _parser.add_argument_group() # -- dill fails in some versions below here --------------------------------- # types module (part of CH 8) @@ -431,14 +368,10 @@ class _Struct(ctypes.Structure): d["OdictItemsType"] = X.items() a["OdictIteratorType"] = iter(X.keys()) #FIXME: list_iterator del X -if PY3: #FIXME: list_iterator - a['DictionaryItemIteratorType'] = iter(type.__dict__.items()) - a['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) - a['DictionaryValueIteratorType'] = iter(type.__dict__.values()) -else: - a['DictionaryItemIteratorType'] = type.__dict__.iteritems() - a['DictionaryKeyIteratorType'] = type.__dict__.iterkeys() - a['DictionaryValueIteratorType'] = type.__dict__.itervalues() +#FIXME: list_iterator +a['DictionaryItemIteratorType'] = iter(type.__dict__.items()) +a['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) +a['DictionaryValueIteratorType'] = iter(type.__dict__.values()) if sys.hexversion >= 0x30800a0: a["DictReversekeyiteratorType"] = reversed({}.keys()) a["DictReversevalueiteratorType"] = reversed({}.values()) @@ -446,8 +379,9 @@ class _Struct(ctypes.Structure): try: import symtable + #FIXME: fails to pickle x["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table -except: #FIXME: fails to pickle +except ImportError: pass if sys.hexversion >= 0x30a00a0: @@ -468,19 +402,12 @@ class _Struct(ctypes.Structure): x['GzipFileType'] = gzip.GzipFile(fileobj=_fileW) # generic operating system services (CH 15) a['StreamHandlerType'] = logging.StreamHandler() -try: # python 2.6 - # numeric and mathematical types (CH 9) - a['PermutationsType'] = itertools.permutations('0') - a['CombinationsType'] = itertools.combinations('0',1) -except AttributeError: - pass -try: # python 2.7 - # numeric and mathematical types (CH 9) - a['RepeatType'] = itertools.repeat(0) - a['CompressType'] = itertools.compress('0',[1]) - #XXX: ...and etc -except AttributeError: - pass +# numeric and mathematical types (CH 9) +a['PermutationsType'] = itertools.permutations('0') +a['CombinationsType'] = itertools.combinations('0',1) +a['RepeatType'] = itertools.repeat(0) +a['CompressType'] = itertools.compress('0',[1]) +#XXX: ...and etc # -- dill fails on all below here ------------------------------------------- # types module (part of CH 8) @@ -513,7 +440,7 @@ class _Struct(ctypes.Structure): x['CSVDictWriterType'] = csv.DictWriter(_cstrO,{}) # cryptographic services (CH 14) x['HashType'] = hashlib.md5() -if (hex(sys.hexversion) < '0x30800a1'): +if (sys.hexversion < 0x30800a1): x['HMACType'] = hmac.new(_in) else: x['HMACType'] = hmac.new(_in, digestmod='md5') @@ -544,38 +471,22 @@ class _Struct(ctypes.Structure): x['FieldType'] = _field = _Struct._field x['CFUNCTYPEType'] = _cfunc = ctypes.CFUNCTYPE(ctypes.c_char) x['CFunctionType'] = _cfunc(str) -try: # python 2.6 - # numeric and mathematical types (CH 9) - a['MethodCallerType'] = operator.methodcaller('mro') # 2.6 -except AttributeError: - pass -try: # python 2.7 - # built-in types (CH 5) - x['MemoryType'] = memoryview(_in) # 2.7 - x['MemoryType2'] = memoryview(bytearray(_in)) # 2.7 - if PY3: - d['DictItemsType'] = _dict.items() # 2.7 - d['DictKeysType'] = _dict.keys() # 2.7 - d['DictValuesType'] = _dict.values() # 2.7 - else: - d['DictItemsType'] = _dict.viewitems() # 2.7 - d['DictKeysType'] = _dict.viewkeys() # 2.7 - d['DictValuesType'] = _dict.viewvalues() # 2.7 - # generic operating system services (CH 15) - a['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG') - a['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG') - a['ArgDefaultsHelpFormatterType'] = argparse.ArgumentDefaultsHelpFormatter('PROG') -except NameError: - pass -try: # python 2.7 (and not 3.1) - x['CmpKeyType'] = _cmpkey = functools.cmp_to_key(_methodwrap) # 2.7, >=3.2 - x['CmpKeyObjType'] = _cmpkey('0') #2.7, >=3.2 -except AttributeError: - pass -if PY3: # oddities: removed, etc - x['BufferType'] = x['MemoryType'] -else: - x['BufferType'] = buffer('') +# numeric and mathematical types (CH 9) +a['MethodCallerType'] = operator.methodcaller('mro') # 2.6 +# built-in types (CH 5) +x['MemoryType'] = memoryview(_in) # 2.7 +x['MemoryType2'] = memoryview(bytearray(_in)) # 2.7 +d['DictItemsType'] = _dict.items() # 2.7 +d['DictKeysType'] = _dict.keys() # 2.7 +d['DictValuesType'] = _dict.values() # 2.7 +# generic operating system services (CH 15) +a['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG') +a['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG') +a['ArgDefaultsHelpFormatterType'] = argparse.ArgumentDefaultsHelpFormatter('PROG') +x['CmpKeyType'] = _cmpkey = functools.cmp_to_key(_methodwrap) # 2.7, >=3.2 +x['CmpKeyObjType'] = _cmpkey('0') #2.7, >=3.2 +# oddities: removed, etc +x['BufferType'] = x['MemoryType'] from dill._dill import _testcapsule if _testcapsule is not None: diff --git a/dill/_shims.py b/dill/_shims.py index 2da6f5be..2e6641ca 100644 --- a/dill/_shims.py +++ b/dill/_shims.py @@ -49,7 +49,8 @@ def _setattr(object, name, value): https://github.com/uqfoundation/dill/pull/443 """ -import inspect, sys +import inspect +import sys _dill = sys.modules['dill._dill'] @@ -188,105 +189,5 @@ def _create_class(): _CELL_EMPTY = register_shim('_CELL_EMPTY', None) -if _dill.OLD37: - if _dill.HAS_CTYPES and hasattr(_dill.ctypes, 'pythonapi') and hasattr(_dill.ctypes.pythonapi, 'PyCell_Set'): - # CPython - ctypes = _dill.ctypes - - _PyCell_Set = ctypes.pythonapi.PyCell_Set - - def _setattr(object, name, value): - if type(object) is _dill.CellType and name == 'cell_contents': - _PyCell_Set.argtypes = (ctypes.py_object, ctypes.py_object) - _PyCell_Set(object, value) - else: - setattr(object, name, value) - - def _delattr(object, name): - if type(object) is _dill.CellType and name == 'cell_contents': - _PyCell_Set.argtypes = (ctypes.py_object, ctypes.c_void_p) - _PyCell_Set(object, None) - else: - delattr(object, name) - - # General Python (not CPython) up to 3.6 is in a weird case, where it is - # possible to pickle recursive cells, but we can't assign directly to the - # cell. - elif _dill.PY3: - # Use nonlocal variables to reassign the cell value. - # https://stackoverflow.com/a/59276835 - __nonlocal = ('nonlocal cell',) - exec('''def _setattr(cell, name, value): - if type(cell) is _dill.CellType and name == 'cell_contents': - def cell_setter(value): - %s - cell = value # pylint: disable=unused-variable - func = _dill.FunctionType(cell_setter.__code__, globals(), "", None, (cell,)) # same as cell_setter, but with cell being the cell's contents - func(value) - else: - setattr(cell, name, value)''' % __nonlocal) - - exec('''def _delattr(cell, name): - if type(cell) is _dill.CellType and name == 'cell_contents': - try: - cell.cell_contents - except: - return - def cell_deleter(): - %s - del cell # pylint: disable=unused-variable - func = _dill.FunctionType(cell_deleter.__code__, globals(), "", None, (cell,)) # same as cell_deleter, but with cell being the cell's contents - func() - else: - delattr(cell, name)''' % __nonlocal) - - else: - # Likely PyPy 2.7. Simulate the nonlocal keyword with bytecode - # manipulation. - - # The following function is based on 'cell_set' from 'cloudpickle' - # https://github.com/cloudpipe/cloudpickle/blob/5d89947288a18029672596a4d719093cc6d5a412/cloudpickle/cloudpickle.py#L393-L482 - # Copyright (c) 2012, Regents of the University of California. - # Copyright (c) 2009 `PiCloud, Inc. `_. - # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE - def _setattr(cell, name, value): - if type(cell) is _dill.CellType and name == 'cell_contents': - _cell_set = _dill.FunctionType( - _cell_set_template_code, {}, '_cell_set', (), (cell,),) - _cell_set(value) - else: - setattr(cell, name, value) - - def _cell_set_factory(value): - lambda: cell - cell = value - - co = _cell_set_factory.__code__ - - _cell_set_template_code = _dill.CodeType( - co.co_argcount, - co.co_nlocals, - co.co_stacksize, - co.co_flags, - co.co_code, - co.co_consts, - co.co_names, - co.co_varnames, - co.co_filename, - co.co_name, - co.co_firstlineno, - co.co_lnotab, - co.co_cellvars, # co_freevars is initialized with co_cellvars - (), # co_cellvars is made empty - ) - - del co - - def _delattr(cell, name): - if type(cell) is _dill.CellType and name == 'cell_contents': - pass - else: - delattr(cell, name) - _setattr = register_shim('_setattr', setattr) _delattr = register_shim('_delattr', delattr) diff --git a/dill/detect.py b/dill/detect.py index 26d5b219..b6a6cb76 100644 --- a/dill/detect.py +++ b/dill/detect.py @@ -11,9 +11,8 @@ import dis from inspect import ismethod, isfunction, istraceback, isframe, iscode -from .pointers import parent, reference, at, parents, children -from ._dill import PY3 +from .pointers import parent, reference, at, parents, children from .logger import trace __all__ = ['baditems','badobjects','badtypes','code','errors','freevars', @@ -25,9 +24,7 @@ def getmodule(object, _filename=None, force=False): from inspect import getmodule as getmod module = getmod(object, _filename) if module or not force: return module - if PY3: builtins = 'builtins' - else: builtins = '__builtin__' - builtins = __import__(builtins) + import builtins from .source import getname name = getname(object, force=True) return builtins if name in vars(builtins).keys() else None @@ -37,26 +34,17 @@ def outermost(func): # is analogous to getsource(func,enclosing=True) NOTE: this is the object-equivalent of getsource(func, enclosing=True) """ - if PY3: - if ismethod(func): - _globals = func.__func__.__globals__ or {} - elif isfunction(func): - _globals = func.__globals__ or {} - else: - return #XXX: or raise? no matches - _globals = _globals.items() + if ismethod(func): + _globals = func.__func__.__globals__ or {} + elif isfunction(func): + _globals = func.__globals__ or {} else: - if ismethod(func): - _globals = func.im_func.func_globals or {} - elif isfunction(func): - _globals = func.func_globals or {} - else: - return #XXX: or raise? no matches - _globals = _globals.iteritems() + return #XXX: or raise? no matches + _globals = _globals.items() # get the enclosing source from .source import getsourcelines try: lines,lnum = getsourcelines(func, enclosing=True) - except: #TypeError, IOError + except Exception: #TypeError, IOError lines,lnum = [],None code = ''.join(lines) # get all possible names,objects that are named in the enclosing source @@ -65,7 +53,7 @@ def outermost(func): # is analogous to getsource(func,enclosing=True) for name,obj in _locals: #XXX: don't really need 'name' try: if getsourcelines(obj) == (lines,lnum): return obj - except: #TypeError, IOError + except Exception: #TypeError, IOError pass return #XXX: or raise? no matches @@ -83,18 +71,12 @@ def nestedcode(func, recurse=True): #XXX: or return dict of {co_name: co} ? return list(nested) def code(func): - '''get the code object for the given function or method + """get the code object for the given function or method NOTE: use dill.source.getsource(CODEOBJ) to get the source code - ''' - if PY3: - im_func = '__func__' - func_code = '__code__' - else: - im_func = 'im_func' - func_code = 'func_code' - if ismethod(func): func = getattr(func, im_func) - if isfunction(func): func = getattr(func, func_code) + """ + if ismethod(func): func = func.__func__ + if isfunction(func): func = func.__code__ if istraceback(func): func = func.tb_frame if isframe(func): func = func.f_code if iscode(func): return func @@ -109,13 +91,6 @@ def referrednested(func, recurse=True): #XXX: return dict of {__name__: obj} ? If possible, python builds code objects, but delays building functions until func() is called. """ - if PY3: - att1 = '__code__' - att0 = '__func__' - else: - att1 = 'func_code' # functions - att0 = 'im_func' # methods - import gc funcs = set() # get the code objects, and try to track down by referrence @@ -123,16 +98,16 @@ def referrednested(func, recurse=True): #XXX: return dict of {__name__: obj} ? # look for function objects that refer to the code object for obj in gc.get_referrers(co): # get methods - _ = getattr(obj, att0, None) # ismethod - if getattr(_, att1, None) is co: funcs.add(obj) + _ = getattr(obj, '__func__', None) # ismethod + if getattr(_, '__code__', None) is co: funcs.add(obj) # get functions - elif getattr(obj, att1, None) is co: funcs.add(obj) + elif getattr(obj, '__code__', None) is co: funcs.add(obj) # get frame objects elif getattr(obj, 'f_code', None) is co: funcs.add(obj) # get code objects elif hasattr(obj, 'co_code') and obj is co: funcs.add(obj) -# frameobjs => func.func_code.co_varnames not in func.func_code.co_cellvars -# funcobjs => func.func_code.co_cellvars not in func.func_code.co_varnames +# frameobjs => func.__code__.co_varnames not in func.__code__.co_cellvars +# funcobjs => func.__code__.co_cellvars not in func.__code__.co_varnames # frameobjs are not found, however funcobjs are... # (see: test_mixins.quad ... and test_mixins.wtf) # after execution, code objects get compiled, and then may be found by gc @@ -143,28 +118,20 @@ def freevars(func): """get objects defined in enclosing code that are referred to by func returns a dict of {name:object}""" - if PY3: - im_func = '__func__' - func_code = '__code__' - func_closure = '__closure__' - else: - im_func = 'im_func' - func_code = 'func_code' - func_closure = 'func_closure' - if ismethod(func): func = getattr(func, im_func) + if ismethod(func): func = func.__func__ if isfunction(func): - closures = getattr(func, func_closure) or () - func = getattr(func, func_code).co_freevars # get freevars + closures = func.__closure__ or () + func = func.__code__.co_freevars # get freevars else: return {} def get_cell_contents(): - for (name,c) in zip(func,closures): + for name, c in zip(func, closures): try: cell_contents = c.cell_contents - except: + except ValueError: # cell is empty continue - yield (name,c.cell_contents) + yield name, c.cell_contents return dict(get_cell_contents()) @@ -175,7 +142,7 @@ def nestedglobals(func, recurse=True): if func is None: return list() import sys from .temp import capture - CAN_NULL = sys.hexversion >= 51052711 #NULL may be prepended >= 3.11a7 + CAN_NULL = sys.hexversion >= 0x30b00a7 # NULL may be prepended >= 3.11a7 names = set() with capture('stdout') as out: dis.dis(func) #XXX: dis.dis(None) disassembles last traceback @@ -199,37 +166,27 @@ def globalvars(func, recurse=True, builtin=False): """get objects defined in global scope that are referred to by func return a dict of {name:object}""" - if PY3: - im_func = '__func__' - func_code = '__code__' - func_globals = '__globals__' - func_closure = '__closure__' - else: - im_func = 'im_func' - func_code = 'func_code' - func_globals = 'func_globals' - func_closure = 'func_closure' - if ismethod(func): func = getattr(func, im_func) + if ismethod(func): func = func.__func__ if isfunction(func): globs = vars(getmodule(sum)).copy() if builtin else {} # get references from within closure orig_func, func = func, set() - for obj in getattr(orig_func, func_closure) or {}: + for obj in orig_func.__closure__ or {}: try: cell_contents = obj.cell_contents - except: + except ValueError: # cell is empty pass else: _vars = globalvars(cell_contents, recurse, builtin) or {} func.update(_vars) #XXX: (above) be wary of infinte recursion? globs.update(_vars) # get globals - globs.update(getattr(orig_func, func_globals) or {}) + globs.update(orig_func.__globals__ or {}) # get names of references if not recurse: - func.update(getattr(orig_func, func_code).co_names) + func.update(orig_func.__code__.co_names) else: - func.update(nestedglobals(getattr(orig_func, func_code))) + func.update(nestedglobals(orig_func.__code__)) # find globals for all entries of func for key in func.copy(): #XXX: unnecessary...? nested_func = globs.get(key) @@ -254,7 +211,7 @@ def globalvars(func, recurse=True, builtin=False): func.update(globalvars(nested_func, True, builtin)) else: return {} - #NOTE: if name not in func_globals, then we skip it... + #NOTE: if name not in __globals__, then we skip it... return dict((name,globs[name]) for name in func if name in globs) diff --git a/dill/logger.py b/dill/logger.py index 7fa2856c..fedff6bf 100644 --- a/dill/logger.py +++ b/dill/logger.py @@ -215,9 +215,9 @@ def format(self, record): def trace(arg: Union[bool, TextIO, str, os.PathLike] = None, *, mode: str = 'a') -> NoReturn: """print a trace through the stack when pickling; useful for debugging - + With a single boolean argument, enable or disable the tracing. - + Example usage: >>> import dill diff --git a/dill/settings.py b/dill/settings.py index 4d0226b0..b105d2e8 100644 --- a/dill/settings.py +++ b/dill/settings.py @@ -9,10 +9,7 @@ global settings for Pickler """ -try: - from pickle import DEFAULT_PROTOCOL -except ImportError: - from pickle import HIGHEST_PROTOCOL as DEFAULT_PROTOCOL +from pickle import DEFAULT_PROTOCOL settings = { #'main' : None, diff --git a/dill/source.py b/dill/source.py index 47064a16..7e70a635 100644 --- a/dill/source.py +++ b/dill/source.py @@ -29,8 +29,6 @@ ismodule, istraceback) from tokenize import TokenError -from ._dill import PY3 - def isfrommain(obj): "check if object was built in __main__" @@ -58,7 +56,7 @@ def _matchlambda(func, line): lhs,rhs = line.split('lambda ',1)[-1].split(":", 1) #FIXME: if !1 inputs try: #FIXME: unsafe _ = eval("lambda %s : %s" % (lhs,rhs), globals(),locals()) - except: _ = dummy + except Exception: _ = dummy # get code objects, for comparison _, code = getcode(_).co_code, getcode(func).co_code # check if func is in closure @@ -80,7 +78,7 @@ def _matchlambda(func, line): _lhs,_rhs = rhs.split('lambda ',1)[-1].split(":",1) #FIXME: if !1 inputs try: #FIXME: unsafe _f = eval("lambda %s : %s" % (_lhs,_rhs), globals(),locals()) - except: _f = dummy + except Exception: _f = dummy # get code objects, for comparison _, code = getcode(_f).co_code, getcode(func).co_code if len(_) != len(code): return False @@ -120,7 +118,7 @@ def findsource(object): try: import readline err = '' - except: + except ImportError: import sys err = sys.exc_info()[1].args[0] if sys.platform[:3] == 'win': @@ -165,16 +163,14 @@ def findsource(object): name = object.__name__ if name == '': pat1 = r'(.*(?': pat1 = r'(.*(?', 'exec'), *a) __globals__ = globals() __locals__ = locals() -wrap2 = ''' -def _wrap(f): - """ encapsulate a function and it's __import__ """ - def func(*args, **kwds): - try: - # _ = eval(getsource(f, force=True)) #XXX: safer but less robust - exec getimportable(f, alias='_') in %s, %s - except: - raise ImportError('cannot import name ' + f.__name__) - return _(*args, **kwds) - func.__name__ = f.__name__ - func.__doc__ = f.__doc__ - return func -''' % ('__globals__', '__locals__') -wrap3 = ''' def _wrap(f): """ encapsulate a function and it's __import__ """ def func(*args, **kwds): try: # _ = eval(getsource(f, force=True)) #XXX: safer but less robust - exec(getimportable(f, alias='_'), %s, %s) - except: + exec(getimportable(f, alias='_'), __globals__, __locals__) + except Exception: raise ImportError('cannot import name ' + f.__name__) return _(*args, **kwds) func.__name__ = f.__name__ func.__doc__ = f.__doc__ return func -''' % ('__globals__', '__locals__') -if PY3: - exec(wrap3) -else: - exec(wrap2) -del wrap2, wrap3 def _enclose(object, alias=''): #FIXME: needs alias to hold returned object @@ -588,10 +561,7 @@ def dumpsource(object, alias='', new=False, enclose=True): else: #XXX: other cases where source code is needed??? code += getsource(object.__class__, alias='', lstrip=True, force=True) mod = repr(object.__module__) # should have a module (no builtins here) - if PY3: - code += pre + 'dill.loads(%s.replace(b%s,bytes(__name__,"UTF-8")))\n' % (pik,mod) - else: - code += pre + 'dill.loads(%s.replace(%s,__name__))\n' % (pik,mod) + code += pre + 'dill.loads(%s.replace(b%s,bytes(__name__,"UTF-8")))\n' % (pik,mod) #code += 'del %s' % object.__class__.__name__ #NOTE: kills any existing! if enclose: @@ -654,7 +624,7 @@ def _namespace(obj): if module in ['builtins','__builtin__']: # BuiltinFunctionType if _intypes(name): return ['types'] + [name] return qual + [name] #XXX: can be wrong for some aliased objects - except: pass + except Exception: pass # special case: numpy.inf and numpy.nan (we don't want them as floats) if str(obj) in ['inf','nan','Inf','NaN']: # is more, but are they needed? return ['numpy'] + [str(obj)] @@ -742,7 +712,7 @@ def getimport(obj, alias='', verify=True, builtin=False, enclosing=False): try: # look for '<...>' and be mindful it might be in lists, dicts, etc... name = repr(obj).split('<',1)[1].split('>',1)[1] name = None # we have a 'object'-style repr - except: # it's probably something 'importable' + except Exception: # it's probably something 'importable' if head in ['builtins','__builtin__']: name = repr(obj) #XXX: catch [1,2], (1,2), set([1,2])... others? else: @@ -800,7 +770,7 @@ def _importable(obj, alias='', source=None, enclosing=False, force=True, \ try: return getsource(obj, alias, enclosing=enclosing, \ force=force, lstrip=lstrip, builtin=builtin) - except: pass + except Exception: pass try: if not _isinstance(obj): return getimport(obj, alias, enclosing=enclosing, \ @@ -815,12 +785,12 @@ def _importable(obj, alias='', source=None, enclosing=False, force=True, \ if alias == name: _alias = "" return _import+_alias+"%s\n" % name - except: pass + except Exception: pass if not source: # try getsource, only if it hasn't been tried yet try: return getsource(obj, alias, enclosing=enclosing, \ force=force, lstrip=lstrip, builtin=builtin) - except: pass + except Exception: pass # get the name (of functions, lambdas, and classes) # or hope that obj can be built from the __repr__ #XXX: what to do about class instances and such? @@ -960,7 +930,7 @@ def importable(obj, alias='', source=None, builtin=True): if len(src) > 1: raise NotImplementedError('not implemented') return list(src.values())[0] - except: + except Exception: if tried_source: raise tried_import = True # we want the source @@ -999,7 +969,7 @@ def _code_stitcher(block): if not obj: return src if not src: return obj return obj + src - except: + except Exception: if tried_import: raise tried_source = True source = not source diff --git a/dill/temp.py b/dill/temp.py index 251a8e30..215ba63a 100644 --- a/dill/temp.py +++ b/dill/temp.py @@ -17,7 +17,6 @@ 'capture'] import contextlib -from ._dill import PY3 @contextlib.contextmanager @@ -32,10 +31,7 @@ def capture(stream='stdout'): """ import sys - if PY3: - from io import StringIO - else: - from StringIO import StringIO + from io import StringIO orig = getattr(sys, stream) setattr(sys, stream, StringIO()) try: @@ -176,10 +172,7 @@ def loadIO(buffer, **kwds): [1, 2, 3, 4, 5] """ import dill as pickle - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO + from io import BytesIO as StringIO value = getattr(buffer, 'getvalue', buffer) # value or buffer.getvalue if value != buffer: value = value() # buffer.getvalue() return pickle.load(StringIO(value)) @@ -193,10 +186,7 @@ def dumpIO(object, **kwds): [1, 2, 3, 4, 5] """ import dill as pickle - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO + from io import BytesIO as StringIO file = StringIO() pickle.dump(object, file) file.flush() @@ -217,7 +207,7 @@ def loadIO_source(buffer, **kwds): alias = kwds.pop('alias', None) source = getattr(buffer, 'getvalue', buffer) # source or buffer.getvalue if source != buffer: source = source() # buffer.getvalue() - if PY3: source = source.decode() # buffer to string + source = source.decode() # buffer to string if not alias: tag = source.strip().splitlines()[-1].split() if tag[0] != '#NAME:': @@ -243,10 +233,7 @@ def dumpIO_source(object, **kwds): If 'alias' is specified, the object will be renamed to the given string. """ from .source import importable, getname - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO + from io import BytesIO as StringIO alias = kwds.pop('alias', '') #XXX: include an alias so a name is known name = str(alias) or getname(object) name = "\n#NAME: %s\n" % name diff --git a/docs/source/conf.py b/docs/source/conf.py index 19171caf..cbf7cf14 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -58,10 +58,10 @@ master_doc = 'index' # General information about the project. -project = u'dill' +project = 'dill' year = datetime.now().year -copyright = u'%d, The Uncertainty Quantification Foundation' % year -author = u'Mike McKerns' +copyright = '%d, The Uncertainty Quantification Foundation' % year +author = 'Mike McKerns' # extension config github_project_url = "https://github.com/uqfoundation/dill" @@ -191,8 +191,8 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'dill.tex', u'dill Documentation', - u'Mike McKerns', 'manual'), + (master_doc, 'dill.tex', 'dill Documentation', + 'Mike McKerns', 'manual'), ] @@ -201,7 +201,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'dill', u'dill Documentation', + (master_doc, 'dill', 'dill Documentation', [author], 1) ] @@ -212,7 +212,7 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'dill', u'dill Documentation', + (master_doc, 'dill', 'dill Documentation', author, 'dill', 'Serialize all of python.', 'Miscellaneous'), ] diff --git a/tests/__main__.py b/tests/__main__.py index 7f7be536..1570b399 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -27,4 +27,4 @@ p = sp.Popen([python, test], shell=shell).wait() if not p: print('.', end='', flush=True) - print('') + print() diff --git a/tests/test_check.py b/tests/test_check.py index 3b3fc590..0a22b276 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -10,7 +10,6 @@ import sys from dill.temp import capture -from dill._dill import PY3 #FIXME: this doesn't catch output... it's from the internal call diff --git a/tests/test_classdef.py b/tests/test_classdef.py index 47f0d00f..8edf5daf 100644 --- a/tests/test_classdef.py +++ b/tests/test_classdef.py @@ -90,19 +90,15 @@ def test_specialtypes(): assert dill.pickles(type(NotImplemented)) assert dill.pickles(type(Ellipsis)) -if hex(sys.hexversion) >= '0x20600f0': - from collections import namedtuple - Z = namedtuple("Z", ['a','b']) - Zi = Z(0,1) - X = namedtuple("Y", ['a','b']) - X.__name__ = "X" - if hex(sys.hexversion) >= '0x30300f0': - X.__qualname__ = "X" #XXX: name must 'match' or fails to pickle - Xi = X(0,1) - Bad = namedtuple("FakeName", ['a','b']) - Badi = Bad(0,1) -else: - Z = Zi = X = Xi = Bad = Badi = None +from collections import namedtuple +Z = namedtuple("Z", ['a','b']) +Zi = Z(0,1) +X = namedtuple("Y", ['a','b']) +X.__name__ = "X" +X.__qualname__ = "X" #XXX: name must 'match' or fails to pickle +Xi = X(0,1) +Bad = namedtuple("FakeName", ['a','b']) +Badi = Bad(0,1) # test namedtuple def test_namedtuple(): @@ -123,8 +119,7 @@ class B(namedtuple("B", ["one", "two"])): assert dill.copy(a) assert dill.copy(A.B).__name__ == 'B' - if dill._dill.PY3: - assert dill.copy(A.B).__qualname__.endswith('..A.B') + assert dill.copy(A.B).__qualname__.endswith('..A.B') assert dill.copy(A.B).__doc__ == 'docstring' assert dill.copy(A.B).__module__ == 'testing' @@ -168,12 +163,12 @@ def __getnewargs__(self): return np.asarray(self), self.color a1 = TestArray(np.zeros(100), color='green') - if dill._dill.PY3 and not dill._dill.IS_PYPY: + if not dill._dill.IS_PYPY: assert dill.pickles(a1) assert a1.__dict__ == dill.copy(a1).__dict__ a2 = a1[0:9] - if dill._dill.PY3 and not dill._dill.IS_PYPY: + if not dill._dill.IS_PYPY: assert dill.pickles(a2) assert a2.__dict__ == dill.copy(a2).__dict__ @@ -182,7 +177,7 @@ class TestArray2(np.ndarray): a3 = TestArray2([1,2,3,4,5]) a3.color = 'green' - if dill._dill.PY3 and not dill._dill.IS_PYPY: + if not dill._dill.IS_PYPY: assert dill.pickles(a3) assert a3.__dict__ == dill.copy(a3).__dict__ @@ -226,37 +221,21 @@ class A: assert dill.copy(v) == v def test_metaclass(): - if dill._dill.PY3: - class metaclass_with_new(type): - def __new__(mcls, name, bases, ns, **kwds): - cls = super().__new__(mcls, name, bases, ns, **kwds) - assert mcls is not None - assert cls.method(mcls) - return cls - def method(cls, mcls): - return isinstance(cls, mcls) - - l = locals() - exec("""class subclass_with_new(metaclass=metaclass_with_new): - def __new__(cls): - self = super().__new__(cls) - return self""", None, l) - subclass_with_new = l['subclass_with_new'] - else: - class metaclass_with_new(type): - def __new__(mcls, name, bases, ns, **kwds): - cls = super(mcls, metaclass_with_new).__new__(mcls, name, bases, ns, **kwds) - assert mcls is not None - assert cls.method(mcls) - return cls - def method(cls, mcls): - return isinstance(cls, mcls) - - class subclass_with_new: - __metaclass__ = metaclass_with_new - def __new__(cls): - self = super(subclass_with_new, cls).__new__(cls) - return self + class metaclass_with_new(type): + def __new__(mcls, name, bases, ns, **kwds): + cls = super().__new__(mcls, name, bases, ns, **kwds) + assert mcls is not None + assert cls.method(mcls) + return cls + def method(cls, mcls): + return isinstance(cls, mcls) + + l = locals() + exec("""class subclass_with_new(metaclass=metaclass_with_new): + def __new__(cls): + self = super().__new__(cls) + return self""", None, l) + subclass_with_new = l['subclass_with_new'] assert dill.copy(subclass_with_new()) diff --git a/tests/test_detect.py b/tests/test_detect.py index 06339acd..4dac7aaf 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -8,7 +8,7 @@ from dill.detect import baditems, badobjects, badtypes, errors, parent, at, globalvars from dill import settings -from dill._dill import IS_PYPY, IS_PYPY2 +from dill._dill import IS_PYPY from pickle import PicklingError import inspect @@ -21,7 +21,7 @@ def test_bad_things(): #assert baditems(globals()) == [f] #XXX assert badobjects(f) is f assert badtypes(f) == type(f) - assert type(errors(f)) is PicklingError if IS_PYPY2 else TypeError + assert type(errors(f)) is TypeError d = badtypes(f, 1) assert isinstance(d, dict) assert list(badobjects(f, 1).keys()) == list(d.keys()) @@ -30,7 +30,7 @@ def test_bad_things(): a = dict(s) if not os.environ.get('COVERAGE'): #XXX: travis-ci assert len(s) is len(a) # TypeError (and possibly PicklingError) - n = 1 if IS_PYPY2 else 2 + n = 2 assert len(a) is n if 'PicklingError' in a.keys() else n-1 def test_parent(): diff --git a/tests/test_diff.py b/tests/test_diff.py index 3277682b..652d819b 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -55,18 +55,15 @@ def test_diff(): assert changed[1] if not IS_PYPY: - try: - import abc - # make sure the "_abc_invaldation_counter" doesn't make test fail - diff.memorise(abc.ABCMeta, force=True) - assert not diff.has_changed(abc) - abc.ABCMeta.zzz = 1 - assert diff.has_changed(abc) - changed = diff.whats_changed(abc) - assert list(changed[0].keys()) == ["ABCMeta"] - assert not changed[1] - except ImportError: - pass + import abc + # make sure the "_abc_invaldation_counter" doesn't make test fail + diff.memorise(abc.ABCMeta, force=True) + assert not diff.has_changed(abc) + abc.ABCMeta.zzz = 1 + assert diff.has_changed(abc) + changed = diff.whats_changed(abc) + assert list(changed[0].keys()) == ["ABCMeta"] + assert not changed[1] ''' import Queue diff --git a/tests/test_extendpickle.py b/tests/test_extendpickle.py index 49b6f152..10dc0804 100644 --- a/tests/test_extendpickle.py +++ b/tests/test_extendpickle.py @@ -7,10 +7,7 @@ # - https://github.com/uqfoundation/dill/blob/master/LICENSE import dill as pickle -try: - from StringIO import StringIO -except ImportError: - from io import BytesIO as StringIO +from io import BytesIO as StringIO def my_fn(x): @@ -47,7 +44,7 @@ def test_isdill(): pickler = mp.reduction.ForkingPickler(obj_io) assert pickle._dill.is_dill(pickler, child=True) is True assert pickle._dill.is_dill(pickler, child=False) is False - except: + except Exception: pass diff --git a/tests/test_file.py b/tests/test_file.py index 8118e3a8..b766552e 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -19,8 +19,6 @@ fname = "_test_file.txt" rand_chars = list(string.ascii_letters) + ["\n"] * 40 # bias newline -if sys.hexversion < 0x03030000: - FileNotFoundError = IOError buffer_error = ValueError("invalid buffer size") dne_error = FileNotFoundError("[Errno 2] No such file or directory: '%s'" % fname) diff --git a/tests/test_functions.py b/tests/test_functions.py index a86cbde0..d82c37e3 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -11,10 +11,6 @@ dill.settings['recurse'] = True -def is_py3(): - return hex(sys.hexversion) >= '0x30000f0' - - def function_a(a): return a @@ -34,18 +30,17 @@ def function_d(d, d1, d2=1): function_d.__module__ = 'a module' -if is_py3(): - exec(''' +exec(''' def function_e(e, *e1, e2=1, e3=2): return e + sum(e1) + e2 + e3''') - globalvar = 0 +globalvar = 0 - @functools.lru_cache(None) - def function_with_cache(x): - global globalvar - globalvar += x - return globalvar +@functools.lru_cache(None) +def function_with_cache(x): + global globalvar + globalvar += x + return globalvar def function_with_unassigned_variable(): @@ -87,28 +82,26 @@ def test_functions(): assert dill.loads(dumped_func_d)(1, 2, 3) == 6 assert dill.loads(dumped_func_d)(1, 2, d2=3) == 6 - if is_py3(): - function_with_cache(1) - globalvar = 0 - dumped_func_cache = dill.dumps(function_with_cache) - assert function_with_cache(2) == 3 - assert function_with_cache(1) == 1 - assert function_with_cache(3) == 6 - assert function_with_cache(2) == 3 + function_with_cache(1) + globalvar = 0 + dumped_func_cache = dill.dumps(function_with_cache) + assert function_with_cache(2) == 3 + assert function_with_cache(1) == 1 + assert function_with_cache(3) == 6 + assert function_with_cache(2) == 3 empty_cell = function_with_unassigned_variable() cell_copy = dill.loads(dill.dumps(empty_cell)) assert 'empty' in str(cell_copy.__closure__[0]) try: cell_copy() - except: + except Exception: # this is good pass else: raise AssertionError('cell_copy() did not read an empty cell') - if is_py3(): - exec(''' + exec(''' dumped_func_e = dill.dumps(function_e) assert dill.loads(dumped_func_e)(1, 2) == 6 assert dill.loads(dumped_func_e)(1, 2, 3) == 9 diff --git a/tests/test_module.py b/tests/test_module.py index 5c5e000c..2d009f26 100644 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -9,8 +9,7 @@ import sys import dill import test_mixins as module -try: from importlib import reload -except ImportError: pass +from importlib import reload dill.settings['recurse'] = True cached = (module.__cached__ if hasattr(module, "__cached__") diff --git a/tests/test_nested.py b/tests/test_nested.py index 144f54b0..950641ca 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -110,7 +110,7 @@ def test_pickled_inner(): def test_moduledict_where_not_main(): try: from . import test_moduledict - except: + except ImportError: import test_moduledict name = 'test_moduledict.py' if os.path.exists(name) and os.path.exists(name+'c'): diff --git a/tests/test_recursive.py b/tests/test_recursive.py index ee71a688..0f3a53a1 100644 --- a/tests/test_recursive.py +++ b/tests/test_recursive.py @@ -6,7 +6,6 @@ # - https://github.com/uqfoundation/dill/blob/master/LICENSE import dill -from dill._dill import PY3 from functools import partial import warnings @@ -15,7 +14,7 @@ def copy(obj, byref=False, recurse=False): if byref: try: return dill.copy(obj, byref=byref, recurse=recurse) - except: + except Exception: pass else: raise AssertionError('Copy of %s with byref=True should have given a warning!' % (obj,)) @@ -113,9 +112,8 @@ def __init__(self): def test_circular_reference(): assert copy(obj4()) obj4_copy = dill.loads(dill.dumps(obj4())) - if PY3: - assert type(obj4_copy) is type(obj4_copy).__init__.__closure__[0].cell_contents - assert type(obj4_copy.b) is type(obj4_copy.b).__init__.__closure__[0].cell_contents + assert type(obj4_copy) is type(obj4_copy).__init__.__closure__[0].cell_contents + assert type(obj4_copy.b) is type(obj4_copy.b).__init__.__closure__[0].cell_contents def f(): @@ -146,7 +144,7 @@ def test_recursive_function(): for _fib in (fib3, fib4): try: _fib(5) - except: + except Exception: # This is expected to fail because fib no longer exists pass else: diff --git a/tests/test_selected.py b/tests/test_selected.py index 9eef8d2d..59ff2ce1 100644 --- a/tests/test_selected.py +++ b/tests/test_selected.py @@ -19,7 +19,7 @@ def test_dict_contents(): for i,j in c.items(): #try: ok = dill.pickles(j) - #except: + #except Exception: # print ("FAIL: %s with %s" % (i, dill.detect.errors(j))) if verbose: print ("%s: %s, %s" % (ok, type(j), j)) assert ok @@ -29,7 +29,7 @@ def _g(x): yield x; def _f(): try: raise - except: + except Exception: from sys import exc_info e, er, tb = exc_info() return er, tb @@ -83,7 +83,7 @@ def test_frame_related(): g = _g(1) f = g.gi_frame e,t = _f() - _is = lambda ok: not ok if dill._dill.IS_PYPY2 else ok + _is = lambda ok: ok ok = dill.pickles(f) if verbose: print ("%s: %s, %s" % (ok, type(f), f)) assert not ok diff --git a/tests/test_session.py b/tests/test_session.py index 8d036fb5..689cc975 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -5,8 +5,12 @@ # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/dill/blob/master/LICENSE -from __future__ import print_function -import atexit, dill, os, sys, __main__ +import atexit +import os +import sys +import __main__ + +import dill session_file = os.path.join(os.path.dirname(__file__), 'session-byref-%s.pkl') @@ -114,9 +118,8 @@ def test_objects(main, copy_dict, byref): for obj in ('x', 'empty', 'names'): assert main_dict[obj] == copy_dict[obj] - globs = '__globals__' if dill._dill.PY3 else 'func_globals' for obj in ['squared', 'cubed']: - assert getattr(main_dict[obj], globs) is main_dict + assert main_dict[obj].__globals__ is main_dict assert main_dict[obj](3) == copy_dict[obj](3) assert main.Person.__module__ == main.__name__ diff --git a/tests/test_source.py b/tests/test_source.py index 57d25d12..01fc1eb3 100644 --- a/tests/test_source.py +++ b/tests/test_source.py @@ -11,9 +11,7 @@ from dill._dill import IS_PYPY import sys -PY3 = sys.version_info[0] >= 3 -IS_PYPY3 = IS_PYPY and PY3 -PY310b = '0x30a00b1' +PY310b = 0x30a00b1 f = lambda x: x**2 def g(x): return f(x) - x @@ -61,14 +59,12 @@ def test_itself(): # builtin functions and objects def test_builtin(): - if PY3: builtin = 'builtins' - else: builtin = '__builtin__' assert likely_import(pow) == 'pow\n' assert likely_import(100) == '100\n' assert likely_import(True) == 'True\n' - assert likely_import(pow, explicit=True) == 'from %s import pow\n' % builtin + assert likely_import(pow, explicit=True) == 'from builtins import pow\n' assert likely_import(100, explicit=True) == '100\n' - assert likely_import(True, explicit=True) == 'True\n' if PY3 else 'from %s import True\n' % builtin + assert likely_import(True, explicit=True) == 'True\n' # this is kinda BS... you can't import a None assert likely_import(None) == 'None\n' assert likely_import(None, explicit=True) == 'None\n' @@ -87,14 +83,9 @@ def test_dynamic(): # classes and class instances def test_classes(): - try: #XXX: should this be a 'special case'? - from StringIO import StringIO - y = "from StringIO import StringIO\n" - x = y - except ImportError: - from io import BytesIO as StringIO - y = "from _io import BytesIO\n" - x = y if (IS_PYPY3 or hex(sys.hexversion) >= PY310b) else "from io import BytesIO\n" + from io import BytesIO as StringIO + y = "from _io import BytesIO\n" + x = y if (IS_PYPY or sys.hexversion >= PY310b) else "from io import BytesIO\n" s = StringIO() assert likely_import(StringIO) == x diff --git a/tests/test_weakref.py b/tests/test_weakref.py index ada7d140..0e99f3ea 100644 --- a/tests/test_weakref.py +++ b/tests/test_weakref.py @@ -76,7 +76,7 @@ def test_dictproxy(): from dill._dill import DictProxyType try: m = DictProxyType({"foo": "bar"}) - except: + except Exception: m = type.__dict__ mp = dill.copy(m) assert mp.items() == m.items()