From fc46578add4df42891e0b3cf94f7dd3ead7a8214 Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Sat, 4 Jun 2022 11:09:06 -0300 Subject: [PATCH 01/13] Kickstart support drop for Python < 3.7 Step 1 of code clean-up: mechanically remove if-else branches that would never run in Python/PyPy >= 3.7. There's still some unused or redundant code, left for the next step to facilitate review. Note: all the remaining `sys.hexversion` tests were standardized to use numerical comparison and hexadecimal integer literals (e.g. `0x30b00a7` for version 3.11.0a7) as most of them already were. --- dill/_dill.py | 654 +++++++++++----------------------------- dill/_objects.py | 108 ++----- dill/_shims.py | 106 ------- dill/detect.py | 69 ++--- dill/source.py | 21 +- dill/temp.py | 23 +- tests/test_check.py | 1 - tests/test_classdef.py | 77 ++--- tests/test_detect.py | 6 +- tests/test_file.py | 2 - tests/test_functions.py | 37 +-- tests/test_recursive.py | 6 +- tests/test_selected.py | 2 +- tests/test_session.py | 3 +- tests/test_source.py | 12 +- 15 files changed, 292 insertions(+), 835 deletions(-) diff --git a/dill/_dill.py b/dill/_dill.py index ee1d985f..5bd770b3 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -35,46 +35,25 @@ def _trace(boolean): import sys diff = None _use_diff = False -PY3 = (sys.hexversion >= 0x3000000) -# OLDER: 3.0 <= x < 3.4 *OR* x < 2.7.10 #NOTE: guessing relevant versions -OLDER = (PY3 and sys.hexversion < 0x3040000) or (sys.hexversion < 0x2070ab1) -OLD33 = (sys.hexversion < 0x3030000) -OLD37 = (sys.hexversion < 0x3070000) OLD39 = (sys.hexversion < 0x3090000) OLD310 = (sys.hexversion < 0x30a0000) -PY34 = (0x3040000 <= sys.hexversion < 0x3050000) -if PY3: #XXX: get types from .objtypes ? - import builtins as __builtin__ - from pickle import _Pickler as StockPickler, Unpickler as StockUnpickler - from _thread import LockType - if (sys.hexversion >= 0x30200f0): - from _thread import RLock as RLockType - else: - from threading import _RLock as RLockType - #from io import IOBase - from types import CodeType, FunctionType, MethodType, GeneratorType, \ - TracebackType, FrameType, ModuleType, BuiltinMethodType - BufferType = memoryview #XXX: unregistered - ClassType = type # no 'old-style' classes - EllipsisType = type(Ellipsis) - #FileType = IOBase - NotImplementedType = type(NotImplemented) - SliceType = slice - TypeType = type # 'new-style' classes #XXX: unregistered - XRangeType = range - if OLD33: - DictProxyType = type(object.__dict__) - else: - from types import MappingProxyType as DictProxyType -else: - import __builtin__ - from pickle import Pickler as StockPickler, Unpickler as StockUnpickler - from thread import LockType - from threading import _RLock as RLockType - from types import CodeType, FunctionType, ClassType, MethodType, \ - GeneratorType, DictProxyType, XRangeType, SliceType, TracebackType, \ - NotImplementedType, EllipsisType, FrameType, ModuleType, \ - BufferType, BuiltinMethodType, TypeType +#XXX: get types from .objtypes ? +import builtins as __builtin__ +from pickle import _Pickler as StockPickler, Unpickler as StockUnpickler +from _thread import LockType +from _thread import RLock as RLockType +#from io import IOBase +from types import CodeType, FunctionType, MethodType, GeneratorType, \ + TracebackType, FrameType, ModuleType, BuiltinMethodType +BufferType = memoryview #XXX: unregistered +ClassType = type # no 'old-style' classes +EllipsisType = type(Ellipsis) +#FileType = IOBase +NotImplementedType = type(NotImplemented) +SliceType = slice +TypeType = type # 'new-style' classes #XXX: unregistered +XRangeType = range +from types import MappingProxyType as DictProxyType from pickle import HIGHEST_PROTOCOL, PickleError, PicklingError, UnpicklingError try: from pickle import DEFAULT_PROTOCOL @@ -87,20 +66,9 @@ def _trace(boolean): from weakref import ReferenceType, ProxyType, CallableProxyType from functools import partial from operator import itemgetter, attrgetter -# new in python3.3 -if sys.hexversion < 0x03030000: - FileNotFoundError = IOError -if PY3 and sys.hexversion < 0x03040000: - GENERATOR_FAIL = True -else: GENERATOR_FAIL = False -if PY3: - import importlib.machinery - EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES) -else: - import imp - EXTENSION_SUFFIXES = tuple(suffix - for (suffix, _, s_type) in imp.get_suffixes() - if s_type == imp.C_EXTENSION) +GENERATOR_FAIL = False +import importlib.machinery +EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES) try: import ctypes HAS_CTYPES = True @@ -109,28 +77,15 @@ def _trace(boolean): except ImportError: HAS_CTYPES = False IS_PYPY = False -IS_PYPY2 = IS_PYPY and not PY3 NumpyUfuncType = None NumpyDType = None NumpyArrayType = None try: - if OLDER: - raise AttributeError('find_spec not found') - import importlib if not importlib.machinery.PathFinder().find_spec('numpy'): raise ImportError("No module named 'numpy'") NumpyUfuncType = True NumpyDType = True NumpyArrayType = True -except AttributeError: - try: - import imp - imp.find_module('numpy') - NumpyUfuncType = True - NumpyDType = True - NumpyArrayType = True - except ImportError: - pass except ImportError: pass def __hook__(): @@ -191,21 +146,16 @@ def numpyufunc(obj): return False def numpydtype(obj): return False # make sure to add these 'hand-built' types to _typemap -if PY3: - CellType = type((lambda x: lambda y: x)(0).__closure__[0]) +CellType = type((lambda x: lambda y: x)(0).__closure__[0]) +from types import GetSetDescriptorType +if not IS_PYPY: + from types import MemberDescriptorType else: - CellType = type((lambda x: lambda y: x)(0).func_closure[0]) -# new in python2.5 -if sys.hexversion >= 0x20500f0: - from types import GetSetDescriptorType - if not IS_PYPY: - from types import MemberDescriptorType - else: - # oddly, MemberDescriptorType is GetSetDescriptorType - # while, member_descriptor does exist otherwise... is this a pypy bug? - class _member(object): - __slots__ = ['descriptor'] - MemberDescriptorType = type(_member.descriptor) + # oddly, MemberDescriptorType is GetSetDescriptorType + # while, member_descriptor does exist otherwise... is this a pypy bug? + class _member(object): + __slots__ = ['descriptor'] + MemberDescriptorType = type(_member.descriptor) if IS_PYPY: WrapperDescriptorType = MethodType MethodDescriptorType = FunctionType @@ -213,7 +163,7 @@ class _member(object): else: WrapperDescriptorType = type(type.__repr__) MethodDescriptorType = type(type.__dict__['mro']) - ClassMethodDescriptorType = type(type.__dict__['__prepare__' if PY3 else 'mro']) + ClassMethodDescriptorType = type(type.__dict__['__prepare__']) MethodWrapperType = type([].__repr__) PartialType = type(partial(int,base=2)) @@ -223,7 +173,7 @@ class _member(object): try: from functools import _lru_cache_wrapper as LRUCacheType -except: +except ImportError: LRUCacheType = None if not isinstance(LRUCacheType, type): @@ -249,20 +199,13 @@ def get_file_type(*args, **kwargs): PyBufferedWriterType = get_file_type('wb', buffering=-1, open=_open) except ImportError: PyTextWrapperType = PyBufferedRandomType = PyBufferedReaderType = PyBufferedWriterType = None -try: - from cStringIO import StringIO, InputType, OutputType +from io import BytesIO as StringIO +InputType = OutputType = None +from socket import socket as SocketType +try: #FIXME: additionally calls ForkingPickler.register several times + from multiprocessing.reduction import _reduce_socket as reduce_socket except ImportError: - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO - InputType = OutputType = None -if not IS_PYPY2: - from socket import socket as SocketType - try: #FIXME: additionally calls ForkingPickler.register several times - from multiprocessing.reduction import _reduce_socket as reduce_socket - except ImportError: - from multiprocessing.reduction import reduce_socket + from multiprocessing.reduction import reduce_socket try: __IPYTHON__ is True # is ipython ExitType = None # IPython.core.autocall.ExitAutocall @@ -405,8 +348,7 @@ def _module_map(): from collections import defaultdict, namedtuple modmap = namedtuple('Modmap', ['by_name', 'by_id', 'top_level']) modmap = modmap(defaultdict(list), defaultdict(list), {}) - items = 'items' if PY3 else 'iteritems' - for modname, module in getattr(sys.modules, items)(): + for modname, module in sys.modules.items(): if not isinstance(module, ModuleType): continue if '.' not in modname: @@ -435,8 +377,7 @@ def _stash_modules(main_module): imported_as = [] imported_top_level = [] # keep separeted for backwards compatibility original = {} - items = 'items' if PY3 else 'iteritems' - for name, obj in getattr(main_module.__dict__, items)(): + for name, obj in vars(main_module).items(): if obj is main_module: original[name] = newmod # self-reference continue @@ -701,16 +642,11 @@ def use_diff(on=True): def _create_typemap(): import types - if PY3: - d = dict(list(__builtin__.__dict__.items()) + \ - list(types.__dict__.items())).items() - builtin = 'builtins' - else: - d = types.__dict__.iteritems() - builtin = '__builtin__' + d = dict(list(__builtin__.__dict__.items()) + \ + list(types.__dict__.items())).items() for key, value in d: - if getattr(value, '__module__', None) == builtin \ - and type(value) is type: + if getattr(value, '__module__', None) == 'builtins' \ + and type(value) is type: yield key, value return _reverse_typemap = dict(_create_typemap()) @@ -742,10 +678,7 @@ def _create_typemap(): _reverse_typemap['ClassMethodDescriptorType'] = ClassMethodDescriptorType else: _reverse_typemap['MemberDescriptorType'] = MemberDescriptorType -if PY3: - _typemap = dict((v, k) for k, v in _reverse_typemap.items()) -else: - _typemap = dict((v, k) for k, v in _reverse_typemap.iteritems()) +_typemap = dict((v, k) for k, v in _reverse_typemap.items()) def _unmarshal(string): return marshal.loads(string) @@ -778,7 +711,7 @@ def _create_code(*args): args = args[1:] else: # from < 3.10 (or pre-LNOTAB storage) LNOTAB = b'' - if PY3 and hasattr(args[-3], 'encode'): #NOTE: from PY2 fails (optcode) + if hasattr(args[-3], 'encode'): #NOTE: from PY2 fails (optcode) args = list(args) if len(args) == 20: # from 3.11a # obj.co_argcount, obj.co_posonlyargcount, @@ -926,24 +859,6 @@ def _create_code(*args): return CodeType(args[0], *argz) elif len(args) == 15: return CodeType(*args) return CodeType(args[0], 0, *args[1:]) # from 2.7 - # obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, - # obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, - # obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, - # obj.co_freevars, obj.co_cellvars - if len(args) == 20: # from 3.11a - args = args[:1] + args[3:12] + args[13:14] + (LNOTAB,) + args[18:] - return CodeType(*args) - elif len(args) == 18: # from 3.11 - args = args[:1] + args[3:12] + args[13:14] + (LNOTAB,) + args[16:] - return CodeType(*args) - elif len(args) == 16: # from 3.10 or from 3.8 - if LNOTAB: # here and above uses stored LNOTAB - argz = args[3:-3] + (LNOTAB,) + args[-2:] - else: - argz = args[3:] - return CodeType(args[0], *argz) - elif len(args) == 15: return CodeType(args[0], *args[2:]) # from 3.7 - return CodeType(*args) def _create_ftype(ftypeobj, func, args, kwds): if kwds is None: @@ -983,9 +898,6 @@ def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdat import tempfile f = tempfile.TemporaryFile(mode) else: - # treat x mode as w mode - if "x" in mode and sys.hexversion < 0x03030000: - raise ValueError("invalid mode: '%s'" % mode) try: exists = os.path.exists(name) except: @@ -1027,28 +939,9 @@ def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdat flags |= os.O_WRONLY f = os.fdopen(os.open(name, flags), mode) # set name to the correct value - if PY3: - r = getattr(f, "buffer", f) - r = getattr(r, "raw", r) - r.name = name - else: - if not HAS_CTYPES: - raise ImportError("No module named 'ctypes'") - class FILE(ctypes.Structure): - _fields_ = [("refcount", ctypes.c_long), - ("type_obj", ctypes.py_object), - ("file_pointer", ctypes.c_voidp), - ("name", ctypes.py_object)] - - class PyObject(ctypes.Structure): - _fields_ = [ - ("ob_refcnt", ctypes.c_int), - ("ob_type", ctypes.py_object) - ] - #FIXME: CONTENTS_FMODE fails for pypy due to issue #1233 - # https://bitbucket.org/pypy/pypy/issues/1233 - ctypes.cast(id(f), ctypes.POINTER(FILE)).contents.name = name - ctypes.cast(id(name), ctypes.POINTER(PyObject)).contents.ob_refcnt += 1 + r = getattr(f, "buffer", f) + r = getattr(r, "raw", r) + r.name = name assert f.name == name else: f = open(name, mode) @@ -1120,26 +1013,15 @@ def __ror__(self, a): _CELL_REF = None _CELL_EMPTY = Sentinel('_CELL_EMPTY') -if PY3: - def _create_cell(contents=None): - if contents is not _CELL_EMPTY: - value = contents - return (lambda: value).__closure__[0] - -else: - def _create_cell(contents=None): - if contents is not _CELL_EMPTY: - value = contents - return (lambda: value).func_closure[0] - +def _create_cell(contents=None): + if contents is not _CELL_EMPTY: + value = contents + return (lambda: value).__closure__[0] def _create_weakref(obj, *args): from weakref import ref if obj is None: # it's dead - if PY3: - from collections import UserDict - else: - from UserDict import UserDict + from collections import UserDict return ref(UserDict(), *args) return ref(obj, *args) @@ -1147,10 +1029,7 @@ def _create_weakproxy(obj, callable=False, *args): from weakref import proxy if obj is None: # it's dead if callable: return proxy(lambda x:x, *args) - if PY3: - from collections import UserDict - else: - from UserDict import UserDict + from collections import UserDict return proxy(UserDict(), *args) return proxy(obj, *args) @@ -1171,23 +1050,13 @@ def _create_dtypemeta(scalar_type): return NumpyDType return type(NumpyDType(scalar_type)) -if OLD37: - def _create_namedtuple(name, fieldnames, modulename, defaults=None): - class_ = _import_module(modulename + '.' + name, safe=True) - if class_ is not None: - return class_ - import collections - t = collections.namedtuple(name, fieldnames) - t.__module__ = modulename - return t -else: - def _create_namedtuple(name, fieldnames, modulename, defaults=None): - class_ = _import_module(modulename + '.' + name, safe=True) - if class_ is not None: - return class_ - import collections - t = collections.namedtuple(name, fieldnames, defaults=defaults, module=modulename) - return t +def _create_namedtuple(name, fieldnames, modulename, defaults=None): + class_ = _import_module(modulename + '.' + name, safe=True) + if class_ is not None: + return class_ + import collections + t = collections.namedtuple(name, fieldnames, defaults=defaults, module=modulename) + return t def _getattr(objclass, name, repr_str): # hack to grab the reference directly @@ -1288,10 +1157,7 @@ def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO else: pickler.save_reduce(*reduction) # pop None created by calling preprocessing step off stack - if PY3: - pickler.write(bytes('0', 'UTF-8')) - else: - pickler.write('0') + pickler.write(bytes('0', 'UTF-8')) #@register(CodeType) #def save_code(pickler, obj): @@ -1307,62 +1173,54 @@ def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO @register(CodeType) def save_code(pickler, obj): log.info("Co: %s" % obj) - if PY3: - if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args) - args = ( - obj.co_lnotab, # for < python 3.10 [not counted in args] - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, - obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable, - obj.co_columntable, obj.co_exceptiontable, obj.co_freevars, - obj.co_cellvars - ) - elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args) - args = ( - obj.co_lnotab, # for < python 3.10 [not counted in args] - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, - obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, - obj.co_freevars, obj.co_cellvars - ) - elif hasattr(obj, "co_linetable"): # python 3.10 (16 args) - args = ( - obj.co_lnotab, # for < python 3.10 [not counted in args] - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_linetable, obj.co_freevars, - obj.co_cellvars - ) - elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args) - args = ( - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, - obj.co_cellvars - ) - else: # python 3.7 (15 args) - args = ( - obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, - obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, - obj.co_names, obj.co_varnames, obj.co_filename, - obj.co_name, obj.co_firstlineno, obj.co_lnotab, - obj.co_freevars, obj.co_cellvars - ) - else: # python 2.7 (14 args) + if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, + obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable, + obj.co_columntable, obj.co_exceptiontable, obj.co_freevars, + obj.co_cellvars + ) + elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args) args = ( - obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, - obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, - obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, + obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, obj.co_freevars, obj.co_cellvars - ) + ) + elif hasattr(obj, "co_linetable"): # python 3.10 (16 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_linetable, obj.co_freevars, + obj.co_cellvars + ) + elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args) + args = ( + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, + obj.co_cellvars + ) + else: # python 3.7 (15 args) + args = ( + obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, + obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, + obj.co_names, obj.co_varnames, obj.co_filename, + obj.co_name, obj.co_firstlineno, obj.co_lnotab, + obj.co_freevars, obj.co_cellvars + ) pickler.save_reduce(_create_code, args, obj=obj) log.info("# Co") @@ -1373,26 +1231,17 @@ def save_module_dict(pickler, obj): if is_dill(pickler, child=False) and obj == pickler._main.__dict__ and \ not (pickler._session and pickler._first_pass): log.info("D1: ')) - owner = int(r.split('owner=')[1].split()[0]) if PY3 else getattr(obj, '_RLock__owner') + owner = int(r.split('owner=')[1].split()[0]) pickler.save_reduce(_create_rlock, (count,owner,), obj=obj) log.info("# RL") return -if not IS_PYPY2: - #@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL - def save_socket(pickler, obj): - log.info("So: %s" % obj) - pickler.save_reduce(*reduce_socket(obj)) - log.info("# So") - return - -if sys.hexversion <= 0x3050000: - @register(ItemGetterType) - def save_itemgetter(pickler, obj): - log.info("Ig: %s" % obj) - helper = _itemgetter_helper() - obj(helper) - pickler.save_reduce(type(obj), tuple(helper.items), obj=obj) - log.info("# Ig") - return - - @register(AttrGetterType) - def save_attrgetter(pickler, obj): - log.info("Ag: %s" % obj) - attrs = [] - helper = _attrgetter_helper(attrs) - obj(helper) - pickler.save_reduce(type(obj), tuple(attrs), obj=obj) - log.info("# Ag") - return +#@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL +def save_socket(pickler, obj): + log.info("So: %s" % obj) + pickler.save_reduce(*reduce_socket(obj)) + log.info("# So") + return def _save_file(pickler, obj, open_): if obj.closed: @@ -1597,15 +1425,6 @@ def save_stringo(pickler, obj): log.info("# Io") return -if 0x2050000 <= sys.hexversion < 0x3010000: - @register(PartialType) - def save_functor(pickler, obj): - log.info("Fu: %s" % obj) - pickler.save_reduce(_create_ftype, (type(obj), obj.func, obj.args, - obj.keywords), obj=obj) - log.info("# Fu") - return - if LRUCacheType is not None: from functools import lru_cache @register(LRUCacheType) @@ -1632,86 +1451,34 @@ def save_super(pickler, obj): log.info("# Su") return -if OLDER or not PY3: - @register(BuiltinMethodType) - def save_builtin_method(pickler, obj): - if obj.__self__ is not None: - if obj.__self__ is __builtin__: - module = 'builtins' if PY3 else '__builtin__' - _t = "B1" - log.info("%s: %s" % (_t, obj)) - else: - module = obj.__self__ - _t = "B3" - log.info("%s: %s" % (_t, obj)) - if is_dill(pickler, child=True): - _recurse = pickler._recurse - pickler._recurse = False - pickler.save_reduce(_get_attr, (module, obj.__name__), obj=obj) - if is_dill(pickler, child=True): - pickler._recurse = _recurse - log.info("# %s" % _t) - else: - log.info("B2: %s" % obj) - name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) - StockPickler.save_global(pickler, obj, name=name) - log.info("# B2") - return - - @register(MethodType) #FIXME: fails for 'hidden' or 'name-mangled' classes - def save_instancemethod0(pickler, obj):# example: cStringIO.StringI - log.info("Me: %s" % obj) #XXX: obj.__dict__ handled elsewhere? - if PY3: - pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) - else: - pickler.save_reduce(MethodType, (obj.im_func, obj.im_self, - obj.im_class), obj=obj) - log.info("# Me") - return - -if sys.hexversion >= 0x20500f0: - if not IS_PYPY: - @register(MemberDescriptorType) - @register(GetSetDescriptorType) - @register(MethodDescriptorType) - @register(WrapperDescriptorType) - @register(ClassMethodDescriptorType) - def save_wrapper_descriptor(pickler, obj): - log.info("Wr: %s" % obj) - pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, - obj.__repr__()), obj=obj) - log.info("# Wr") - return - else: - @register(MemberDescriptorType) - @register(GetSetDescriptorType) - def save_wrapper_descriptor(pickler, obj): - log.info("Wr: %s" % obj) - pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, - obj.__repr__()), obj=obj) - log.info("# Wr") - return - - @register(MethodWrapperType) - def save_instancemethod(pickler, obj): - log.info("Mw: %s" % obj) - if IS_PYPY2 and obj.__self__ is None and obj.im_class: - # Can be a class method in PYPY2 if __self__ is none - pickler.save_reduce(getattr, (obj.im_class, obj.__name__), obj=obj) - return - pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) - log.info("# Mw") - return - -elif not IS_PYPY: +if not IS_PYPY: + @register(MemberDescriptorType) + @register(GetSetDescriptorType) @register(MethodDescriptorType) @register(WrapperDescriptorType) + @register(ClassMethodDescriptorType) def save_wrapper_descriptor(pickler, obj): log.info("Wr: %s" % obj) pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, obj.__repr__()), obj=obj) log.info("# Wr") return +else: + @register(MemberDescriptorType) + @register(GetSetDescriptorType) + def save_wrapper_descriptor(pickler, obj): + log.info("Wr: %s" % obj) + pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, + obj.__repr__()), obj=obj) + log.info("# Wr") + return + +@register(MethodWrapperType) +def save_instancemethod(pickler, obj): + log.info("Mw: %s" % obj) + pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) + log.info("# Mw") + return @register(CellType) def save_cell(pickler, obj): @@ -1733,10 +1500,7 @@ def save_cell(pickler, obj): # The result of this function call will be None pickler.save_reduce(_shims._delattr, (obj, 'cell_contents')) # pop None created by calling _delattr off stack - if PY3: - pickler.write(bytes('0', 'UTF-8')) - else: - pickler.write('0') + pickler.write(bytes('0', 'UTF-8')) log.info("# Ce3") return if is_dill(pickler, child=True): @@ -1770,29 +1534,12 @@ def save_dictproxy(pickler, obj): log.info("# Mp") return elif not IS_PYPY: - if not OLD33: - @register(DictProxyType) - def save_dictproxy(pickler, obj): - log.info("Mp: %s" % obj) - pickler.save_reduce(DictProxyType, (obj.copy(),), obj=obj) - log.info("# Mp") - return - else: - # The following function is based on 'saveDictProxy' from spickle - # Copyright (c) 2011 by science+computing ag - # License: http://www.apache.org/licenses/LICENSE-2.0 - @register(DictProxyType) - def save_dictproxy(pickler, obj): - log.info("Dp: %s" % obj) - attr = obj.get('__dict__') - #pickler.save_reduce(_create_dictproxy, (attr,'nested'), obj=obj) - if type(attr) == GetSetDescriptorType and attr.__name__ == "__dict__" \ - and getattr(attr.__objclass__, "__dict__", None) == obj: - pickler.save_reduce(getattr, (attr.__objclass__,"__dict__"),obj=obj) - log.info("# Dp") - return - # all bad below... so throw ReferenceError or TypeError - raise ReferenceError("%s does not reference a class __dict__" % obj) + @register(DictProxyType) + def save_dictproxy(pickler, obj): + log.info("Mp: %s" % obj) + pickler.save_reduce(DictProxyType, (obj.copy(),), obj=obj) + log.info("# Mp") + return @register(SliceType) def save_slice(pickler, obj): @@ -1837,10 +1584,7 @@ def _locate_object(address, module=None): for obj in special: if address == id(obj): return obj if module: - if PY3: - objects = iter(module.__dict__.values()) - else: - objects = module.__dict__.itervalues() + objects = iter(module.__dict__.values()) else: objects = iter(gc.get_objects()) for obj in objects: if address == id(obj): return obj @@ -1916,7 +1660,7 @@ def save_module(pickler, obj): pickler.save_reduce(_import_module, (obj.__name__,), obj=obj, state=_main_dict) log.info("# M1") - elif PY3 and obj.__name__ == "dill._dill": + elif obj.__name__ == "dill._dill": log.info("M2: %s" % obj) pickler.save_global(obj, name="_dill") log.info("# M2") @@ -1936,7 +1680,7 @@ def save_type(pickler, obj, postproc_list=None): elif obj.__bases__ == (tuple,) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]): # special case: namedtuples log.info("T6: %s" % obj) - if OLD37 or (not obj._field_defaults): + if not obj._field_defaults: pickler.save_reduce(_create_namedtuple, (obj.__name__, obj._fields, obj.__module__), obj=obj) else: defaults = [obj._field_defaults[field] for field in obj._fields if field in obj._field_defaults] @@ -1948,10 +1692,7 @@ def save_type(pickler, obj, postproc_list=None): elif obj is type(None): log.info("T7: %s" % obj) #XXX: pickler.save_reduce(type, (None,), obj=obj) - if PY3: - pickler.write(bytes('c__builtin__\nNoneType\n', 'UTF-8')) - else: - pickler.write('c__builtin__\nNoneType\n') + pickler.write(bytes('c__builtin__\nNoneType\n', 'UTF-8')) log.info("# T7") elif obj is NotImplementedType: log.info("T7: %s" % obj) @@ -1982,7 +1723,7 @@ def save_type(pickler, obj, postproc_list=None): #print ("%s\n%s" % (obj.__bases__, obj.__dict__)) for name in _dict.get("__slots__", []): del _dict[name] - if PY3 and obj_name != obj.__name__: + if obj_name != obj.__name__: if postproc_list is None: postproc_list = [] postproc_list.append((setattr, (obj, '__qualname__', obj_name))) @@ -2003,12 +1744,6 @@ def save_type(pickler, obj, postproc_list=None): log.info("# T4") return -# Error in PyPy 2.7 when adding ABC support -if IS_PYPY2: - @register(FrameType) - def save_frame(pickler, obj): - raise PicklingError('Cannot pickle a Python stack frame') - @register(property) def save_property(pickler, obj): log.info("Pr: %s" % obj) @@ -2020,7 +1755,7 @@ def save_property(pickler, obj): @register(classmethod) def save_classmethod(pickler, obj): log.info("Cm: %s" % obj) - im_func = '__func__' if PY3 else 'im_func' + im_func = '__func__' try: orig_func = getattr(obj, im_func) except AttributeError: # Python 2.6 @@ -2028,16 +1763,13 @@ def save_classmethod(pickler, obj): if isinstance(obj, classmethod): orig_func = getattr(orig_func, im_func) # Unbind - # if PY3: - # if type(obj.__dict__) is dict: - # if obj.__dict__: - # state = obj.__dict__ - # else: - # state = None + # if type(obj.__dict__) is dict: + # if obj.__dict__: + # state = obj.__dict__ # else: - # state = (None, {'__dict__', obj.__dict__}) + # state = None # else: - # state = None + # state = (None, {'__dict__', obj.__dict__}) pickler.save_reduce(type(obj), (orig_func,), obj=obj) log.info("# Cm") @@ -2063,7 +1795,7 @@ def save_function(pickler, obj): # is created to correctly handle recursion. globs = {'__name__': obj.__module__} else: - globs_copy = obj.__globals__ if PY3 else obj.func_globals + globs_copy = obj.__globals__ # If the globals is the __dict__ from the module being saved as a # session, substitute it by the dictionary being actually saved. @@ -2081,10 +1813,7 @@ def save_function(pickler, obj): # In the case that the globals are copied, we need to ensure that # the globals dictionary is updated when all objects in the # dictionary are already created. - if PY3: - glob_ids = {id(g) for g in globs_copy.values()} - else: - glob_ids = {id(g) for g in globs_copy.itervalues()} + glob_ids = {id(g) for g in globs_copy.values()} for stack_element in _postproc: if stack_element in glob_ids: _postproc[stack_element].append((_setitems, (globs, globs_copy))) @@ -2092,42 +1821,28 @@ def save_function(pickler, obj): else: postproc_list.append((_setitems, (globs, globs_copy))) - if PY3: - closure = obj.__closure__ - state_dict = {} - for fattrname in ('__doc__', '__kwdefaults__', '__annotations__'): - fattr = getattr(obj, fattrname, None) - if fattr is not None: - state_dict[fattrname] = fattr - if obj.__qualname__ != obj.__name__: - state_dict['__qualname__'] = obj.__qualname__ - if '__name__' not in globs or obj.__module__ != globs['__name__']: - state_dict['__module__'] = obj.__module__ - - state = obj.__dict__ - if type(state) is not dict: - state_dict['__dict__'] = state - state = None - if state_dict: - state = state, state_dict - - _save_with_postproc(pickler, (_create_function, ( - obj.__code__, globs, obj.__name__, obj.__defaults__, - closure - ), state), obj=obj, postproc_list=postproc_list) - else: - closure = obj.func_closure - if obj.__doc__ is not None: - postproc_list.append((setattr, (obj, '__doc__', obj.__doc__))) - if '__name__' not in globs or obj.__module__ != globs['__name__']: - postproc_list.append((setattr, (obj, '__module__', obj.__module__))) - if obj.__dict__: - postproc_list.append((setattr, (obj, '__dict__', obj.__dict__))) - - _save_with_postproc(pickler, (_create_function, ( - obj.func_code, globs, obj.func_name, obj.func_defaults, + closure = obj.__closure__ + state_dict = {} + for fattrname in ('__doc__', '__kwdefaults__', '__annotations__'): + fattr = getattr(obj, fattrname, None) + if fattr is not None: + state_dict[fattrname] = fattr + if obj.__qualname__ != obj.__name__: + state_dict['__qualname__'] = obj.__qualname__ + if '__name__' not in globs or obj.__module__ != globs['__name__']: + state_dict['__module__'] = obj.__module__ + + state = obj.__dict__ + if type(state) is not dict: + state_dict['__dict__'] = state + state = None + if state_dict: + state = state, state_dict + + _save_with_postproc(pickler, (_create_function, ( + obj.__code__, globs, obj.__name__, obj.__defaults__, closure - )), obj=obj, postproc_list=postproc_list) + ), state), obj=obj, postproc_list=postproc_list) # Lift closure cell update to earliest function (#458) if _postproc: @@ -2143,10 +1858,7 @@ def save_function(pickler, obj): # Change the value of the cell pickler.save_reduce(*possible_postproc) # pop None created by calling preprocessing step off stack - if PY3: - pickler.write(bytes('0', 'UTF-8')) - else: - pickler.write('0') + pickler.write(bytes('0', 'UTF-8')) log.info("# F1") else: @@ -2238,7 +1950,7 @@ def check(obj, *args, **kwds): # use to protect against missing attributes def is_dill(pickler, child=None): "check the dill-ness of your pickler" - if (child is False) or PY34 or (not hasattr(pickler.__class__, 'mro')): + if child is False or not hasattr(pickler.__class__, 'mro'): return 'dill' in pickler.__module__ return Pickler in pickler.__class__.mro() diff --git a/dill/_objects.py b/dill/_objects.py index 8b1cb65c..bd8a2b3b 100644 --- a/dill/_objects.py +++ b/dill/_objects.py @@ -15,22 +15,9 @@ # helper imports import warnings; warnings.filterwarnings("ignore", category=DeprecationWarning) import sys -PY3 = (hex(sys.hexversion) >= '0x30000f0') -if PY3: - import queue as Queue - import dbm as anydbm -else: - import Queue - import anydbm - import sets # deprecated/removed - import mutex # removed -try: - from cStringIO import StringIO # has StringI and StringO types -except ImportError: # only has StringIO type - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO +import queue as Queue +import dbm as anydbm +from io import BytesIO as StringIO import re import array import collections @@ -64,8 +51,7 @@ try: import bz2 import sqlite3 - if PY3: import dbm.ndbm as dbm - else: import dbm + import dbm.ndbm as dbm HAS_ALL = True except ImportError: # Ubuntu HAS_ALL = False @@ -157,12 +143,8 @@ class _Struct(ctypes.Structure): a['StringType'] = _str = str(1) a['TupleType'] = _tuple = () a['TypeType'] = type -if PY3: - a['LongType'] = _int - a['UnicodeType'] = _str -else: - a['LongType'] = long(1) - a['UnicodeType'] = unicode(1) +a['LongType'] = _int +a['UnicodeType'] = _str # built-in constants (CH 4) a['CopyrightType'] = copyright # built-in types (CH 5) @@ -181,10 +163,6 @@ class _Struct(ctypes.Structure): a['TZInfoType'] = datetime.tzinfo() a['DateTimeType'] = datetime.datetime.today() a['CalendarType'] = calendar.Calendar() -if not PY3: - a['SetsType'] = sets.Set() - a['ImmutableSetType'] = sets.ImmutableSet() - a['MutexType'] = mutex.mutex() # numeric and mathematical types (CH 9) a['DecimalType'] = decimal.Decimal(1) a['CountType'] = itertools.count(0) @@ -288,32 +266,22 @@ class _Struct(ctypes.Structure): except ImportError: pass # other (concrete) object types -if PY3: - d['CellType'] = (_lambda)(0).__closure__[0] - a['XRangeType'] = _xrange = range(1) -else: - d['CellType'] = (_lambda)(0).func_closure[0] - a['XRangeType'] = _xrange = xrange(1) +d['CellType'] = (_lambda)(0).__closure__[0] +a['XRangeType'] = _xrange = range(1) if not IS_PYPY: d['MethodDescriptorType'] = type.__dict__['mro'] d['WrapperDescriptorType'] = type.__repr__ a['WrapperDescriptorType2'] = type.__dict__['__module__'] - d['ClassMethodDescriptorType'] = type.__dict__['__prepare__' if PY3 else 'mro'] + d['ClassMethodDescriptorType'] = type.__dict__['__prepare__'] # built-in functions (CH 2) -if PY3 or IS_PYPY: - _methodwrap = (1).__lt__ -else: - _methodwrap = (1).__cmp__ +_methodwrap = (1).__lt__ d['MethodWrapperType'] = _methodwrap a['StaticMethodType'] = staticmethod(_method) a['ClassMethodType'] = classmethod(_method) a['PropertyType'] = property() d['SuperType'] = super(Exception, _exception) # string services (CH 7) -if PY3: - _in = _bytes -else: - _in = _str +_in = _bytes a['InputType'] = _cstrI = StringIO(_in) a['OutputType'] = _cstrO = StringIO() # data types (CH 8) @@ -328,16 +296,12 @@ class _Struct(ctypes.Structure): a['QueueType'] = Queue.Queue() # numeric and mathematical types (CH 9) d['PartialType'] = functools.partial(int,base=2) -if PY3: - a['IzipType'] = zip('0','1') -else: - a['IzipType'] = itertools.izip('0','1') +a['IzipType'] = zip('0','1') a['ChainType'] = itertools.chain('0','1') d['ItemGetterType'] = operator.itemgetter(0) d['AttrGetterType'] = operator.attrgetter('__repr__') # file and directory access (CH 10) -if PY3: _fileW = _cstrO -else: _fileW = _tmpf +_fileW = _cstrO # data persistence (CH 11) if HAS_ALL: a['ConnectionType'] = _conn = sqlite3.connect(':memory:') @@ -345,8 +309,7 @@ class _Struct(ctypes.Structure): a['ShelveType'] = shelve.Shelf({}) # data compression and archiving (CH 12) if HAS_ALL: - if (hex(sys.hexversion) < '0x2070ef0') or PY3: - a['BZ2FileType'] = bz2.BZ2File(os.devnull) #FIXME: fail >= 3.3, 2.7.14 + a['BZ2FileType'] = bz2.BZ2File(os.devnull) #FIXME: fail >= 3.3, 2.7.14 a['BZ2CompressorType'] = bz2.BZ2Compressor() a['BZ2DecompressorType'] = bz2.BZ2Decompressor() #a['ZipFileType'] = _zip = zipfile.ZipFile(os.devnull,'w') #FIXME: fail >= 3.2 @@ -363,17 +326,10 @@ class _Struct(ctypes.Structure): a['NamedLoggerType'] = _logger = logging.getLogger(__name__) #FIXME: fail >= 3.2 and <= 2.6 #a['FrozenModuleType'] = __hello__ #FIXME: prints "Hello world..." # interprocess communication (CH 17) -if PY3: - a['SocketType'] = _socket = socket.socket() #FIXME: fail >= 3.3 - a['SocketPairType'] = socket.socketpair()[0] #FIXME: fail >= 3.3 -else: - a['SocketType'] = _socket = socket.socket() - a['SocketPairType'] = _socket._sock +a['SocketType'] = _socket = socket.socket() #FIXME: fail >= 3.3 +a['SocketPairType'] = socket.socketpair()[0] #FIXME: fail >= 3.3 # python runtime services (CH 27) -if PY3: - a['GeneratorContextManagerType'] = contextlib.contextmanager(max)([1]) -else: - a['GeneratorContextManagerType'] = contextlib.GeneratorContextManager(max) +a['GeneratorContextManagerType'] = contextlib.contextmanager(max)([1]) try: # ipython __IPYTHON__ is True # is ipython @@ -454,14 +410,9 @@ class _Struct(ctypes.Structure): # built-in functions (CH 2) x['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty # built-in types (CH 5) -if PY3: - x['DictionaryItemIteratorType'] = iter(type.__dict__.items()) - x['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) - x['DictionaryValueIteratorType'] = iter(type.__dict__.values()) -else: - x['DictionaryItemIteratorType'] = type.__dict__.iteritems() - x['DictionaryKeyIteratorType'] = type.__dict__.iterkeys() - x['DictionaryValueIteratorType'] = type.__dict__.itervalues() +x['DictionaryItemIteratorType'] = iter(type.__dict__.items()) +x['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) +x['DictionaryValueIteratorType'] = iter(type.__dict__.values()) # string services (CH 7) x['StructType'] = struct.Struct('c') x['CallableIteratorType'] = _srepattern.finditer('') @@ -484,7 +435,7 @@ class _Struct(ctypes.Structure): x['CSVDictWriterType'] = csv.DictWriter(_cstrO,{}) # cryptographic services (CH 14) x['HashType'] = hashlib.md5() -if (hex(sys.hexversion) < '0x30800a1'): +if (sys.hexversion < 0x30800a1): x['HMACType'] = hmac.new(_in) else: x['HMACType'] = hmac.new(_in, digestmod='md5') @@ -524,14 +475,9 @@ class _Struct(ctypes.Structure): # built-in types (CH 5) x['MemoryType'] = memoryview(_in) # 2.7 x['MemoryType2'] = memoryview(bytearray(_in)) # 2.7 - if PY3: - x['DictItemsType'] = _dict.items() # 2.7 - x['DictKeysType'] = _dict.keys() # 2.7 - x['DictValuesType'] = _dict.values() # 2.7 - else: - x['DictItemsType'] = _dict.viewitems() # 2.7 - x['DictKeysType'] = _dict.viewkeys() # 2.7 - x['DictValuesType'] = _dict.viewvalues() # 2.7 + x['DictItemsType'] = _dict.items() # 2.7 + x['DictKeysType'] = _dict.keys() # 2.7 + x['DictValuesType'] = _dict.values() # 2.7 # generic operating system services (CH 15) x['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG') x['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG') @@ -543,10 +489,8 @@ class _Struct(ctypes.Structure): x['CmpKeyObjType'] = _cmpkey('0') #2.7, >=3.2 except AttributeError: pass -if PY3: # oddities: removed, etc - x['BufferType'] = x['MemoryType'] -else: - x['BufferType'] = buffer('') +# oddities: removed, etc +x['BufferType'] = x['MemoryType'] # -- cleanup ---------------------------------------------------------------- a.update(d) # registered also succeed diff --git a/dill/_shims.py b/dill/_shims.py index 6bda5136..dfb336f2 100644 --- a/dill/_shims.py +++ b/dill/_shims.py @@ -156,111 +156,5 @@ def decorator(func): _CELL_EMPTY = Getattr(_dill, '_CELL_EMPTY', None) -if _dill.OLD37: - if _dill.HAS_CTYPES and hasattr(_dill.ctypes, 'pythonapi') and hasattr(_dill.ctypes.pythonapi, 'PyCell_Set'): - # CPython - ctypes = _dill.ctypes - - _PyCell_Set = ctypes.pythonapi.PyCell_Set - - @move_to(_dill) - def _setattr(object, name, value): - if type(object) is _dill.CellType and name == 'cell_contents': - _PyCell_Set.argtypes = (ctypes.py_object, ctypes.py_object) - _PyCell_Set(object, value) - else: - setattr(object, name, value) - - @move_to(_dill) - def _delattr(object, name): - if type(object) is _dill.CellType and name == 'cell_contents': - _PyCell_Set.argtypes = (ctypes.py_object, ctypes.c_void_p) - _PyCell_Set(object, None) - else: - delattr(object, name) - - # General Python (not CPython) up to 3.6 is in a weird case, where it is - # possible to pickle recursive cells, but we can't assign directly to the - # cell. - elif _dill.PY3: - # Use nonlocal variables to reassign the cell value. - # https://stackoverflow.com/a/59276835 - __nonlocal = ('nonlocal cell',) - exec('''def _setattr(cell, name, value): - if type(cell) is _dill.CellType and name == 'cell_contents': - def cell_setter(value): - %s - cell = value # pylint: disable=unused-variable - func = _dill.FunctionType(cell_setter.__code__, globals(), "", None, (cell,)) # same as cell_setter, but with cell being the cell's contents - func(value) - else: - setattr(cell, name, value)''' % __nonlocal) - move_to(_dill)(_setattr) - - exec('''def _delattr(cell, name): - if type(cell) is _dill.CellType and name == 'cell_contents': - try: - cell.cell_contents - except: - return - def cell_deleter(): - %s - del cell # pylint: disable=unused-variable - func = _dill.FunctionType(cell_deleter.__code__, globals(), "", None, (cell,)) # same as cell_deleter, but with cell being the cell's contents - func() - else: - delattr(cell, name)''' % __nonlocal) - move_to(_dill)(_delattr) - - else: - # Likely PyPy 2.7. Simulate the nonlocal keyword with bytecode - # manipulation. - - # The following function is based on 'cell_set' from 'cloudpickle' - # https://github.com/cloudpipe/cloudpickle/blob/5d89947288a18029672596a4d719093cc6d5a412/cloudpickle/cloudpickle.py#L393-L482 - # Copyright (c) 2012, Regents of the University of California. - # Copyright (c) 2009 `PiCloud, Inc. `_. - # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE - @move_to(_dill) - def _setattr(cell, name, value): - if type(cell) is _dill.CellType and name == 'cell_contents': - _cell_set = _dill.FunctionType( - _cell_set_template_code, {}, '_cell_set', (), (cell,),) - _cell_set(value) - else: - setattr(cell, name, value) - - def _cell_set_factory(value): - lambda: cell - cell = value - - co = _cell_set_factory.__code__ - - _cell_set_template_code = _dill.CodeType( - co.co_argcount, - co.co_nlocals, - co.co_stacksize, - co.co_flags, - co.co_code, - co.co_consts, - co.co_names, - co.co_varnames, - co.co_filename, - co.co_name, - co.co_firstlineno, - co.co_lnotab, - co.co_cellvars, # co_freevars is initialized with co_cellvars - (), # co_cellvars is made empty - ) - - del co - - @move_to(_dill) - def _delattr(cell, name): - if type(cell) is _dill.CellType and name == 'cell_contents': - pass - else: - delattr(cell, name) - _setattr = Getattr(_dill, '_setattr', setattr) _delattr = Getattr(_dill, '_delattr', delattr) diff --git a/dill/detect.py b/dill/detect.py index 41575205..2297c6cd 100644 --- a/dill/detect.py +++ b/dill/detect.py @@ -14,7 +14,6 @@ from .pointers import parent, reference, at, parents, children from ._dill import _trace as trace -from ._dill import PY3 __all__ = ['baditems','badobjects','badtypes','code','errors','freevars', 'getmodule','globalvars','nestedcode','nestedglobals','outermost', @@ -25,9 +24,7 @@ def getmodule(object, _filename=None, force=False): from inspect import getmodule as getmod module = getmod(object, _filename) if module or not force: return module - if PY3: builtins = 'builtins' - else: builtins = '__builtin__' - builtins = __import__(builtins) + import builtins from .source import getname name = getname(object, force=True) return builtins if name in vars(builtins).keys() else None @@ -37,22 +34,13 @@ def outermost(func): # is analogous to getsource(func,enclosing=True) NOTE: this is the object-equivalent of getsource(func, enclosing=True) """ - if PY3: - if ismethod(func): - _globals = func.__func__.__globals__ or {} - elif isfunction(func): - _globals = func.__globals__ or {} - else: - return #XXX: or raise? no matches - _globals = _globals.items() + if ismethod(func): + _globals = func.__func__.__globals__ or {} + elif isfunction(func): + _globals = func.__globals__ or {} else: - if ismethod(func): - _globals = func.im_func.func_globals or {} - elif isfunction(func): - _globals = func.func_globals or {} - else: - return #XXX: or raise? no matches - _globals = _globals.iteritems() + return #XXX: or raise? no matches + _globals = _globals.items() # get the enclosing source from .source import getsourcelines try: lines,lnum = getsourcelines(func, enclosing=True) @@ -87,12 +75,8 @@ def code(func): NOTE: use dill.source.getsource(CODEOBJ) to get the source code ''' - if PY3: - im_func = '__func__' - func_code = '__code__' - else: - im_func = 'im_func' - func_code = 'func_code' + im_func = '__func__' + func_code = '__code__' if ismethod(func): func = getattr(func, im_func) if isfunction(func): func = getattr(func, func_code) if istraceback(func): func = func.tb_frame @@ -109,12 +93,8 @@ def referrednested(func, recurse=True): #XXX: return dict of {__name__: obj} ? If possible, python builds code objects, but delays building functions until func() is called. """ - if PY3: - att1 = '__code__' - att0 = '__func__' - else: - att1 = 'func_code' # functions - att0 = 'im_func' # methods + att1 = '__code__' # functions + att0 = '__func__' # methods import gc funcs = set() @@ -143,14 +123,9 @@ def freevars(func): """get objects defined in enclosing code that are referred to by func returns a dict of {name:object}""" - if PY3: - im_func = '__func__' - func_code = '__code__' - func_closure = '__closure__' - else: - im_func = 'im_func' - func_code = 'func_code' - func_closure = 'func_closure' + im_func = '__func__' + func_code = '__code__' + func_closure = '__closure__' if ismethod(func): func = getattr(func, im_func) if isfunction(func): closures = getattr(func, func_closure) or () @@ -175,7 +150,7 @@ def nestedglobals(func, recurse=True): if func is None: return list() import sys from .temp import capture - CAN_NULL = sys.hexversion >= 51052711 #NULL may be prepended >= 3.11a7 + CAN_NULL = sys.hexversion >= 0x30b00a7 # NULL may be prepended >= 3.11a7 names = set() with capture('stdout') as out: dis.dis(func) #XXX: dis.dis(None) disassembles last traceback @@ -199,16 +174,10 @@ def globalvars(func, recurse=True, builtin=False): """get objects defined in global scope that are referred to by func return a dict of {name:object}""" - if PY3: - im_func = '__func__' - func_code = '__code__' - func_globals = '__globals__' - func_closure = '__closure__' - else: - im_func = 'im_func' - func_code = 'func_code' - func_globals = 'func_globals' - func_closure = 'func_closure' + im_func = '__func__' + func_code = '__code__' + func_globals = '__globals__' + func_closure = '__closure__' if ismethod(func): func = getattr(func, im_func) if isfunction(func): globs = vars(getmodule(sum)).copy() if builtin else {} diff --git a/dill/source.py b/dill/source.py index 47064a16..1d8bba11 100644 --- a/dill/source.py +++ b/dill/source.py @@ -29,8 +29,6 @@ ismodule, istraceback) from tokenize import TokenError -from ._dill import PY3 - def isfrommain(obj): "check if object was built in __main__" @@ -165,16 +163,14 @@ def findsource(object): name = object.__name__ if name == '': pat1 = r'(.*(?': pat1 = r'(.*(?= '0x20600f0': - from collections import namedtuple - Z = namedtuple("Z", ['a','b']) - Zi = Z(0,1) - X = namedtuple("Y", ['a','b']) - X.__name__ = "X" - if hex(sys.hexversion) >= '0x30300f0': - X.__qualname__ = "X" #XXX: name must 'match' or fails to pickle - Xi = X(0,1) - Bad = namedtuple("FakeName", ['a','b']) - Badi = Bad(0,1) -else: - Z = Zi = X = Xi = Bad = Badi = None +from collections import namedtuple +Z = namedtuple("Z", ['a','b']) +Zi = Z(0,1) +X = namedtuple("Y", ['a','b']) +X.__name__ = "X" +X.__qualname__ = "X" #XXX: name must 'match' or fails to pickle +Xi = X(0,1) +Bad = namedtuple("FakeName", ['a','b']) +Badi = Bad(0,1) # test namedtuple def test_namedtuple(): @@ -123,8 +119,7 @@ class B(namedtuple("B", ["one", "two"])): assert dill.copy(a) assert dill.copy(A.B).__name__ == 'B' - if dill._dill.PY3: - assert dill.copy(A.B).__qualname__.endswith('..A.B') + assert dill.copy(A.B).__qualname__.endswith('..A.B') assert dill.copy(A.B).__doc__ == 'docstring' assert dill.copy(A.B).__module__ == 'testing' @@ -168,12 +163,12 @@ def __getnewargs__(self): return np.asarray(self), self.color a1 = TestArray(np.zeros(100), color='green') - if dill._dill.PY3 and not dill._dill.IS_PYPY: + if not dill._dill.IS_PYPY: assert dill.pickles(a1) assert a1.__dict__ == dill.copy(a1).__dict__ a2 = a1[0:9] - if dill._dill.PY3 and not dill._dill.IS_PYPY: + if not dill._dill.IS_PYPY: assert dill.pickles(a2) assert a2.__dict__ == dill.copy(a2).__dict__ @@ -182,7 +177,7 @@ class TestArray2(np.ndarray): a3 = TestArray2([1,2,3,4,5]) a3.color = 'green' - if dill._dill.PY3 and not dill._dill.IS_PYPY: + if not dill._dill.IS_PYPY: assert dill.pickles(a3) assert a3.__dict__ == dill.copy(a3).__dict__ @@ -217,37 +212,21 @@ def test_slots(): assert dill.copy(y).y == value def test_metaclass(): - if dill._dill.PY3: - class metaclass_with_new(type): - def __new__(mcls, name, bases, ns, **kwds): - cls = super().__new__(mcls, name, bases, ns, **kwds) - assert mcls is not None - assert cls.method(mcls) - return cls - def method(cls, mcls): - return isinstance(cls, mcls) - - l = locals() - exec("""class subclass_with_new(metaclass=metaclass_with_new): - def __new__(cls): - self = super().__new__(cls) - return self""", None, l) - subclass_with_new = l['subclass_with_new'] - else: - class metaclass_with_new(type): - def __new__(mcls, name, bases, ns, **kwds): - cls = super(mcls, metaclass_with_new).__new__(mcls, name, bases, ns, **kwds) - assert mcls is not None - assert cls.method(mcls) - return cls - def method(cls, mcls): - return isinstance(cls, mcls) - - class subclass_with_new: - __metaclass__ = metaclass_with_new - def __new__(cls): - self = super(subclass_with_new, cls).__new__(cls) - return self + class metaclass_with_new(type): + def __new__(mcls, name, bases, ns, **kwds): + cls = super().__new__(mcls, name, bases, ns, **kwds) + assert mcls is not None + assert cls.method(mcls) + return cls + def method(cls, mcls): + return isinstance(cls, mcls) + + l = locals() + exec("""class subclass_with_new(metaclass=metaclass_with_new): + def __new__(cls): + self = super().__new__(cls) + return self""", None, l) + subclass_with_new = l['subclass_with_new'] assert dill.copy(subclass_with_new()) diff --git a/tests/test_detect.py b/tests/test_detect.py index 06339acd..4dac7aaf 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -8,7 +8,7 @@ from dill.detect import baditems, badobjects, badtypes, errors, parent, at, globalvars from dill import settings -from dill._dill import IS_PYPY, IS_PYPY2 +from dill._dill import IS_PYPY from pickle import PicklingError import inspect @@ -21,7 +21,7 @@ def test_bad_things(): #assert baditems(globals()) == [f] #XXX assert badobjects(f) is f assert badtypes(f) == type(f) - assert type(errors(f)) is PicklingError if IS_PYPY2 else TypeError + assert type(errors(f)) is TypeError d = badtypes(f, 1) assert isinstance(d, dict) assert list(badobjects(f, 1).keys()) == list(d.keys()) @@ -30,7 +30,7 @@ def test_bad_things(): a = dict(s) if not os.environ.get('COVERAGE'): #XXX: travis-ci assert len(s) is len(a) # TypeError (and possibly PicklingError) - n = 1 if IS_PYPY2 else 2 + n = 2 assert len(a) is n if 'PicklingError' in a.keys() else n-1 def test_parent(): diff --git a/tests/test_file.py b/tests/test_file.py index 8118e3a8..b766552e 100644 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -19,8 +19,6 @@ fname = "_test_file.txt" rand_chars = list(string.ascii_letters) + ["\n"] * 40 # bias newline -if sys.hexversion < 0x03030000: - FileNotFoundError = IOError buffer_error = ValueError("invalid buffer size") dne_error = FileNotFoundError("[Errno 2] No such file or directory: '%s'" % fname) diff --git a/tests/test_functions.py b/tests/test_functions.py index ec9670e2..1bf05026 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -11,10 +11,6 @@ dill.settings['recurse'] = True -def is_py3(): - return hex(sys.hexversion) >= '0x30000f0' - - def function_a(a): return a @@ -34,18 +30,17 @@ def function_d(d, d1, d2=1): function_d.__module__ = 'a module' -if is_py3(): - exec(''' +exec(''' def function_e(e, *e1, e2=1, e3=2): return e + sum(e1) + e2 + e3''') - globalvar = 0 +globalvar = 0 - @functools.lru_cache(None) - def function_with_cache(x): - global globalvar - globalvar += x - return globalvar +@functools.lru_cache(None) +def function_with_cache(x): + global globalvar + globalvar += x + return globalvar def function_with_unassigned_variable(): @@ -72,14 +67,13 @@ def test_functions(): assert dill.loads(dumped_func_d)(1, 2, 3) == 6 assert dill.loads(dumped_func_d)(1, 2, d2=3) == 6 - if is_py3(): - function_with_cache(1) - globalvar = 0 - dumped_func_cache = dill.dumps(function_with_cache) - assert function_with_cache(2) == 3 - assert function_with_cache(1) == 1 - assert function_with_cache(3) == 6 - assert function_with_cache(2) == 3 + function_with_cache(1) + globalvar = 0 + dumped_func_cache = dill.dumps(function_with_cache) + assert function_with_cache(2) == 3 + assert function_with_cache(1) == 1 + assert function_with_cache(3) == 6 + assert function_with_cache(2) == 3 empty_cell = function_with_unassigned_variable() cell_copy = dill.loads(dill.dumps(empty_cell)) @@ -92,8 +86,7 @@ def test_functions(): else: raise AssertionError('cell_copy() did not read an empty cell') - if is_py3(): - exec(''' + exec(''' dumped_func_e = dill.dumps(function_e) assert dill.loads(dumped_func_e)(1, 2) == 6 assert dill.loads(dumped_func_e)(1, 2, 3) == 9 diff --git a/tests/test_recursive.py b/tests/test_recursive.py index ee71a688..a144f301 100644 --- a/tests/test_recursive.py +++ b/tests/test_recursive.py @@ -6,7 +6,6 @@ # - https://github.com/uqfoundation/dill/blob/master/LICENSE import dill -from dill._dill import PY3 from functools import partial import warnings @@ -113,9 +112,8 @@ def __init__(self): def test_circular_reference(): assert copy(obj4()) obj4_copy = dill.loads(dill.dumps(obj4())) - if PY3: - assert type(obj4_copy) is type(obj4_copy).__init__.__closure__[0].cell_contents - assert type(obj4_copy.b) is type(obj4_copy.b).__init__.__closure__[0].cell_contents + assert type(obj4_copy) is type(obj4_copy).__init__.__closure__[0].cell_contents + assert type(obj4_copy.b) is type(obj4_copy.b).__init__.__closure__[0].cell_contents def f(): diff --git a/tests/test_selected.py b/tests/test_selected.py index bd790835..53b9cda2 100644 --- a/tests/test_selected.py +++ b/tests/test_selected.py @@ -76,7 +76,7 @@ def test_frame_related(): g = _g(1) f = g.gi_frame e,t = _f() - _is = lambda ok: not ok if dill._dill.IS_PYPY2 else ok + _is = lambda ok: ok ok = dill.pickles(f) if verbose: print ("%s: %s, %s" % (ok, type(f), f)) assert not ok diff --git a/tests/test_session.py b/tests/test_session.py index fd71ea05..b2f53728 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -114,9 +114,8 @@ def test_objects(main, copy_dict, byref): for obj in ('x', 'empty', 'names'): assert main_dict[obj] == copy_dict[obj] - globs = '__globals__' if dill._dill.PY3 else 'func_globals' for obj in ['squared', 'cubed']: - assert getattr(main_dict[obj], globs) is main_dict + assert main_dict[obj].__globals__ is main_dict assert main_dict[obj](3) == copy_dict[obj](3) assert main.Person.__module__ == main.__name__ diff --git a/tests/test_source.py b/tests/test_source.py index 57d25d12..dfdb27df 100644 --- a/tests/test_source.py +++ b/tests/test_source.py @@ -11,9 +11,7 @@ from dill._dill import IS_PYPY import sys -PY3 = sys.version_info[0] >= 3 -IS_PYPY3 = IS_PYPY and PY3 -PY310b = '0x30a00b1' +PY310b = 0x30a00b1 f = lambda x: x**2 def g(x): return f(x) - x @@ -61,14 +59,12 @@ def test_itself(): # builtin functions and objects def test_builtin(): - if PY3: builtin = 'builtins' - else: builtin = '__builtin__' assert likely_import(pow) == 'pow\n' assert likely_import(100) == '100\n' assert likely_import(True) == 'True\n' - assert likely_import(pow, explicit=True) == 'from %s import pow\n' % builtin + assert likely_import(pow, explicit=True) == 'from builtins import pow\n' assert likely_import(100, explicit=True) == '100\n' - assert likely_import(True, explicit=True) == 'True\n' if PY3 else 'from %s import True\n' % builtin + assert likely_import(True, explicit=True) == 'True\n' # this is kinda BS... you can't import a None assert likely_import(None) == 'None\n' assert likely_import(None, explicit=True) == 'None\n' @@ -94,7 +90,7 @@ def test_classes(): except ImportError: from io import BytesIO as StringIO y = "from _io import BytesIO\n" - x = y if (IS_PYPY3 or hex(sys.hexversion) >= PY310b) else "from io import BytesIO\n" + x = y if (IS_PYPY or sys.hexversion >= PY310b) else "from io import BytesIO\n" s = StringIO() assert likely_import(StringIO) == x From 28635974e887ade946d23b60dda5d89610b1c3f1 Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Tue, 21 Jun 2022 11:00:53 -0300 Subject: [PATCH 02/13] convert single function defined via exec() due to Python2/3 differences --- dill/source.py | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/dill/source.py b/dill/source.py index 1d8bba11..792b53e6 100644 --- a/dill/source.py +++ b/dill/source.py @@ -506,39 +506,20 @@ def outdent(code, spaces=None, all=True): #XXX: not sure what the point of _wrap is... -#exec_ = lambda s, *a: eval(compile(s, '', 'exec'), *a) __globals__ = globals() __locals__ = locals() -wrap2 = ''' def _wrap(f): """ encapsulate a function and it's __import__ """ def func(*args, **kwds): try: # _ = eval(getsource(f, force=True)) #XXX: safer but less robust - exec getimportable(f, alias='_') in %s, %s + exec(getimportable(f, alias='_'), __globals__, __locals__) except: raise ImportError('cannot import name ' + f.__name__) return _(*args, **kwds) func.__name__ = f.__name__ func.__doc__ = f.__doc__ return func -''' % ('__globals__', '__locals__') -wrap3 = ''' -def _wrap(f): - """ encapsulate a function and it's __import__ """ - def func(*args, **kwds): - try: - # _ = eval(getsource(f, force=True)) #XXX: safer but less robust - exec(getimportable(f, alias='_'), %s, %s) - except: - raise ImportError('cannot import name ' + f.__name__) - return _(*args, **kwds) - func.__name__ = f.__name__ - func.__doc__ = f.__doc__ - return func -''' % ('__globals__', '__locals__') -exec(wrap3) -del wrap2, wrap3 def _enclose(object, alias=''): #FIXME: needs alias to hold returned object From 71fcc7be35d5b2adb76aca6f3f649b837d5148f6 Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Tue, 21 Jun 2022 11:07:29 -0300 Subject: [PATCH 03/13] drop 'from __future__ import print_function' statements --- tests/__main__.py | 1 - tests/test_session.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/__main__.py b/tests/__main__.py index 312e5dae..0ee6d9ae 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -5,7 +5,6 @@ # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/dill/blob/master/LICENSE -from __future__ import print_function import glob import os try: diff --git a/tests/test_session.py b/tests/test_session.py index ed78f7f6..5d458834 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -5,7 +5,6 @@ # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/dill/blob/master/LICENSE -from __future__ import print_function import atexit, dill, os, sys, __main__ session_file = os.path.join(os.path.dirname(__file__), 'session-byref-%s.pkl') From bc87125705a8c71ce86f2c4df68247f99a8371c8 Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Tue, 21 Jun 2022 13:00:20 -0300 Subject: [PATCH 04/13] remove conditional imports due to Python 2 and old Python 3 --- dill/__diff.py | 8 +- dill/__init__.py | 21 +---- dill/_dill.py | 12 +-- dill/_objects.py | 177 +++++++++++++++---------------------- dill/settings.py | 5 +- tests/__main__.py | 8 +- tests/test_diff.py | 21 ++--- tests/test_extendpickle.py | 5 +- tests/test_module.py | 3 +- tests/test_source.py | 11 +-- 10 files changed, 95 insertions(+), 176 deletions(-) diff --git a/dill/__diff.py b/dill/__diff.py index 3ff65763..1ef424c0 100644 --- a/dill/__diff.py +++ b/dill/__diff.py @@ -10,18 +10,12 @@ Module to show if an object has changed since it was memorised """ -import os -import sys -import types +import builtins, os, sys, types try: import numpy HAS_NUMPY = True except: HAS_NUMPY = False -try: - import builtins -except ImportError: - import __builtin__ as builtins # pypy doesn't use reference counting getrefcount = getattr(sys, 'getrefcount', lambda x:0) diff --git a/dill/__init__.py b/dill/__init__.py index ac93ff6a..b4abca21 100644 --- a/dill/__init__.py +++ b/dill/__init__.py @@ -296,23 +296,9 @@ # make sure "trace" is turned off detect.trace(False) -try: - from importlib import reload -except ImportError: - try: - from imp import reload - except ImportError: - pass - -# put the objects in order, if possible -try: - from collections import OrderedDict as odict -except ImportError: - try: - from ordereddict import OrderedDict as odict - except ImportError: - odict = dict -objects = odict() +from importlib import reload + +objects = {} # local import of dill._objects #from . import _objects #objects.update(_objects.succeeds) @@ -373,7 +359,6 @@ def extend(use_dill=True): return extend() -del odict def license(): diff --git a/dill/_dill.py b/dill/_dill.py index 776c2cce..b6d09af7 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -57,11 +57,7 @@ def _trace(boolean): TypeType = type # 'new-style' classes #XXX: unregistered XRangeType = range from types import MappingProxyType as DictProxyType -from pickle import HIGHEST_PROTOCOL, PickleError, PicklingError, UnpicklingError -try: - from pickle import DEFAULT_PROTOCOL -except ImportError: - DEFAULT_PROTOCOL = HIGHEST_PROTOCOL +from pickle import DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, PickleError, PicklingError, UnpicklingError import __main__ as _main_module import marshal import gc @@ -206,10 +202,8 @@ def get_file_type(*args, **kwargs): from io import BytesIO as StringIO InputType = OutputType = None from socket import socket as SocketType -try: #FIXME: additionally calls ForkingPickler.register several times - from multiprocessing.reduction import _reduce_socket as reduce_socket -except ImportError: - from multiprocessing.reduction import reduce_socket +#FIXME: additionally calls ForkingPickler.register several times +from multiprocessing.reduction import _reduce_socket as reduce_socket try: __IPYTHON__ is True # is ipython ExitType = None # IPython.core.autocall.ExitAutocall diff --git a/dill/_objects.py b/dill/_objects.py index abdcb14c..da5d15ef 100644 --- a/dill/_objects.py +++ b/dill/_objects.py @@ -43,6 +43,7 @@ import hmac import os import logging +import logging.handlers import optparse #import __hello__ import threading @@ -109,20 +110,12 @@ class _Struct(ctypes.Structure): _filedescrip, _tempfile = tempfile.mkstemp('r') # deleted in cleanup _tmpf = tempfile.TemporaryFile('w') -# put the objects in order, if possible -try: - from collections import OrderedDict as odict -except ImportError: - try: - from ordereddict import OrderedDict as odict - except ImportError: - odict = dict # objects used by dill for type declaration -registered = d = odict() +registered = d = {} # objects dill fails to pickle -failures = x = odict() +failures = x = {} # all other type objects -succeeds = a = odict() +succeeds = a = {} # types module (part of CH 8) a['BooleanType'] = bool(1) @@ -169,7 +162,7 @@ class _Struct(ctypes.Structure): # data compression and archiving (CH 12) a['TarInfoType'] = tarfile.TarInfo() # generic operating system services (CH 15) -a['LoggerType'] = logging.getLogger() +a['LoggerType'] = _logger = logging.getLogger() a['FormatterType'] = logging.Formatter() # pickle ok a['FilterType'] = logging.Filter() # pickle ok a['LogRecordType'] = logging.makeLogRecord(_dict) # pickle ok @@ -201,40 +194,32 @@ class _Struct(ctypes.Structure): #NOTE: ctypes.c_int._objects is memberdescriptor for object's __dict__ #NOTE: base class of all ctypes data types is non-public _CData -try: # python 2.6 - import fractions - import number - import io - from io import StringIO as TextIO - # built-in functions (CH 2) - a['ByteArrayType'] = bytearray([1]) - # numeric and mathematical types (CH 9) - a['FractionType'] = fractions.Fraction() - a['NumberType'] = numbers.Number() - # generic operating system services (CH 15) - a['IOBaseType'] = io.IOBase() - a['RawIOBaseType'] = io.RawIOBase() - a['TextIOBaseType'] = io.TextIOBase() - a['BufferedIOBaseType'] = io.BufferedIOBase() - a['UnicodeIOType'] = TextIO() # the new StringIO - a['LoggingAdapterType'] = logging.LoggingAdapter(_logger,_dict) # pickle ok - if HAS_CTYPES: - a['CBoolType'] = ctypes.c_bool(1) - a['CLongDoubleType'] = ctypes.c_longdouble() -except ImportError: - pass -try: # python 2.7 - import argparse - # data types (CH 8) - a['OrderedDictType'] = collections.OrderedDict(_dict) - a['CounterType'] = collections.Counter(_dict) - if HAS_CTYPES: - a['CSSizeTType'] = ctypes.c_ssize_t() - # generic operating system services (CH 15) - a['NullHandlerType'] = logging.NullHandler() # pickle ok # new 2.7 - a['ArgParseFileType'] = argparse.FileType() # pickle ok -except (AttributeError, ImportError): - pass +import fractions +import io +from io import StringIO as TextIO +# built-in functions (CH 2) +a['ByteArrayType'] = bytearray([1]) +# numeric and mathematical types (CH 9) +a['FractionType'] = fractions.Fraction() +# generic operating system services (CH 15) +a['IOBaseType'] = io.IOBase() +a['RawIOBaseType'] = io.RawIOBase() +a['TextIOBaseType'] = io.TextIOBase() +a['BufferedIOBaseType'] = io.BufferedIOBase() +a['UnicodeIOType'] = TextIO() # the new StringIO +a['LoggerAdapterType'] = logging.LoggerAdapter(_logger,_dict) # pickle ok +if HAS_CTYPES: + a['CBoolType'] = ctypes.c_bool(1) + a['CLongDoubleType'] = ctypes.c_longdouble() +import argparse +# data types (CH 8) +a['OrderedDictType'] = collections.OrderedDict(_dict) +a['CounterType'] = collections.Counter(_dict) +if HAS_CTYPES: + a['CSSizeTType'] = ctypes.c_ssize_t() +# generic operating system services (CH 15) +a['NullHandlerType'] = logging.NullHandler() # pickle ok # new 2.7 +a['ArgParseFileType'] = argparse.FileType() # pickle ok # -- pickle fails on all below here ----------------------------------------- # types module (part of CH 8) @@ -346,27 +331,21 @@ class _Struct(ctypes.Structure): a['NumpyInt32Type'] = _numpy_int32 except ImportError: pass -try: # python 2.6 - # numeric and mathematical types (CH 9) - a['ProductType'] = itertools.product('0','1') - # generic operating system services (CH 15) - a['FileHandlerType'] = logging.FileHandler(os.devnull) #FIXME: fail >= 3.2 and <= 2.6 - a['RotatingFileHandlerType'] = logging.handlers.RotatingFileHandler(os.devnull) - a['SocketHandlerType'] = logging.handlers.SocketHandler('localhost',514) - a['MemoryHandlerType'] = logging.handlers.MemoryHandler(1) -except AttributeError: - pass -try: # python 2.7 - # data types (CH 8) - a['WeakSetType'] = weakref.WeakSet() # 2.7 -# # generic operating system services (CH 15) [errors when dill is imported] -# a['ArgumentParserType'] = _parser = argparse.ArgumentParser('PROG') -# a['NamespaceType'] = _parser.parse_args() # pickle ok -# a['SubParsersActionType'] = _parser.add_subparsers() -# a['MutuallyExclusiveGroupType'] = _parser.add_mutually_exclusive_group() -# a['ArgumentGroupType'] = _parser.add_argument_group() -except AttributeError: - pass +# numeric and mathematical types (CH 9) +a['ProductType'] = itertools.product('0','1') +# generic operating system services (CH 15) +a['FileHandlerType'] = logging.FileHandler(os.devnull) #FIXME: fail >= 3.2 and <= 2.6 +a['RotatingFileHandlerType'] = logging.handlers.RotatingFileHandler(os.devnull) +a['SocketHandlerType'] = logging.handlers.SocketHandler('localhost',514) +a['MemoryHandlerType'] = logging.handlers.MemoryHandler(1) +# data types (CH 8) +a['WeakSetType'] = weakref.WeakSet() # 2.7 +# generic operating system services (CH 15) [errors when dill is imported] +#a['ArgumentParserType'] = _parser = argparse.ArgumentParser('PROG') +#a['NamespaceType'] = _parser.parse_args() # pickle ok +#a['SubParsersActionType'] = _parser.add_subparsers() +#a['MutuallyExclusiveGroupType'] = _parser.add_mutually_exclusive_group() +#a['ArgumentGroupType'] = _parser.add_argument_group() # -- dill fails in some versions below here --------------------------------- # types module (part of CH 8) @@ -412,28 +391,21 @@ class _Struct(ctypes.Structure): a['PositionsIteratorType'] = compile('3', '', 'eval').co_positions() # data types (CH 8) -a['PrettyPrinterType'] = pprint.PrettyPrinter() #FIXME: fail >= 3.2 and == 2.5 +a['PrettyPrinterType'] = pprint.PrettyPrinter() #FIXME: fail >= 3.2 # numeric and mathematical types (CH 9) a['CycleType'] = itertools.cycle('0') #FIXME: fail < 3.2 # file and directory access (CH 10) -a['TemporaryFileType'] = _tmpf #FIXME: fail >= 3.2 and == 2.5 +a['TemporaryFileType'] = _tmpf #FIXME: fail >= 3.2 # data compression and archiving (CH 12) -a['GzipFileType'] = gzip.GzipFile(fileobj=_fileW) #FIXME: fail > 3.2 and <= 2.6 +a['GzipFileType'] = gzip.GzipFile(fileobj=_fileW) #FIXME: fail > 3.2 # generic operating system services (CH 15) -a['StreamHandlerType'] = logging.StreamHandler() #FIXME: fail >= 3.2 and == 2.5 -try: # python 2.6 - # numeric and mathematical types (CH 9) - a['PermutationsType'] = itertools.permutations('0') #FIXME: fail < 3.2 - a['CombinationsType'] = itertools.combinations('0',1) #FIXME: fail < 3.2 -except AttributeError: - pass -try: # python 2.7 - # numeric and mathematical types (CH 9) - a['RepeatType'] = itertools.repeat(0) #FIXME: fail < 3.2 - a['CompressType'] = itertools.compress('0',[1]) #FIXME: fail < 3.2 - #XXX: ...and etc -except AttributeError: - pass +a['StreamHandlerType'] = logging.StreamHandler() #FIXME: fail >= 3.2 +# numeric and mathematical types (CH 9) +a['PermutationsType'] = itertools.permutations('0') +a['CombinationsType'] = itertools.combinations('0',1) +a['RepeatType'] = itertools.repeat(0) +a['CompressType'] = itertools.compress('0',[1]) +#XXX: ...and etc # -- dill fails on all below here ------------------------------------------- # types module (part of CH 8) @@ -497,29 +469,20 @@ class _Struct(ctypes.Structure): x['FieldType'] = _field = _Struct._field x['CFUNCTYPEType'] = _cfunc = ctypes.CFUNCTYPE(ctypes.c_char) x['CFunctionType'] = _cfunc(str) -try: # python 2.6 - # numeric and mathematical types (CH 9) - x['MethodCallerType'] = operator.methodcaller('mro') # 2.6 -except AttributeError: - pass -try: # python 2.7 - # built-in types (CH 5) - x['MemoryType'] = memoryview(_in) # 2.7 - x['MemoryType2'] = memoryview(bytearray(_in)) # 2.7 - x['DictItemsType'] = _dict.items() # 2.7 - x['DictKeysType'] = _dict.keys() # 2.7 - x['DictValuesType'] = _dict.values() # 2.7 - # generic operating system services (CH 15) - x['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG') - x['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG') - x['ArgDefaultsHelpFormatterType'] = argparse.ArgumentDefaultsHelpFormatter('PROG') -except NameError: - pass -try: # python 2.7 (and not 3.1) - x['CmpKeyType'] = _cmpkey = functools.cmp_to_key(_methodwrap) # 2.7, >=3.2 - x['CmpKeyObjType'] = _cmpkey('0') #2.7, >=3.2 -except AttributeError: - pass +# numeric and mathematical types (CH 9) +x['MethodCallerType'] = operator.methodcaller('mro') # 2.6 +# built-in types (CH 5) +x['MemoryType'] = memoryview(_in) # 2.7 +x['MemoryType2'] = memoryview(bytearray(_in)) # 2.7 +x['DictItemsType'] = _dict.items() # 2.7 +x['DictKeysType'] = _dict.keys() # 2.7 +x['DictValuesType'] = _dict.values() # 2.7 +# generic operating system services (CH 15) +x['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG') +x['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG') +x['ArgDefaultsHelpFormatterType'] = argparse.ArgumentDefaultsHelpFormatter('PROG') +x['CmpKeyType'] = _cmpkey = functools.cmp_to_key(_methodwrap) # 2.7, >=3.2 +x['CmpKeyObjType'] = _cmpkey('0') #2.7, >=3.2 # oddities: removed, etc x['BufferType'] = x['MemoryType'] diff --git a/dill/settings.py b/dill/settings.py index 4d0226b0..b105d2e8 100644 --- a/dill/settings.py +++ b/dill/settings.py @@ -9,10 +9,7 @@ global settings for Pickler """ -try: - from pickle import DEFAULT_PROTOCOL -except ImportError: - from pickle import HIGHEST_PROTOCOL as DEFAULT_PROTOCOL +from pickle import DEFAULT_PROTOCOL settings = { #'main' : None, diff --git a/tests/__main__.py b/tests/__main__.py index 0ee6d9ae..e332076a 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -5,14 +5,12 @@ # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/dill/blob/master/LICENSE -import glob -import os +import glob, os, subprocess as sp try: import pox - python = pox.which_python(version=True, fullpath=False) or 'python' + python = pox.which_python(version=True, fullpath=False) or 'python3' except ImportError: - python = 'python' -import subprocess as sp + python = 'python3' from sys import platform shell = platform[:3] == 'win' diff --git a/tests/test_diff.py b/tests/test_diff.py index 3277682b..652d819b 100644 --- a/tests/test_diff.py +++ b/tests/test_diff.py @@ -55,18 +55,15 @@ def test_diff(): assert changed[1] if not IS_PYPY: - try: - import abc - # make sure the "_abc_invaldation_counter" doesn't make test fail - diff.memorise(abc.ABCMeta, force=True) - assert not diff.has_changed(abc) - abc.ABCMeta.zzz = 1 - assert diff.has_changed(abc) - changed = diff.whats_changed(abc) - assert list(changed[0].keys()) == ["ABCMeta"] - assert not changed[1] - except ImportError: - pass + import abc + # make sure the "_abc_invaldation_counter" doesn't make test fail + diff.memorise(abc.ABCMeta, force=True) + assert not diff.has_changed(abc) + abc.ABCMeta.zzz = 1 + assert diff.has_changed(abc) + changed = diff.whats_changed(abc) + assert list(changed[0].keys()) == ["ABCMeta"] + assert not changed[1] ''' import Queue diff --git a/tests/test_extendpickle.py b/tests/test_extendpickle.py index 49b6f152..0a346ae7 100644 --- a/tests/test_extendpickle.py +++ b/tests/test_extendpickle.py @@ -7,10 +7,7 @@ # - https://github.com/uqfoundation/dill/blob/master/LICENSE import dill as pickle -try: - from StringIO import StringIO -except ImportError: - from io import BytesIO as StringIO +from io import BytesIO as StringIO def my_fn(x): diff --git a/tests/test_module.py b/tests/test_module.py index 5c5e000c..2d009f26 100644 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -9,8 +9,7 @@ import sys import dill import test_mixins as module -try: from importlib import reload -except ImportError: pass +from importlib import reload dill.settings['recurse'] = True cached = (module.__cached__ if hasattr(module, "__cached__") diff --git a/tests/test_source.py b/tests/test_source.py index dfdb27df..01fc1eb3 100644 --- a/tests/test_source.py +++ b/tests/test_source.py @@ -83,14 +83,9 @@ def test_dynamic(): # classes and class instances def test_classes(): - try: #XXX: should this be a 'special case'? - from StringIO import StringIO - y = "from StringIO import StringIO\n" - x = y - except ImportError: - from io import BytesIO as StringIO - y = "from _io import BytesIO\n" - x = y if (IS_PYPY or sys.hexversion >= PY310b) else "from io import BytesIO\n" + from io import BytesIO as StringIO + y = "from _io import BytesIO\n" + x = y if (IS_PYPY or sys.hexversion >= PY310b) else "from io import BytesIO\n" s = StringIO() assert likely_import(StringIO) == x From 1f8db14c8e17f81559102bf6f5a3f1621fe36cd0 Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Tue, 21 Jun 2022 13:18:39 -0300 Subject: [PATCH 05/13] substitute bare 'except' clauses by 'except Exception' or more specific where it's obvious --- dill/__diff.py | 7 ++----- dill/_dill.py | 27 +++++++++++++-------------- dill/_objects.py | 4 ++-- dill/detect.py | 8 ++++---- dill/source.py | 22 +++++++++++----------- tests/test_extendpickle.py | 2 +- tests/test_functions.py | 2 +- tests/test_nested.py | 2 +- tests/test_recursive.py | 4 ++-- tests/test_selected.py | 4 ++-- tests/test_weakref.py | 2 +- 11 files changed, 40 insertions(+), 44 deletions(-) diff --git a/dill/__diff.py b/dill/__diff.py index 1ef424c0..3a0eabce 100644 --- a/dill/__diff.py +++ b/dill/__diff.py @@ -14,7 +14,7 @@ try: import numpy HAS_NUMPY = True -except: +except ImportError: HAS_NUMPY = False # pypy doesn't use reference counting @@ -38,10 +38,7 @@ def get_attrs(obj): if type(obj) in builtins_types \ or type(obj) is type and obj in builtins_types: return - try: - return obj.__dict__ - except: - return + return getattr(obj, '__dict__', None) def get_seq(obj, cache={str: False, frozenset: False, list: True, set: True, diff --git a/dill/_dill.py b/dill/_dill.py index b6d09af7..ed6e84b1 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -631,7 +631,7 @@ def use_diff(on=True): if _use_diff and diff is None: try: from . import diff as d - except: + except ImportError: import diff as d diff = d @@ -697,7 +697,7 @@ def _create_typemap(): try: import symtable _incedental_reverse_typemap["SymtableStentryType"] = type(symtable.symtable("", "string", "exec")._table) -except: +except ImportError: pass if sys.hexversion >= 0x30a00a0: @@ -711,7 +711,7 @@ def _create_typemap(): try: import winreg _incedental_reverse_typemap["HKEYType"] = winreg.HKEYType -except: +except ImportError: pass _reverse_typemap.update(_incedental_reverse_typemap) @@ -941,7 +941,7 @@ def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdat else: try: exists = os.path.exists(name) - except: + except Exception: exists = False if not exists: if strictio: @@ -1043,7 +1043,7 @@ def __ror__(self, a): # mapping referenced by the proxy. It may work for other implementations, # but is not guaranteed. MAPPING_PROXY_TRICK = __d is (DictProxyType(__d) | _dictproxy_helper_instance) -except: +except Exception: MAPPING_PROXY_TRICK = False del __d @@ -1108,7 +1108,7 @@ def _create_capsule(pointer, name, context, destructor): names = uname.rsplit('.', i) try: module = __import__(names[0]) - except: + except ImportError: pass obj = module for attr in names[1:]: @@ -1116,7 +1116,7 @@ def _create_capsule(pointer, name, context, destructor): capsule = obj attr_found = True break - except: + except Exception: pass if attr_found: @@ -1134,14 +1134,14 @@ def _getattr(objclass, name, repr_str): try: #XXX: works only for __builtin__ ? attr = repr_str.split("'")[3] return eval(attr+'.__dict__["'+name+'"]') - except: + except Exception: try: attr = objclass.__dict__ if type(attr) is DictProxyType: attr = attr[name] else: attr = getattr(objclass,name) - except: + except (AttributeError, KeyError): attr = getattr(objclass,name) return attr @@ -1194,7 +1194,7 @@ def _locate_function(obj, pickler=None): try: found, _ = _getattribute(module, obj.__qualname__) return found is obj - except: + except AttributeError: return False else: found = _import_module(module_name + '.' + obj.__name__, safe=True) @@ -1595,7 +1595,7 @@ def save_wrapper_descriptor(pickler, obj): def save_cell(pickler, obj): try: f = obj.cell_contents - except: + except ValueError: log.info("Ce3: %s" % obj) # _shims._CELL_EMPTY is defined in _shims.py to support PyPy 2.7. # It unpickles to a sentinel object _dill._CELL_EMPTY, also created in @@ -1902,7 +1902,7 @@ def save_function(pickler, obj): found, _ = _getattribute(module, obj.__qualname__) if getattr(found, '__func__', None) is obj: _pypy_builtin = True - except: + except AttributeError: pass if _pypy_builtin: @@ -2142,9 +2142,8 @@ def _extend(): for t,func in Pickler.dispatch.items(): try: StockPickler.dispatch[t] = func - except: #TypeError, PicklingError, UnpicklingError + except Exception: #TypeError, PicklingError, UnpicklingError log.info("skip: %s" % t) - else: pass return del diff, _use_diff, use_diff diff --git a/dill/_objects.py b/dill/_objects.py index da5d15ef..615aefb0 100644 --- a/dill/_objects.py +++ b/dill/_objects.py @@ -99,7 +99,7 @@ class _newclass2(object): def _function(x): yield x def _function2(): try: raise - except: + except Exception: from sys import exc_info e, er, tb = exc_info() return er, tb @@ -379,7 +379,7 @@ class _Struct(ctypes.Structure): try: import symtable a["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table -except: +except ImportError: pass if sys.hexversion >= 0x30a00a0: diff --git a/dill/detect.py b/dill/detect.py index 2297c6cd..db673dcf 100644 --- a/dill/detect.py +++ b/dill/detect.py @@ -44,7 +44,7 @@ def outermost(func): # is analogous to getsource(func,enclosing=True) # get the enclosing source from .source import getsourcelines try: lines,lnum = getsourcelines(func, enclosing=True) - except: #TypeError, IOError + except Exception: #TypeError, IOError lines,lnum = [],None code = ''.join(lines) # get all possible names,objects that are named in the enclosing source @@ -53,7 +53,7 @@ def outermost(func): # is analogous to getsource(func,enclosing=True) for name,obj in _locals: #XXX: don't really need 'name' try: if getsourcelines(obj) == (lines,lnum): return obj - except: #TypeError, IOError + except Exception: #TypeError, IOError pass return #XXX: or raise? no matches @@ -137,7 +137,7 @@ def get_cell_contents(): for (name,c) in zip(func,closures): try: cell_contents = c.cell_contents - except: + except ValueError: continue yield (name,c.cell_contents) @@ -186,7 +186,7 @@ def globalvars(func, recurse=True, builtin=False): for obj in getattr(orig_func, func_closure) or {}: try: cell_contents = obj.cell_contents - except: + except ValueError: pass else: _vars = globalvars(cell_contents, recurse, builtin) or {} diff --git a/dill/source.py b/dill/source.py index 792b53e6..7e70a635 100644 --- a/dill/source.py +++ b/dill/source.py @@ -56,7 +56,7 @@ def _matchlambda(func, line): lhs,rhs = line.split('lambda ',1)[-1].split(":", 1) #FIXME: if !1 inputs try: #FIXME: unsafe _ = eval("lambda %s : %s" % (lhs,rhs), globals(),locals()) - except: _ = dummy + except Exception: _ = dummy # get code objects, for comparison _, code = getcode(_).co_code, getcode(func).co_code # check if func is in closure @@ -78,7 +78,7 @@ def _matchlambda(func, line): _lhs,_rhs = rhs.split('lambda ',1)[-1].split(":",1) #FIXME: if !1 inputs try: #FIXME: unsafe _f = eval("lambda %s : %s" % (_lhs,_rhs), globals(),locals()) - except: _f = dummy + except Exception: _f = dummy # get code objects, for comparison _, code = getcode(_f).co_code, getcode(func).co_code if len(_) != len(code): return False @@ -118,7 +118,7 @@ def findsource(object): try: import readline err = '' - except: + except ImportError: import sys err = sys.exc_info()[1].args[0] if sys.platform[:3] == 'win': @@ -514,7 +514,7 @@ def func(*args, **kwds): try: # _ = eval(getsource(f, force=True)) #XXX: safer but less robust exec(getimportable(f, alias='_'), __globals__, __locals__) - except: + except Exception: raise ImportError('cannot import name ' + f.__name__) return _(*args, **kwds) func.__name__ = f.__name__ @@ -624,7 +624,7 @@ def _namespace(obj): if module in ['builtins','__builtin__']: # BuiltinFunctionType if _intypes(name): return ['types'] + [name] return qual + [name] #XXX: can be wrong for some aliased objects - except: pass + except Exception: pass # special case: numpy.inf and numpy.nan (we don't want them as floats) if str(obj) in ['inf','nan','Inf','NaN']: # is more, but are they needed? return ['numpy'] + [str(obj)] @@ -712,7 +712,7 @@ def getimport(obj, alias='', verify=True, builtin=False, enclosing=False): try: # look for '<...>' and be mindful it might be in lists, dicts, etc... name = repr(obj).split('<',1)[1].split('>',1)[1] name = None # we have a 'object'-style repr - except: # it's probably something 'importable' + except Exception: # it's probably something 'importable' if head in ['builtins','__builtin__']: name = repr(obj) #XXX: catch [1,2], (1,2), set([1,2])... others? else: @@ -770,7 +770,7 @@ def _importable(obj, alias='', source=None, enclosing=False, force=True, \ try: return getsource(obj, alias, enclosing=enclosing, \ force=force, lstrip=lstrip, builtin=builtin) - except: pass + except Exception: pass try: if not _isinstance(obj): return getimport(obj, alias, enclosing=enclosing, \ @@ -785,12 +785,12 @@ def _importable(obj, alias='', source=None, enclosing=False, force=True, \ if alias == name: _alias = "" return _import+_alias+"%s\n" % name - except: pass + except Exception: pass if not source: # try getsource, only if it hasn't been tried yet try: return getsource(obj, alias, enclosing=enclosing, \ force=force, lstrip=lstrip, builtin=builtin) - except: pass + except Exception: pass # get the name (of functions, lambdas, and classes) # or hope that obj can be built from the __repr__ #XXX: what to do about class instances and such? @@ -930,7 +930,7 @@ def importable(obj, alias='', source=None, builtin=True): if len(src) > 1: raise NotImplementedError('not implemented') return list(src.values())[0] - except: + except Exception: if tried_source: raise tried_import = True # we want the source @@ -969,7 +969,7 @@ def _code_stitcher(block): if not obj: return src if not src: return obj return obj + src - except: + except Exception: if tried_import: raise tried_source = True source = not source diff --git a/tests/test_extendpickle.py b/tests/test_extendpickle.py index 0a346ae7..10dc0804 100644 --- a/tests/test_extendpickle.py +++ b/tests/test_extendpickle.py @@ -44,7 +44,7 @@ def test_isdill(): pickler = mp.reduction.ForkingPickler(obj_io) assert pickle._dill.is_dill(pickler, child=True) is True assert pickle._dill.is_dill(pickler, child=False) is False - except: + except Exception: pass diff --git a/tests/test_functions.py b/tests/test_functions.py index 4e8936c4..d82c37e3 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -95,7 +95,7 @@ def test_functions(): assert 'empty' in str(cell_copy.__closure__[0]) try: cell_copy() - except: + except Exception: # this is good pass else: diff --git a/tests/test_nested.py b/tests/test_nested.py index 144f54b0..950641ca 100644 --- a/tests/test_nested.py +++ b/tests/test_nested.py @@ -110,7 +110,7 @@ def test_pickled_inner(): def test_moduledict_where_not_main(): try: from . import test_moduledict - except: + except ImportError: import test_moduledict name = 'test_moduledict.py' if os.path.exists(name) and os.path.exists(name+'c'): diff --git a/tests/test_recursive.py b/tests/test_recursive.py index a144f301..0f3a53a1 100644 --- a/tests/test_recursive.py +++ b/tests/test_recursive.py @@ -14,7 +14,7 @@ def copy(obj, byref=False, recurse=False): if byref: try: return dill.copy(obj, byref=byref, recurse=recurse) - except: + except Exception: pass else: raise AssertionError('Copy of %s with byref=True should have given a warning!' % (obj,)) @@ -144,7 +144,7 @@ def test_recursive_function(): for _fib in (fib3, fib4): try: _fib(5) - except: + except Exception: # This is expected to fail because fib no longer exists pass else: diff --git a/tests/test_selected.py b/tests/test_selected.py index 53b9cda2..0a6a01e6 100644 --- a/tests/test_selected.py +++ b/tests/test_selected.py @@ -19,7 +19,7 @@ def test_dict_contents(): for i,j in c.items(): #try: ok = dill.pickles(j) - #except: + #except Exception: # print ("FAIL: %s with %s" % (i, dill.detect.errors(j))) if verbose: print ("%s: %s, %s" % (ok, type(j), j)) assert ok @@ -29,7 +29,7 @@ def _g(x): yield x; def _f(): try: raise - except: + except Exception: from sys import exc_info e, er, tb = exc_info() return er, tb diff --git a/tests/test_weakref.py b/tests/test_weakref.py index ada7d140..0e99f3ea 100644 --- a/tests/test_weakref.py +++ b/tests/test_weakref.py @@ -76,7 +76,7 @@ def test_dictproxy(): from dill._dill import DictProxyType try: m = DictProxyType({"foo": "bar"}) - except: + except Exception: m = type.__dict__ mp = dill.copy(m) assert mp.items() == m.items() From 5488acc75ae999c89f96c7c994b0656a50ebb2b2 Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Thu, 23 Jun 2022 22:22:56 -0300 Subject: [PATCH 06/13] split module imports into separated lines --- dill/__diff.py | 5 ++++- dill/_shims.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dill/__diff.py b/dill/__diff.py index 3a0eabce..60268a6d 100644 --- a/dill/__diff.py +++ b/dill/__diff.py @@ -10,7 +10,10 @@ Module to show if an object has changed since it was memorised """ -import builtins, os, sys, types +import builtins +import os +import sys +import types try: import numpy HAS_NUMPY = True diff --git a/dill/_shims.py b/dill/_shims.py index bfdb385c..2e6641ca 100644 --- a/dill/_shims.py +++ b/dill/_shims.py @@ -49,7 +49,8 @@ def _setattr(object, name, value): https://github.com/uqfoundation/dill/pull/443 """ -import inspect, sys +import inspect +import sys _dill = sys.modules['dill._dill'] From 0df4aca4448e5fb895d1144a7ffa1731ec9d865a Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Thu, 23 Jun 2022 22:26:49 -0300 Subject: [PATCH 07/13] idem, but for test files --- tests/__main__.py | 4 +++- tests/test_session.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/__main__.py b/tests/__main__.py index e332076a..89cb7afa 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -5,7 +5,9 @@ # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/dill/blob/master/LICENSE -import glob, os, subprocess as sp +import glob +import os +import subprocess as sp try: import pox python = pox.which_python(version=True, fullpath=False) or 'python3' diff --git a/tests/test_session.py b/tests/test_session.py index 5d458834..689cc975 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -5,7 +5,12 @@ # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/dill/blob/master/LICENSE -import atexit, dill, os, sys, __main__ +import atexit +import os +import sys +import __main__ + +import dill session_file = os.path.join(os.path.dirname(__file__), 'session-byref-%s.pkl') From d8727dc3492422083ad99cdfcec93a1053f0d400 Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Fri, 24 Jun 2022 07:26:37 -0300 Subject: [PATCH 08/13] review: adjustments --- dill/detect.py | 54 ++++++++++++++++++++------------------------------ 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/dill/detect.py b/dill/detect.py index db673dcf..e3bcc2e3 100644 --- a/dill/detect.py +++ b/dill/detect.py @@ -11,8 +11,8 @@ import dis from inspect import ismethod, isfunction, istraceback, isframe, iscode -from .pointers import parent, reference, at, parents, children +from .pointers import parent, reference, at, parents, children from ._dill import _trace as trace __all__ = ['baditems','badobjects','badtypes','code','errors','freevars', @@ -71,14 +71,12 @@ def nestedcode(func, recurse=True): #XXX: or return dict of {co_name: co} ? return list(nested) def code(func): - '''get the code object for the given function or method + """get the code object for the given function or method NOTE: use dill.source.getsource(CODEOBJ) to get the source code - ''' - im_func = '__func__' - func_code = '__code__' - if ismethod(func): func = getattr(func, im_func) - if isfunction(func): func = getattr(func, func_code) + """ + if ismethod(func): func = func.__func__ + if isfunction(func): func = func.__code__ if istraceback(func): func = func.tb_frame if isframe(func): func = func.f_code if iscode(func): return func @@ -93,9 +91,6 @@ def referrednested(func, recurse=True): #XXX: return dict of {__name__: obj} ? If possible, python builds code objects, but delays building functions until func() is called. """ - att1 = '__code__' # functions - att0 = '__func__' # methods - import gc funcs = set() # get the code objects, and try to track down by referrence @@ -103,16 +98,16 @@ def referrednested(func, recurse=True): #XXX: return dict of {__name__: obj} ? # look for function objects that refer to the code object for obj in gc.get_referrers(co): # get methods - _ = getattr(obj, att0, None) # ismethod - if getattr(_, att1, None) is co: funcs.add(obj) + _ = getattr(obj, '__func__', None) # ismethod + if getattr(_, '__code__', None) is co: funcs.add(obj) # get functions - elif getattr(obj, att1, None) is co: funcs.add(obj) + elif getattr(obj, '__code__', None) is co: funcs.add(obj) # get frame objects elif getattr(obj, 'f_code', None) is co: funcs.add(obj) # get code objects elif hasattr(obj, 'co_code') and obj is co: funcs.add(obj) -# frameobjs => func.func_code.co_varnames not in func.func_code.co_cellvars -# funcobjs => func.func_code.co_cellvars not in func.func_code.co_varnames +# frameobjs => func.__code__.co_varnames not in func.__code__.co_cellvars +# funcobjs => func.__code__.co_cellvars not in func.__code__.co_varnames # frameobjs are not found, however funcobjs are... # (see: test_mixins.quad ... and test_mixins.wtf) # after execution, code objects get compiled, and then may be found by gc @@ -123,23 +118,20 @@ def freevars(func): """get objects defined in enclosing code that are referred to by func returns a dict of {name:object}""" - im_func = '__func__' - func_code = '__code__' - func_closure = '__closure__' - if ismethod(func): func = getattr(func, im_func) + if ismethod(func): func = func.__func__ if isfunction(func): - closures = getattr(func, func_closure) or () - func = getattr(func, func_code).co_freevars # get freevars + closures = func.__closure__ or () + func = func.__code__.co_freevars # get freevars else: return {} def get_cell_contents(): - for (name,c) in zip(func,closures): + for name, c in zip(func, closures): try: cell_contents = c.cell_contents except ValueError: continue - yield (name,c.cell_contents) + yield name, c.cell_contents return dict(get_cell_contents()) @@ -174,16 +166,12 @@ def globalvars(func, recurse=True, builtin=False): """get objects defined in global scope that are referred to by func return a dict of {name:object}""" - im_func = '__func__' - func_code = '__code__' - func_globals = '__globals__' - func_closure = '__closure__' - if ismethod(func): func = getattr(func, im_func) + if ismethod(func): func = func.__func__ if isfunction(func): globs = vars(getmodule(sum)).copy() if builtin else {} # get references from within closure orig_func, func = func, set() - for obj in getattr(orig_func, func_closure) or {}: + for obj in orig_func.__closure__ or {}: try: cell_contents = obj.cell_contents except ValueError: @@ -193,12 +181,12 @@ def globalvars(func, recurse=True, builtin=False): func.update(_vars) #XXX: (above) be wary of infinte recursion? globs.update(_vars) # get globals - globs.update(getattr(orig_func, func_globals) or {}) + globs.update(orig_func.__globals__ or {}) # get names of references if not recurse: - func.update(getattr(orig_func, func_code).co_names) + func.update(orig_func.__code__.co_names) else: - func.update(nestedglobals(getattr(orig_func, func_code))) + func.update(nestedglobals(orig_func.__code__)) # find globals for all entries of func for key in func.copy(): #XXX: unnecessary...? nested_func = globs.get(key) @@ -223,7 +211,7 @@ def globalvars(func, recurse=True, builtin=False): func.update(globalvars(nested_func, True, builtin)) else: return {} - #NOTE: if name not in func_globals, then we skip it... + #NOTE: if name not in __globals__, then we skip it... return dict((name,globs[name]) for name in func if name in globs) From d05acd02e87eb241ee8abfcc75f59afd32ea39f0 Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Fri, 24 Jun 2022 07:32:02 -0300 Subject: [PATCH 09/13] tests: get a default Python 3 executable --- tests/__main__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/__main__.py b/tests/__main__.py index 89cb7afa..4b2db127 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -7,23 +7,25 @@ import glob import os +import shutil import subprocess as sp + +from sys import platform +shell = (platform[:3] == 'win') + +default_python = 'python3' if shutil.which('python3') else 'python' try: import pox - python = pox.which_python(version=True, fullpath=False) or 'python3' + python = pox.which_python(version=True, fullpath=False) or default_python except ImportError: - python = 'python3' -from sys import platform -shell = platform[:3] == 'win' + python = default_python suite = os.path.dirname(__file__) or os.path.curdir tests = glob.glob(suite + os.path.sep + 'test_*.py') - if __name__ == '__main__': - for test in tests: p = sp.Popen([python, test], shell=shell).wait() if not p: print('.', end='', flush=True) - print('') + print() From 6221f624f4e285cc92fcc88aabf0c22fe7397f47 Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Fri, 24 Jun 2022 09:59:47 -0300 Subject: [PATCH 10/13] update module docs --- dill/__init__.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/dill/__init__.py b/dill/__init__.py index b4abca21..54ee6fe4 100644 --- a/dill/__init__.py +++ b/dill/__init__.py @@ -42,7 +42,9 @@ ``dill`` is part of ``pathos``, a python framework for heterogeneous computing. ``dill`` is in active development, so any user feedback, bug reports, comments, -or suggestions are highly appreciated. A list of issues is located at https://github.com/uqfoundation/dill/issues, with a legacy list maintained at https://uqfoundation.github.io/project/pathos/query. +or suggestions are highly appreciated. A list of issues is located at +https://github.com/uqfoundation/dill/issues, with a legacy list maintained at +https://uqfoundation.github.io/project/pathos/query. Major Features @@ -50,19 +52,18 @@ ``dill`` can pickle the following standard types: - - none, type, bool, int, long, float, complex, str, unicode, + - none, type, bool, int, float, complex, bytes, str, - tuple, list, dict, file, buffer, builtin, - - both old and new style classes, - - instances of old and new style classes, + - python classes, namedtuples, dataclasses, metaclasses, + - instances of classes, - set, frozenset, array, functions, exceptions ``dill`` can also pickle more 'exotic' standard types: - functions with yields, nested functions, lambdas, - cell, method, unboundmethod, module, code, methodwrapper, - - dictproxy, methoddescriptor, getsetdescriptor, memberdescriptor, - - wrapperdescriptor, xrange, slice, - - notimplemented, ellipsis, quit + - methoddescriptor, getsetdescriptor, memberdescriptor, wrapperdescriptor, + - dictproxy, slice, notimplemented, ellipsis, quit ``dill`` cannot yet pickle these standard types: @@ -148,7 +149,7 @@ as keyword arguments to several ``dill`` functions: * with *protocol*, the pickle protocol level can be set. This uses the - same value as the ``pickle`` module, *HIGHEST_PROTOCOL* or *DEFAULT_PROTOCOL*. + same value as the ``pickle`` module, *DEFAULT_PROTOCOL*. * with *byref=True*, ``dill`` to behave a lot more like pickle with certain objects (like modules) pickled by reference as opposed to attempting to pickle the object itself. From de6a1916a2425407c9fe2c6dd15b1858222b0ef1 Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Fri, 24 Jun 2022 19:03:10 -0300 Subject: [PATCH 11/13] update README --- README.md | 19 ++++++++++--------- dill/__init__.py | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 2b3688e6..a9c39f19 100644 --- a/README.md +++ b/README.md @@ -28,26 +28,27 @@ a trustworthy source. ``dill`` is part of ``pathos``, a python framework for heterogeneous computing. ``dill`` is in active development, so any user feedback, bug reports, comments, -or suggestions are highly appreciated. A list of issues is located at https://github.com/uqfoundation/dill/issues, with a legacy list maintained at https://uqfoundation.github.io/project/pathos/query. +or suggestions are highly appreciated. A list of issues is located at +https://github.com/uqfoundation/dill/issues, with a legacy list maintained at +https://uqfoundation.github.io/project/pathos/query. Major Features -------------- ``dill`` can pickle the following standard types: -* none, type, bool, int, long, float, complex, str, unicode, +* none, type, bool, int, float, complex, bytes, str, * tuple, list, dict, file, buffer, builtin, -* both old and new style classes, -* instances of old and new style classes, +* python classes, namedtuples, dataclasses, metaclasses, +* instances of classes, * set, frozenset, array, functions, exceptions ``dill`` can also pickle more 'exotic' standard types: -* functions with yields, nested functions, lambdas +* functions with yields, nested functions, lambdas, * cell, method, unboundmethod, module, code, methodwrapper, -* dictproxy, methoddescriptor, getsetdescriptor, memberdescriptor, -* wrapperdescriptor, xrange, slice, -* notimplemented, ellipsis, quit +* methoddescriptor, getsetdescriptor, memberdescriptor, wrapperdescriptor, +* dictproxy, slice, notimplemented, ellipsis, quit ``dill`` cannot yet pickle these standard types: @@ -133,7 +134,7 @@ There are a number of options to control serialization which are provided as keyword arguments to several ``dill`` functions: * with *protocol*, the pickle protocol level can be set. This uses the - same value as the ``pickle`` module, *HIGHEST_PROTOCOL* or *DEFAULT_PROTOCOL*. + same value as the ``pickle`` module, *DEFAULT_PROTOCOL*. * with *byref=True*, ``dill`` to behave a lot more like pickle with certain objects (like modules) pickled by reference as opposed to attempting to pickle the object itself. diff --git a/dill/__init__.py b/dill/__init__.py index 54ee6fe4..fdc01bfe 100644 --- a/dill/__init__.py +++ b/dill/__init__.py @@ -26,7 +26,7 @@ ``dill`` provides the user the same interface as the ``pickle`` module, and also includes some additional features. In addition to pickling python objects, ``dill`` provides the ability to save the state of an interpreter -session in a single command. Hence, it would be feasable to save an +session in a single command. Hence, it would be feasible to save an interpreter session, close the interpreter, ship the pickled file to another computer, open a new interpreter, unpickle the session and thus continue from the 'saved' state of the original interpreter From af4e4eebe127d4b39091a70faa5218d97da769d5 Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Fri, 24 Jun 2022 20:02:26 -0300 Subject: [PATCH 12/13] review: corrections --- dill/_dill.py | 14 ++++---------- dill/detect.py | 4 ++-- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/dill/_dill.py b/dill/_dill.py index ed6e84b1..c43a8fea 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -373,7 +373,7 @@ def _stash_modules(main_module): imported_as = [] imported_top_level = [] # keep separeted for backwards compatibility original = {} - for name, obj in vars(main_module).items(): + for name, obj in main_module.__dict__.items(): if obj is main_module: original[name] = newmod # self-reference continue @@ -880,7 +880,7 @@ def _create_code(*args): return CodeType(*args) elif len(args) == 15: return CodeType(args[0], 0, *args[1:]) # from 3.7 return CodeType(args[0], 0, 0, *args[1:]) # from 2.7 - elif hasattr(CodeType, 'co_kwonlyargcount'): # python 3.7 + else: # python 3.7 # obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, # obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, # obj.co_names, obj.co_varnames, obj.co_filename, @@ -1595,7 +1595,7 @@ def save_wrapper_descriptor(pickler, obj): def save_cell(pickler, obj): try: f = obj.cell_contents - except ValueError: + except ValueError: # cell is empty log.info("Ce3: %s" % obj) # _shims._CELL_EMPTY is defined in _shims.py to support PyPy 2.7. # It unpickles to a sentinel object _dill._CELL_EMPTY, also created in @@ -1868,13 +1868,7 @@ def save_property(pickler, obj): @register(classmethod) def save_classmethod(pickler, obj): log.info("Cm: %s" % obj) - im_func = '__func__' - try: - orig_func = getattr(obj, im_func) - except AttributeError: # Python 2.6 - orig_func = obj.__get__(None, object) - if isinstance(obj, classmethod): - orig_func = getattr(orig_func, im_func) # Unbind + orig_func = obj.__func__ # if type(obj.__dict__) is dict: # if obj.__dict__: diff --git a/dill/detect.py b/dill/detect.py index e3bcc2e3..8037ddc9 100644 --- a/dill/detect.py +++ b/dill/detect.py @@ -129,7 +129,7 @@ def get_cell_contents(): for name, c in zip(func, closures): try: cell_contents = c.cell_contents - except ValueError: + except ValueError: # cell is empty continue yield name, c.cell_contents @@ -174,7 +174,7 @@ def globalvars(func, recurse=True, builtin=False): for obj in orig_func.__closure__ or {}: try: cell_contents = obj.cell_contents - except ValueError: + except ValueError: # cell is empty pass else: _vars = globalvars(cell_contents, recurse, builtin) or {} From 42fa293a58ba041373c53f2684e88b68f44aa322 Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Fri, 1 Jul 2022 19:27:27 -0300 Subject: [PATCH 13/13] remove unicode prefix from strings in docs/source/conf.py --- docs/source/conf.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 19171caf..cbf7cf14 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -58,10 +58,10 @@ master_doc = 'index' # General information about the project. -project = u'dill' +project = 'dill' year = datetime.now().year -copyright = u'%d, The Uncertainty Quantification Foundation' % year -author = u'Mike McKerns' +copyright = '%d, The Uncertainty Quantification Foundation' % year +author = 'Mike McKerns' # extension config github_project_url = "https://github.com/uqfoundation/dill" @@ -191,8 +191,8 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'dill.tex', u'dill Documentation', - u'Mike McKerns', 'manual'), + (master_doc, 'dill.tex', 'dill Documentation', + 'Mike McKerns', 'manual'), ] @@ -201,7 +201,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'dill', u'dill Documentation', + (master_doc, 'dill', 'dill Documentation', [author], 1) ] @@ -212,7 +212,7 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'dill', u'dill Documentation', + (master_doc, 'dill', 'dill Documentation', author, 'dill', 'Serialize all of python.', 'Miscellaneous'), ]