diff --git a/.gitignore b/.gitignore index 9e136965..477f7cec 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ /docs/build /build /README -/dill/info.py \ No newline at end of file +/dill/__info__.py diff --git a/.travis.yml b/.travis.yml index 4fbe2282..423c3a2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,8 +40,8 @@ install: - python -m pip install . script: - - for test in tests/__init__.py; do echo $test ; if [[ $COVERAGE == "true" ]]; then coverage run -a $test > /dev/null; else python $test > /dev/null; fi ; done - - for test in tests/test_*.py; do echo $test ; if [[ $COVERAGE == "true" ]]; then coverage run -a $test > /dev/null; else python $test > /dev/null; fi ; done + - for test in dill/tests/__init__.py; do echo $test ; if [[ $COVERAGE == "true" ]]; then coverage run -a $test > /dev/null; else python $test > /dev/null; fi ; done + - for test in dill/tests/test_*.py; do echo $test ; if [[ $COVERAGE == "true" ]]; then coverage run -a $test > /dev/null; else python $test > /dev/null; fi ; done after_success: - if [[ $COVERAGE == "true" ]]; then bash <(curl -s https://codecov.io/bash); else echo ''; fi diff --git a/MANIFEST.in b/MANIFEST.in index 0cd47401..1309d768 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,8 +3,8 @@ include README* include MANIFEST.in include pyproject.toml include tox.ini +include version.py include scripts/* -include tests/*py recursive-include docs * include .* prune .git diff --git a/README.md b/README.md index 2b3688e6..de42db1b 100644 --- a/README.md +++ b/README.md @@ -28,26 +28,27 @@ a trustworthy source. ``dill`` is part of ``pathos``, a python framework for heterogeneous computing. ``dill`` is in active development, so any user feedback, bug reports, comments, -or suggestions are highly appreciated. A list of issues is located at https://github.com/uqfoundation/dill/issues, with a legacy list maintained at https://uqfoundation.github.io/project/pathos/query. +or suggestions are highly appreciated. A list of issues is located at +https://github.com/uqfoundation/dill/issues, with a legacy list maintained at +https://uqfoundation.github.io/project/pathos/query. Major Features -------------- ``dill`` can pickle the following standard types: -* none, type, bool, int, long, float, complex, str, unicode, +* none, type, bool, int, float, complex, bytes, str, * tuple, list, dict, file, buffer, builtin, -* both old and new style classes, -* instances of old and new style classes, +* python classes, namedtuples, dataclasses, metaclasses, +* instances of classes, * set, frozenset, array, functions, exceptions ``dill`` can also pickle more 'exotic' standard types: -* functions with yields, nested functions, lambdas +* functions with yields, nested functions, lambdas, * cell, method, unboundmethod, module, code, methodwrapper, -* dictproxy, methoddescriptor, getsetdescriptor, memberdescriptor, -* wrapperdescriptor, xrange, slice, -* notimplemented, ellipsis, quit +* methoddescriptor, getsetdescriptor, memberdescriptor, wrapperdescriptor, +* dictproxy, slice, notimplemented, ellipsis, quit ``dill`` cannot yet pickle these standard types: @@ -133,7 +134,7 @@ There are a number of options to control serialization which are provided as keyword arguments to several ``dill`` functions: * with *protocol*, the pickle protocol level can be set. This uses the - same value as the ``pickle`` module, *HIGHEST_PROTOCOL* or *DEFAULT_PROTOCOL*. + same value as the ``pickle`` module, *DEFAULT_PROTOCOL*. * with *byref=True*, ``dill`` to behave a lot more like pickle with certain objects (like modules) pickled by reference as opposed to attempting to pickle the object itself. @@ -170,26 +171,30 @@ To aid in debugging pickling issues, use *dill.detect* which provides tools like pickle tracing:: >>> import dill.detect - >>> dill.detect.trace(True) - >>> f = dumps(squared) - F1: at 0x108899e18> - F2: - # F2 - Co: at 0x10866a270, file "", line 1> - F2: - # F2 - # Co - D1: - # D1 - D2: - # D2 - # F1 - >>> dill.detect.trace(False) + >>> with dill.detect.trace(): + >>> dumps(squared) + ┬ F1: at 0x7fe074f8c280> + ├┬ F2: + │└ # F2 [34 B] + ├┬ Co: at 0x7fe07501eb30, file "", line 1> + │├┬ F2: + ││└ # F2 [19 B] + │└ # Co [87 B] + ├┬ D1: + │└ # D1 [22 B] + ├┬ D2: + │└ # D2 [2 B] + ├┬ D2: + │├┬ D2: + ││└ # D2 [2 B] + │└ # D2 [23 B] + └ # F1 [180 B] With trace, we see how ``dill`` stored the lambda (``F1``) by first storing ``_create_function``, the underlying code object (``Co``) and ``_create_code`` (which is used to handle code objects), then we handle the reference to -the global dict (``D2``). A ``#`` marks when the object is actually stored. +the global dict (``D2``) plus other dictionaries (``D1`` and ``D2``) that +save the lambda object's state. A ``#`` marks when the object is actually stored. More Information diff --git a/dill/__diff.py b/dill/__diff.py index df2589eb..60268a6d 100644 --- a/dill/__diff.py +++ b/dill/__diff.py @@ -10,18 +10,15 @@ Module to show if an object has changed since it was memorised """ +import builtins import os import sys import types try: import numpy HAS_NUMPY = True -except: - HAS_NUMPY = False -try: - import builtins except ImportError: - import __builtin__ as builtins + HAS_NUMPY = False # pypy doesn't use reference counting getrefcount = getattr(sys, 'getrefcount', lambda x:0) @@ -44,10 +41,7 @@ def get_attrs(obj): if type(obj) in builtins_types \ or type(obj) is type and obj in builtins_types: return - try: - return obj.__dict__ - except: - return + return getattr(obj, '__dict__', None) def get_seq(obj, cache={str: False, frozenset: False, list: True, set: True, @@ -235,6 +229,6 @@ def _imp(*args, **kwds): # memorise all already imported modules. This implies that this must be # imported first for any changes to be recorded -for mod in sys.modules.values(): +for mod in list(sys.modules.values()): memorise(mod) release_gone() diff --git a/dill/__init__.py b/dill/__init__.py index ac93ff6a..6f71bbe5 100644 --- a/dill/__init__.py +++ b/dill/__init__.py @@ -7,312 +7,44 @@ # - https://github.com/uqfoundation/dill/blob/master/LICENSE # author, version, license, and long description -__version__ = '0.3.6.dev0' -__author__ = 'Mike McKerns' - -__doc__ = """ ------------------------------ -dill: serialize all of python ------------------------------ - -About Dill -========== - -``dill`` extends python's ``pickle`` module for serializing and de-serializing -python objects to the majority of the built-in python types. Serialization -is the process of converting an object to a byte stream, and the inverse -of which is converting a byte stream back to a python object hierarchy. - -``dill`` provides the user the same interface as the ``pickle`` module, and -also includes some additional features. In addition to pickling python -objects, ``dill`` provides the ability to save the state of an interpreter -session in a single command. Hence, it would be feasable to save an -interpreter session, close the interpreter, ship the pickled file to -another computer, open a new interpreter, unpickle the session and -thus continue from the 'saved' state of the original interpreter -session. - -``dill`` can be used to store python objects to a file, but the primary -usage is to send python objects across the network as a byte stream. -``dill`` is quite flexible, and allows arbitrary user defined classes -and functions to be serialized. Thus ``dill`` is not intended to be -secure against erroneously or maliciously constructed data. It is -left to the user to decide whether the data they unpickle is from -a trustworthy source. - -``dill`` is part of ``pathos``, a python framework for heterogeneous computing. -``dill`` is in active development, so any user feedback, bug reports, comments, -or suggestions are highly appreciated. A list of issues is located at https://github.com/uqfoundation/dill/issues, with a legacy list maintained at https://uqfoundation.github.io/project/pathos/query. - - -Major Features -============== - -``dill`` can pickle the following standard types: - - - none, type, bool, int, long, float, complex, str, unicode, - - tuple, list, dict, file, buffer, builtin, - - both old and new style classes, - - instances of old and new style classes, - - set, frozenset, array, functions, exceptions - -``dill`` can also pickle more 'exotic' standard types: - - - functions with yields, nested functions, lambdas, - - cell, method, unboundmethod, module, code, methodwrapper, - - dictproxy, methoddescriptor, getsetdescriptor, memberdescriptor, - - wrapperdescriptor, xrange, slice, - - notimplemented, ellipsis, quit - -``dill`` cannot yet pickle these standard types: - - - frame, generator, traceback - -``dill`` also provides the capability to: - - - save and load python interpreter sessions - - save and extract the source code from functions and classes - - interactively diagnose pickling errors - - -Current Release -=============== - -The latest released version of ``dill`` is available from: - - https://pypi.org/project/dill - -``dill`` is distributed under a 3-clause BSD license. - - -Development Version -=================== - -You can get the latest development version with all the shiny new features at: - - https://github.com/uqfoundation - -If you have a new contribution, please submit a pull request. - - -Installation -============ - -``dill`` can be installed with ``pip``:: - - $ pip install dill - -To optionally include the ``objgraph`` diagnostic tool in the install:: - - $ pip install dill[graph] - -For windows users, to optionally install session history tools:: - - $ pip install dill[readline] - - -Requirements -============ - -``dill`` requires: - - - ``python`` (or ``pypy``), **>=3.7** - - ``setuptools``, **>=42** - -Optional requirements: - - - ``objgraph``, **>=1.7.2** - - ``pyreadline``, **>=1.7.1** (on windows) - - -Basic Usage -=========== - -``dill`` is a drop-in replacement for ``pickle``. Existing code can be -updated to allow complete pickling using:: - - >>> import dill as pickle - -or:: - - >>> from dill import dumps, loads - -``dumps`` converts the object to a unique byte string, and ``loads`` performs -the inverse operation:: - - >>> squared = lambda x: x**2 - >>> loads(dumps(squared))(3) - 9 - -There are a number of options to control serialization which are provided -as keyword arguments to several ``dill`` functions: - -* with *protocol*, the pickle protocol level can be set. This uses the - same value as the ``pickle`` module, *HIGHEST_PROTOCOL* or *DEFAULT_PROTOCOL*. -* with *byref=True*, ``dill`` to behave a lot more like pickle with - certain objects (like modules) pickled by reference as opposed to - attempting to pickle the object itself. -* with *recurse=True*, objects referred to in the global dictionary are - recursively traced and pickled, instead of the default behavior of - attempting to store the entire global dictionary. -* with *fmode*, the contents of the file can be pickled along with the file - handle, which is useful if the object is being sent over the wire to a - remote system which does not have the original file on disk. Options are - *HANDLE_FMODE* for just the handle, *CONTENTS_FMODE* for the file content - and *FILE_FMODE* for content and handle. -* with *ignore=False*, objects reconstructed with types defined in the - top-level script environment use the existing type in the environment - rather than a possibly different reconstructed type. - -The default serialization can also be set globally in *dill.settings*. -Thus, we can modify how ``dill`` handles references to the global dictionary -locally or globally:: - - >>> import dill.settings - >>> dumps(absolute) == dumps(absolute, recurse=True) - False - >>> dill.settings['recurse'] = True - >>> dumps(absolute) == dumps(absolute, recurse=True) - True - -``dill`` also includes source code inspection, as an alternate to pickling:: - - >>> import dill.source - >>> print(dill.source.getsource(squared)) - squared = lambda x:x**2 - -To aid in debugging pickling issues, use *dill.detect* which provides -tools like pickle tracing:: - - >>> import dill.detect - >>> dill.detect.trace(True) - >>> f = dumps(squared) - F1: at 0x108899e18> - F2: - # F2 - Co: at 0x10866a270, file "", line 1> - F2: - # F2 - # Co - D1: - # D1 - D2: - # D2 - # F1 - >>> dill.detect.trace(False) - -With trace, we see how ``dill`` stored the lambda (``F1``) by first storing -``_create_function``, the underlying code object (``Co``) and ``_create_code`` -(which is used to handle code objects), then we handle the reference to -the global dict (``D2``). A ``#`` marks when the object is actually stored. - - -More Information -================ - -Probably the best way to get started is to look at the documentation at -http://dill.rtfd.io. Also see ``dill.tests`` for a set of scripts that -demonstrate how ``dill`` can serialize different python objects. You can -run the test suite with ``python -m dill.tests``. The contents of any -pickle file can be examined with ``undill``. As ``dill`` conforms to -the ``pickle`` interface, the examples and documentation found at -http://docs.python.org/library/pickle.html also apply to ``dill`` -if one will ``import dill as pickle``. The source code is also generally -well documented, so further questions may be resolved by inspecting the -code itself. Please feel free to submit a ticket on github, or ask a -question on stackoverflow (**@Mike McKerns**). -If you would like to share how you use ``dill`` in your work, please send -an email (to **mmckerns at uqfoundation dot org**). - - -Citation -======== - -If you use ``dill`` to do research that leads to publication, we ask that you -acknowledge use of ``dill`` by citing the following in your publication:: - - M.M. McKerns, L. Strand, T. Sullivan, A. Fang, M.A.G. Aivazis, - "Building a framework for predictive science", Proceedings of - the 10th Python in Science Conference, 2011; - http://arxiv.org/pdf/1202.1056 - - Michael McKerns and Michael Aivazis, - "pathos: a framework for heterogeneous computing", 2010- ; - https://uqfoundation.github.io/project/pathos - -Please see https://uqfoundation.github.io/project/pathos or -http://arxiv.org/pdf/1202.1056 for further information. - -""" - -__license__ = """ -Copyright (c) 2004-2016 California Institute of Technology. -Copyright (c) 2016-2022 The Uncertainty Quantification Foundation. -All rights reserved. - -This software is available subject to the conditions and terms laid -out below. By downloading and using this software you are agreeing -to the following conditions. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met:: - - - Redistribution of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - - Redistribution in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentations and/or other materials provided with the distribution. - - - Neither the names of the copyright holders nor the names of any of - the contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -""" - -from ._dill import dump, dumps, load, loads, dump_session, load_session, \ - Pickler, Unpickler, register, copy, pickle, pickles, check, \ - HIGHEST_PROTOCOL, DEFAULT_PROTOCOL, PicklingError, UnpicklingError, \ - HANDLE_FMODE, CONTENTS_FMODE, FILE_FMODE, PickleError, PickleWarning, \ - PicklingWarning, UnpicklingWarning -from . import source, temp, detect +try: # the package is installed + from .__info__ import __version__, __author__, __doc__, __license__ +except: # pragma: no cover + import os + import sys + parent = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) + sys.path.append(parent) + # get distribution meta info + from version import (__version__, __author__, + get_license_text, get_readme_as_rst) + __license__ = get_license_text(os.path.join(parent, 'LICENSE')) + __license__ = "\n%s" % __license__ + __doc__ = get_readme_as_rst(os.path.join(parent, 'README.md')) + del os, sys, parent, get_license_text, get_readme_as_rst + + +from ._dill import ( + Pickler, Unpickler, + check, copy, dump, dumps, load, loads, pickle, pickles, register, + DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, CONTENTS_FMODE, FILE_FMODE, HANDLE_FMODE, + PickleError, PickleWarning, PicklingError, PicklingWarning, UnpicklingError, + UnpicklingWarning, +) +from .session import ( + dump_module, load_module, load_module_asdict, + dump_session, load_session # backward compatibility +) +from . import detect, logger, session, source, temp # get global settings from .settings import settings # make sure "trace" is turned off -detect.trace(False) +logger.trace(False) -try: - from importlib import reload -except ImportError: - try: - from imp import reload - except ImportError: - pass +from importlib import reload -# put the objects in order, if possible -try: - from collections import OrderedDict as odict -except ImportError: - try: - from ordereddict import OrderedDict as odict - except ImportError: - odict = dict -objects = odict() +objects = {} # local import of dill._objects #from . import _objects #objects.update(_objects.succeeds) @@ -373,7 +105,6 @@ def extend(use_dill=True): return extend() -del odict def license(): diff --git a/dill/_dill.py b/dill/_dill.py index 0869d766..90f05806 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -15,94 +15,59 @@ Test against "all" python types (Std. Lib. CH 1-15 @ 2.7) by mmckerns. Test against CH16+ Std. Lib. ... TBD. """ -__all__ = ['dump','dumps','load','loads','dump_session','load_session', - 'Pickler','Unpickler','register','copy','pickle','pickles', - 'check','HIGHEST_PROTOCOL','DEFAULT_PROTOCOL','PicklingError', - 'UnpicklingError','HANDLE_FMODE','CONTENTS_FMODE','FILE_FMODE', - 'PickleError','PickleWarning','PicklingWarning','UnpicklingWarning'] - -import logging -log = logging.getLogger("dill") -log.addHandler(logging.StreamHandler()) -def _trace(boolean): - """print a trace through the stack when pickling; useful for debugging""" - if boolean: log.setLevel(logging.INFO) - else: log.setLevel(logging.WARN) - return +__all__ = [ + 'Pickler','Unpickler', + 'check','copy','dump','dumps','load','loads','pickle','pickles','register', + 'DEFAULT_PROTOCOL','HIGHEST_PROTOCOL','CONTENTS_FMODE','FILE_FMODE','HANDLE_FMODE', + 'PickleError','PickleWarning','PicklingError','PicklingWarning','UnpicklingError', + 'UnpicklingWarning', +] + +__module__ = 'dill' + import warnings +from .logger import adapter as logger +from .logger import trace as _trace import os import sys diff = None _use_diff = False -PY3 = (sys.hexversion >= 0x3000000) -# OLDER: 3.0 <= x < 3.4 *OR* x < 2.7.10 #NOTE: guessing relevant versions -OLDER = (PY3 and sys.hexversion < 0x3040000) or (sys.hexversion < 0x2070ab1) -OLD33 = (sys.hexversion < 0x3030000) -OLD37 = (sys.hexversion < 0x3070000) +OLD38 = (sys.hexversion < 0x3080000) OLD39 = (sys.hexversion < 0x3090000) OLD310 = (sys.hexversion < 0x30a0000) -PY34 = (0x3040000 <= sys.hexversion < 0x3050000) -if PY3: #XXX: get types from .objtypes ? - import builtins as __builtin__ - from pickle import _Pickler as StockPickler, Unpickler as StockUnpickler - from _thread import LockType - if (sys.hexversion >= 0x30200f0): - from _thread import RLock as RLockType - else: - from threading import _RLock as RLockType - #from io import IOBase - from types import CodeType, FunctionType, MethodType, GeneratorType, \ - TracebackType, FrameType, ModuleType, BuiltinMethodType - BufferType = memoryview #XXX: unregistered - ClassType = type # no 'old-style' classes - EllipsisType = type(Ellipsis) - #FileType = IOBase - NotImplementedType = type(NotImplemented) - SliceType = slice - TypeType = type # 'new-style' classes #XXX: unregistered - XRangeType = range - if OLD33: - DictProxyType = type(object.__dict__) - else: - from types import MappingProxyType as DictProxyType -else: - import __builtin__ - from pickle import Pickler as StockPickler, Unpickler as StockUnpickler - from thread import LockType - from threading import _RLock as RLockType - from types import CodeType, FunctionType, ClassType, MethodType, \ - GeneratorType, DictProxyType, XRangeType, SliceType, TracebackType, \ - NotImplementedType, EllipsisType, FrameType, ModuleType, \ - BufferType, BuiltinMethodType, TypeType -from pickle import HIGHEST_PROTOCOL, PickleError, PicklingError, \ - UnpicklingError -try: - from pickle import DEFAULT_PROTOCOL -except ImportError: - DEFAULT_PROTOCOL = HIGHEST_PROTOCOL +#XXX: get types from .objtypes ? +import builtins as __builtin__ +from pickle import _Pickler as StockPickler, Unpickler as StockUnpickler +from _thread import LockType +from _thread import RLock as RLockType +#from io import IOBase +from types import CodeType, FunctionType, MethodType, GeneratorType, \ + TracebackType, FrameType, ModuleType, BuiltinMethodType +BufferType = memoryview #XXX: unregistered +ClassType = type # no 'old-style' classes +EllipsisType = type(Ellipsis) +#FileType = IOBase +NotImplementedType = type(NotImplemented) +SliceType = slice +TypeType = type # 'new-style' classes #XXX: unregistered +XRangeType = range +from types import MappingProxyType as DictProxyType +from pickle import DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, PickleError, \ + PicklingError, UnpicklingError import __main__ as _main_module import marshal import gc # import zlib import abc from weakref import ReferenceType, ProxyType, CallableProxyType +from collections import OrderedDict +from enum import Enum, EnumMeta from functools import partial from operator import itemgetter, attrgetter -# new in python3.3 -if sys.hexversion < 0x03030000: - FileNotFoundError = IOError -if PY3 and sys.hexversion < 0x03040000: - GENERATOR_FAIL = True -else: GENERATOR_FAIL = False -if PY3: - import importlib.machinery - EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES) -else: - import imp - EXTENSION_SUFFIXES = tuple(suffix - for (suffix, _, s_type) in imp.get_suffixes() - if s_type == imp.C_EXTENSION) +GENERATOR_FAIL = False +import importlib.machinery +EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES) try: import ctypes HAS_CTYPES = True @@ -111,28 +76,15 @@ def _trace(boolean): except ImportError: HAS_CTYPES = False IS_PYPY = False -IS_PYPY2 = IS_PYPY and not PY3 NumpyUfuncType = None NumpyDType = None NumpyArrayType = None try: - if OLDER: - raise AttributeError('find_spec not found') - import importlib if not importlib.machinery.PathFinder().find_spec('numpy'): raise ImportError("No module named 'numpy'") NumpyUfuncType = True NumpyDType = True NumpyArrayType = True -except AttributeError: - try: - import imp - imp.find_module('numpy') - NumpyUfuncType = True - NumpyDType = True - NumpyArrayType = True - except ImportError: - pass except ImportError: pass def __hook__(): @@ -192,40 +144,20 @@ def ndarraysubclassinstance(obj): return False def numpyufunc(obj): return False def numpydtype(obj): return False -# make sure to add these 'hand-built' types to _typemap -if PY3: - CellType = type((lambda x: lambda y: x)(0).__closure__[0]) -else: - CellType = type((lambda x: lambda y: x)(0).func_closure[0]) -# new in python2.5 -if sys.hexversion >= 0x20500f0: - from types import GetSetDescriptorType - if not IS_PYPY: - from types import MemberDescriptorType - else: - # oddly, MemberDescriptorType is GetSetDescriptorType - # while, member_descriptor does exist otherwise... is this a pypy bug? - class _member(object): - __slots__ = ['descriptor'] - MemberDescriptorType = type(_member.descriptor) -if IS_PYPY: - WrapperDescriptorType = MethodType - MethodDescriptorType = FunctionType - ClassMethodDescriptorType = FunctionType -else: - WrapperDescriptorType = type(type.__repr__) - MethodDescriptorType = type(type.__dict__['mro']) - ClassMethodDescriptorType = type(type.__dict__['__prepare__' if PY3 else 'mro']) +from types import GetSetDescriptorType, ClassMethodDescriptorType, \ + WrapperDescriptorType, MethodDescriptorType, MemberDescriptorType, \ + MethodWrapperType #XXX: unused -MethodWrapperType = type([].__repr__) -PartialType = type(partial(int,base=2)) +# make sure to add these 'hand-built' types to _typemap +CellType = type((lambda x: lambda y: x)(0).__closure__[0]) +PartialType = type(partial(int, base=2)) SuperType = type(super(Exception, TypeError())) ItemGetterType = type(itemgetter(0)) AttrGetterType = type(attrgetter('__repr__')) try: from functools import _lru_cache_wrapper as LRUCacheType -except: +except ImportError: LRUCacheType = None if not isinstance(LRUCacheType, type): @@ -251,41 +183,27 @@ def get_file_type(*args, **kwargs): PyBufferedWriterType = get_file_type('wb', buffering=-1, open=_open) except ImportError: PyTextWrapperType = PyBufferedRandomType = PyBufferedReaderType = PyBufferedWriterType = None +from io import BytesIO as StringIO +InputType = OutputType = None +from socket import socket as SocketType +#FIXME: additionally calls ForkingPickler.register several times +from multiprocessing.reduction import _reduce_socket as reduce_socket try: - from cStringIO import StringIO, InputType, OutputType -except ImportError: - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO - InputType = OutputType = None -if not IS_PYPY2: - from socket import socket as SocketType - try: #FIXME: additionally calls ForkingPickler.register several times - from multiprocessing.reduction import _reduce_socket as reduce_socket - except ImportError: - from multiprocessing.reduction import reduce_socket -try: - __IPYTHON__ is True # is ipython + IS_IPYTHON = __IPYTHON__ # is True ExitType = None # IPython.core.autocall.ExitAutocall singletontypes = ['exit', 'quit', 'get_ipython'] except NameError: + IS_IPYTHON = False try: ExitType = type(exit) # apparently 'exit' can be removed except NameError: ExitType = None singletontypes = [] -from collections import OrderedDict +import inspect +import dataclasses +import typing -try: - from enum import Enum, EnumMeta -except: - try: - from enum34 import Enum, EnumMeta - except: - Enum = None - EnumMeta = None +from pickle import GLOBAL -import inspect ### Shims for different versions of Python and dill class Sentinel(object): @@ -407,138 +325,6 @@ def loads(str, ignore=None, **kwds): ### End: Shorthands ### -### Pickle the Interpreter Session -SESSION_IMPORTED_AS_TYPES = (ModuleType, ClassType, TypeType, Exception, - FunctionType, MethodType, BuiltinMethodType) - -def _module_map(): - """get map of imported modules""" - from collections import defaultdict, namedtuple - modmap = namedtuple('Modmap', ['by_name', 'by_id', 'top_level']) - modmap = modmap(defaultdict(list), defaultdict(list), {}) - items = 'items' if PY3 else 'iteritems' - for modname, module in getattr(sys.modules, items)(): - if not isinstance(module, ModuleType): - continue - if '.' not in modname: - modmap.top_level[id(module)] = modname - for objname, modobj in module.__dict__.items(): - modmap.by_name[objname].append((modobj, modname)) - modmap.by_id[id(modobj)].append((modobj, objname, modname)) - return modmap - -def _lookup_module(modmap, name, obj, main_module): - """lookup name or id of obj if module is imported""" - for modobj, modname in modmap.by_name[name]: - if modobj is obj and sys.modules[modname] is not main_module: - return modname, name - if isinstance(obj, SESSION_IMPORTED_AS_TYPES): - for modobj, objname, modname in modmap.by_id[id(obj)]: - if sys.modules[modname] is not main_module: - return modname, objname - return None, None - -def _stash_modules(main_module): - modmap = _module_map() - newmod = ModuleType(main_module.__name__) - - imported = [] - imported_as = [] - imported_top_level = [] # keep separeted for backwards compatibility - original = {} - items = 'items' if PY3 else 'iteritems' - for name, obj in getattr(main_module.__dict__, items)(): - if obj is main_module: - original[name] = newmod # self-reference - continue - - # Avoid incorrectly matching a singleton value in another package (ex.: __doc__). - if any(obj is singleton for singleton in (None, False, True)) or \ - isinstance(obj, ModuleType) and _is_builtin_module(obj): # always saved by ref - original[name] = obj - continue - - source_module, objname = _lookup_module(modmap, name, obj, main_module) - if source_module: - if objname == name: - imported.append((source_module, name)) - else: - imported_as.append((source_module, objname, name)) - else: - try: - imported_top_level.append((modmap.top_level[id(obj)], name)) - except KeyError: - original[name] = obj - - if len(original) < len(main_module.__dict__): - newmod.__dict__.update(original) - newmod.__dill_imported = imported - newmod.__dill_imported_as = imported_as - newmod.__dill_imported_top_level = imported_top_level - return newmod - else: - return main_module - -def _restore_modules(unpickler, main_module): - try: - for modname, name in main_module.__dict__.pop('__dill_imported'): - main_module.__dict__[name] = unpickler.find_class(modname, name) - for modname, objname, name in main_module.__dict__.pop('__dill_imported_as'): - main_module.__dict__[name] = unpickler.find_class(modname, objname) - for modname, name in main_module.__dict__.pop('__dill_imported_top_level'): - main_module.__dict__[name] = __import__(modname) - except KeyError: - pass - -#NOTE: 06/03/15 renamed main_module to main -def dump_session(filename='/tmp/session.pkl', main=None, byref=False, **kwds): - """pickle the current state of __main__ to a file""" - from .settings import settings - protocol = settings['protocol'] - if main is None: main = _main_module - if hasattr(filename, 'write'): - f = filename - else: - f = open(filename, 'wb') - try: - pickler = Pickler(f, protocol, **kwds) - pickler._original_main = main - if byref: - main = _stash_modules(main) - pickler._main = main #FIXME: dill.settings are disabled - pickler._byref = False # disable pickling by name reference - pickler._recurse = False # disable pickling recursion for globals - pickler._session = True # is best indicator of when pickling a session - pickler._first_pass = True - pickler._main_modified = main is not pickler._original_main - pickler.dump(main) - finally: - if f is not filename: # If newly opened file - f.close() - return - -def load_session(filename='/tmp/session.pkl', main=None, **kwds): - """update the __main__ module with the state from the session file""" - if main is None: main = _main_module - if hasattr(filename, 'read'): - f = filename - else: - f = open(filename, 'rb') - try: #FIXME: dill.settings are disabled - unpickler = Unpickler(f, **kwds) - unpickler._main = main - unpickler._session = True - module = unpickler.load() - unpickler._session = False - main.__dict__.update(module.__dict__) - _restore_modules(unpickler, main) - finally: - if f is not filename: # If newly opened file - f.close() - return - -### End: Pickle the Interpreter - class MetaCatchingDict(dict): def get(self, key, default=None): try: @@ -568,21 +354,21 @@ class Pickler(StockPickler): _session = False from .settings import settings - def __init__(self, *args, **kwds): + def __init__(self, file, *args, **kwds): settings = Pickler.settings _byref = kwds.pop('byref', None) #_strictio = kwds.pop('strictio', None) _fmode = kwds.pop('fmode', None) _recurse = kwds.pop('recurse', None) - StockPickler.__init__(self, *args, **kwds) + StockPickler.__init__(self, file, *args, **kwds) self._main = _main_module self._diff_cache = {} self._byref = settings['byref'] if _byref is None else _byref self._strictio = False #_strictio self._fmode = settings['fmode'] if _fmode is None else _fmode self._recurse = settings['recurse'] if _recurse is None else _recurse - from collections import OrderedDict self._postproc = OrderedDict() + self._file = file def dump(self, obj): #NOTE: if settings change, need to update attributes # register if the object is a numpy ufunc @@ -590,10 +376,10 @@ def dump(self, obj): #NOTE: if settings change, need to update attributes if NumpyUfuncType and numpyufunc(obj): @register(type(obj)) def save_numpy_ufunc(pickler, obj): - log.info("Nu: %s" % obj) + logger.trace(pickler, "Nu: %s", obj) name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) StockPickler.save_global(pickler, obj, name=name) - log.info("# Nu") + logger.trace(pickler, "# Nu") return # NOTE: the above 'save' performs like: # import copy_reg @@ -604,9 +390,9 @@ def save_numpy_ufunc(pickler, obj): if NumpyDType and numpydtype(obj): @register(type(obj)) def save_numpy_dtype(pickler, obj): - log.info("Dt: %s" % obj) + logger.trace(pickler, "Dt: %s", obj) pickler.save_reduce(_create_dtypemeta, (obj.type,), obj=obj) - log.info("# Dt") + logger.trace(pickler, "# Dt") return # NOTE: the above 'save' performs like: # import copy_reg @@ -617,21 +403,20 @@ def save_numpy_dtype(pickler, obj): if NumpyArrayType and ndarraysubclassinstance(obj): @register(type(obj)) def save_numpy_array(pickler, obj): - log.info("Nu: (%s, %s)" % (obj.shape,obj.dtype)) + logger.trace(pickler, "Nu: (%s, %s)", obj.shape, obj.dtype) npdict = getattr(obj, '__dict__', None) f, args, state = obj.__reduce__() pickler.save_reduce(_create_array, (f,args,state,npdict), obj=obj) - log.info("# Nu") + logger.trace(pickler, "# Nu") return # end hack if GENERATOR_FAIL and type(obj) == GeneratorType: msg = "Can't pickle %s: attribute lookup builtins.generator failed" % GeneratorType raise PicklingError(msg) - else: - StockPickler.dump(self, obj) - return + logger.trace_setup(self) + StockPickler.dump(self, obj) + dump.__doc__ = StockPickler.dump.__doc__ - pass class Unpickler(StockUnpickler): """python's Unpickler extended to interpreter sessions and more types""" @@ -706,32 +491,38 @@ def use_diff(on=True): if _use_diff and diff is None: try: from . import diff as d - except: + except ImportError: import diff as d diff = d def _create_typemap(): import types - if PY3: - d = dict(list(__builtin__.__dict__.items()) + \ - list(types.__dict__.items())).items() - builtin = 'builtins' - else: - d = types.__dict__.iteritems() - builtin = '__builtin__' + d = dict(list(__builtin__.__dict__.items()) + \ + list(types.__dict__.items())).items() for key, value in d: - if getattr(value, '__module__', None) == builtin \ - and type(value) is type: + if getattr(value, '__module__', None) == 'builtins' \ + and type(value) is type: yield key, value return _reverse_typemap = dict(_create_typemap()) _reverse_typemap.update({ - 'CellType': CellType, - 'MethodWrapperType': MethodWrapperType, 'PartialType': PartialType, 'SuperType': SuperType, 'ItemGetterType': ItemGetterType, 'AttrGetterType': AttrGetterType, +}) +if sys.hexversion < 0x30800a2: + _reverse_typemap.update({ + 'CellType': CellType, + }) + +# "Incidental" implementation specific types. Unpickling these types in another +# implementation of Python (PyPy -> CPython) is not guaranteed to work + +# This dictionary should contain all types that appear in Python implementations +# but are not defined in https://docs.python.org/3/library/types.html#standard-interpreter-types +x=OrderedDict() +_incedental_reverse_typemap = { 'FileType': FileType, 'BufferedRandomType': BufferedRandomType, 'BufferedReaderType': BufferedReaderType, @@ -741,22 +532,54 @@ def _create_typemap(): 'PyBufferedReaderType': PyBufferedReaderType, 'PyBufferedWriterType': PyBufferedWriterType, 'PyTextWrapperType': PyTextWrapperType, +} + +_incedental_reverse_typemap.update({ + "DictKeysType": type({}.keys()), + "DictValuesType": type({}.values()), + "DictItemsType": type({}.items()), + + "OdictKeysType": type(x.keys()), + "OdictValuesType": type(x.values()), + "OdictItemsType": type(x.items()), }) + if ExitType: - _reverse_typemap['ExitType'] = ExitType + _incedental_reverse_typemap['ExitType'] = ExitType if InputType: - _reverse_typemap['InputType'] = InputType - _reverse_typemap['OutputType'] = OutputType -if not IS_PYPY: - _reverse_typemap['WrapperDescriptorType'] = WrapperDescriptorType - _reverse_typemap['MethodDescriptorType'] = MethodDescriptorType - _reverse_typemap['ClassMethodDescriptorType'] = ClassMethodDescriptorType -else: - _reverse_typemap['MemberDescriptorType'] = MemberDescriptorType -if PY3: - _typemap = dict((v, k) for k, v in _reverse_typemap.items()) -else: - _typemap = dict((v, k) for k, v in _reverse_typemap.iteritems()) + _incedental_reverse_typemap['InputType'] = InputType + _incedental_reverse_typemap['OutputType'] = OutputType + +''' +try: + import symtable + _incedental_reverse_typemap["SymtableEntryType"] = type(symtable.symtable("", "string", "exec")._table) +except: #FIXME: fails to pickle + pass + +if sys.hexversion >= 0x30a00a0: + _incedental_reverse_typemap['LineIteratorType'] = type(compile('3', '', 'eval').co_lines()) +''' + +if sys.hexversion >= 0x30b00b0: + from types import GenericAlias + _incedental_reverse_typemap["GenericAliasIteratorType"] = type(iter(GenericAlias(list, (int,)))) + ''' + _incedental_reverse_typemap['PositionsIteratorType'] = type(compile('3', '', 'eval').co_positions()) + ''' + +try: + import winreg + _incedental_reverse_typemap["HKEYType"] = winreg.HKEYType +except ImportError: + pass + +_reverse_typemap.update(_incedental_reverse_typemap) +_incedental_types = set(_incedental_reverse_typemap.values()) + +del x + +_typemap = dict((v, k) for k, v in _reverse_typemap.items()) def _unmarshal(string): return marshal.loads(string) @@ -783,177 +606,157 @@ def _create_function(fcode, fglobals, fname=None, fdefaults=None, # assert id(fglobals) == id(func.__globals__) return func +class match: + """ + Make avaialable a limited structural pattern matching-like syntax for Python < 3.10 + + Patterns can be only tuples (without types) currently. + Inspired by the package pattern-matching-PEP634. + + Usage: + >>> with match(args) as m: + >>> if m.case(('x', 'y')): + >>> # use m.x and m.y + >>> elif m.case(('x', 'y', 'z')): + >>> # use m.x, m.y and m.z + + Equivalent native code for Python >= 3.10: + >>> match args: + >>> case (x, y): + >>> # use x and y + >>> case (x, y, z): + >>> # use x, y and z + """ + def __init__(self, value): + self.value = value + self._fields = None + def __enter__(self): + return self + def __exit__(self, *exc_info): + return False + def case(self, args): # *args, **kwargs): + """just handles tuple patterns""" + if len(self.value) != len(args): # + len(kwargs): + return False + #if not all(isinstance(arg, pat) for arg, pat in zip(self.value[len(args):], kwargs.values())): + # return False + self.args = args # (*args, *kwargs) + return True + @property + def fields(self): + # Only bind names to values if necessary. + if self._fields is None: + self._fields = dict(zip(self.args, self.value)) + return self._fields + def __getattr__(self, item): + return self.fields[item] + +ALL_CODE_PARAMS = [ + # Version New attribute CodeType parameters + ((3,11,'a'), 'co_endlinetable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable endlinetable columntable exceptiontable freevars cellvars'), + ((3,11), 'co_exceptiontable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable exceptiontable freevars cellvars'), + ((3,10), 'co_linetable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno linetable freevars cellvars'), + ((3,8), 'co_posonlyargcount', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno lnotab freevars cellvars'), + ((3,7), 'co_kwonlyargcount', 'argcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno lnotab freevars cellvars'), + ] +for version, new_attr, params in ALL_CODE_PARAMS: + if hasattr(CodeType, new_attr): + CODE_VERSION = version + CODE_PARAMS = params.split() + break +ENCODE_PARAMS = set(CODE_PARAMS).intersection( + ['code', 'lnotab', 'linetable', 'endlinetable', 'columntable', 'exceptiontable']) + def _create_code(*args): - if type(args[0]) is not int: # co_lnotab stored from >= 3.10 - LNOTAB = args[0].encode() if hasattr(args[0], 'encode') else args[0] - args = args[1:] + if not isinstance(args[0], int): # co_lnotab stored from >= 3.10 + LNOTAB, *args = args else: # from < 3.10 (or pre-LNOTAB storage) LNOTAB = b'' - if PY3 and hasattr(args[-3], 'encode'): #NOTE: from PY2 fails (optcode) - args = list(args) - if len(args) == 20: # from 3.11a - # obj.co_argcount, obj.co_posonlyargcount, - # obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - # obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - # obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, - # obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable, - # obj.co_columntable, obj.co_exceptiontable, obj.co_freevars, - # obj.co_cellvars - args[-3] = args[-3].encode() # co_exceptiontable - args[-6] = args[-6].encode() # co_linetable - args[-14] = args[-14].encode() # co_code - if args[-4] is not None: - args[-4] = args[-4].encode() # co_columntable - if args[-5] is not None: - args[-5] = args[-5].encode() # co_endlinetable - elif len(args) == 18: # from 3.11 - # obj.co_argcount, obj.co_posonlyargcount, - # obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - # obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - # obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, - # obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, - # obj.co_freevars, obj.co_cellvars - args[-3] = args[-3].encode() # co_exceptiontable - args[-4] = args[-4].encode() # co_linetable - args[-12] = args[-12].encode() # co_code - else: # from 3.10 - # obj.co_argcount, obj.co_posonlyargcount, - # obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - # obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - # obj.co_varnames, obj.co_filename, obj.co_name, - # obj.co_firstlineno, obj.co_linetable, obj.co_freevars, - # obj.co_cellvars - args[-3] = args[-3].encode() # co_linetable (or co_lnotab) - args[-10] = args[-10].encode() # co_code - args = tuple(args) - if hasattr(CodeType, 'co_endlinetable'): # python 3.11a - # obj.co_argcount, obj.co_posonlyargcount, - # obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - # obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - # obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, - # obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable, - # obj.co_columntable, obj.co_exceptiontable, obj.co_freevars, - # obj.co_cellvars - if len(args) == 20: return CodeType(*args) - elif len(args) == 18: # from 3.11 - argz = (None, None) - argz = args[:-3] + argz + args[-3:] - return CodeType(*argz) - elif len(args) == 16: # from 3.10 or from 3.8 - if LNOTAB: # here and above uses stored co_linetable - argz = (None, None, b'') - argz = args[:-4] + args[-5:-4] + args[-4:-2] + argz + args[-2:] - else: # here and below drops stored co_lnotab - argz = (LNOTAB, None, None, b'') - argz = args[:-4] + args[-5:-4] + args[-4:-3] + argz + args[-2:] - return CodeType(*argz) - elif len(args) == 15: # from 3.7 - argz = (LNOTAB, None, None, b'') - argz = args[1:-4] + args[-5:-4] + args[-4:-3] + argz + args[-2:] - return CodeType(args[0], 0, *argz) - argz = (LNOTAB, None, None, b'') # from 2.7 - argz = args[1:-4] + args[-5:-4] + args[-4:-3] + argz + args[-2:] - return CodeType(args[0], 0, 0, *argz) - elif hasattr(CodeType, 'co_exceptiontable'): # python 3.11 - # obj.co_argcount, obj.co_posonlyargcount, - # obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - # obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - # obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, - # obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, - # obj.co_freevars, obj.co_cellvars - if len(args) == 20: return CodeType(*(args[:15] + args[17:])) - elif len(args) == 18: return CodeType(*args) - elif len(args) == 16: # from 3.10 or from 3.8 - if LNOTAB: # here and above uses stored co_linetable - argz = (b'',) - argz = args[:-4] + args[-5:-4] + args[-4:-2] + argz + args[-2:] - else: # here and below drops stored co_lnotab - argz = (LNOTAB, b'') - argz = args[:-4] + args[-5:-4] + args[-4:-3] + argz + args[-2:] - return CodeType(*argz) - elif len(args) == 15: # from 3.7 - argz = (LNOTAB, b'') - argz = args[1:-4] + args[-5:-4] + args[-4:-3] + argz + args[-2:] - return CodeType(args[0], 0, *argz) - argz = (LNOTAB, b'') # from 2.7 - argz = args[1:-4] + args[-5:-4] + args[-4:-3] + argz + args[-2:] - return CodeType(args[0], 0, 0, *argz) - elif hasattr(CodeType, 'co_linetable'): # python 3.10 - # obj.co_argcount, obj.co_posonlyargcount, - # obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - # obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - # obj.co_varnames, obj.co_filename, obj.co_name, - # obj.co_firstlineno, obj.co_linetable, obj.co_freevars, - # obj.co_cellvars - if len(args) == 20: # from 3.11a - return CodeType(*(args[:12] + args[13:15] + args[18:])) - elif len(args) == 18: # from 3.11 - return CodeType(*(args[:12] + args[13:15] + args[16:])) - elif len(args) == 16: # from 3.10 or from 3.8 - if not LNOTAB: # here and below drops stored co_lnotab - args = args[:-3] + (LNOTAB,) + args[-2:] - return CodeType(*args) - elif len(args) == 15: # from 3.7 - argz = args[1:-3] + (LNOTAB,) + args[-2:] - return CodeType(args[0], 0, *argz) - argz = args[1:-3] + (LNOTAB,) + args[-2:] - return CodeType(args[0], 0, 0, *argz) # from 2.7 - elif hasattr(CodeType, 'co_posonlyargcount'): # python 3.8 - # obj.co_argcount, obj.co_posonlyargcount, - # obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - # obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - # obj.co_varnames, obj.co_filename, obj.co_name, - # obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, - # obj.co_cellvars - if len(args) == 20: # from 3.11a - args = args[:12] + args[13:14] + (LNOTAB,) + args[18:] - return CodeType(*args) - elif len(args) == 18: # from 3.11 - args = args[:12] + args[13:14] + (LNOTAB,) + args[16:] - return CodeType(*args) - elif len(args) == 16: # from 3.10 or from 3.8 - if LNOTAB: # here and above uses stored LNOTAB - args = args[:-3] + (LNOTAB,) + args[-2:] - return CodeType(*args) - elif len(args) == 15: return CodeType(args[0], 0, *args[1:]) # from 3.7 - return CodeType(args[0], 0, 0, *args[1:]) # from 2.7 - elif hasattr(CodeType, 'co_kwonlyargcount'): # python 3.7 - # obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, - # obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, - # obj.co_names, obj.co_varnames, obj.co_filename, - # obj.co_name, obj.co_firstlineno, obj.co_lnotab, - # obj.co_freevars, obj.co_cellvars - if len(args) == 20: # from 3.11a - args = args[:1] + args[2:12] + args[13:14] + (LNOTAB,) + args[18:] - return CodeType(*args) - elif len(args) == 18: # from 3.11 - args = args[:1] + args[2:12] + args[13:14] + (LNOTAB,) + args[16:] - return CodeType(*args) - elif len(args) == 16: # from 3.10 or from 3.8 - if LNOTAB: # here and above uses stored LNOTAB - argz = args[2:-3] + (LNOTAB,) + args[-2:] + + with match(args) as m: + # Python 3.11/3.12a (18 members) + if m.case(( + 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6] + 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', 'firstlineno', # args[6:14] + 'linetable', 'exceptiontable', 'freevars', 'cellvars' # args[14:] + )): + if CODE_VERSION == (3,11): + return CodeType( + *args[:6], + args[6].encode() if hasattr(args[6], 'encode') else args[6], # code + *args[7:14], + args[14].encode() if hasattr(args[14], 'encode') else args[14], # linetable + args[15].encode() if hasattr(args[15], 'encode') else args[15], # exceptiontable + args[16], + args[17], + ) + fields = m.fields + # Python 3.10 or 3.8/3.9 (16 members) + elif m.case(( + 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6] + 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'firstlineno', # args[6:13] + 'LNOTAB_OR_LINETABLE', 'freevars', 'cellvars' # args[13:] + )): + if CODE_VERSION == (3,10) or CODE_VERSION == (3,8): + return CodeType( + *args[:6], + args[6].encode() if hasattr(args[6], 'encode') else args[6], # code + *args[7:13], + args[13].encode() if hasattr(args[13], 'encode') else args[13], # lnotab/linetable + args[14], + args[15], + ) + fields = m.fields + if CODE_VERSION >= (3,10): + fields['linetable'] = m.LNOTAB_OR_LINETABLE else: - argz = args[2:] - return CodeType(args[0], *argz) - elif len(args) == 15: return CodeType(*args) - return CodeType(args[0], 0, *args[1:]) # from 2.7 - # obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, - # obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, - # obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, - # obj.co_freevars, obj.co_cellvars - if len(args) == 20: # from 3.11a - args = args[:1] + args[3:12] + args[13:14] + (LNOTAB,) + args[18:] - return CodeType(*args) - elif len(args) == 18: # from 3.11 - args = args[:1] + args[3:12] + args[13:14] + (LNOTAB,) + args[16:] - return CodeType(*args) - elif len(args) == 16: # from 3.10 or from 3.8 - if LNOTAB: # here and above uses stored LNOTAB - argz = args[3:-3] + (LNOTAB,) + args[-2:] + fields['lnotab'] = LNOTAB if LNOTAB else m.LNOTAB_OR_LINETABLE + # Python 3.7 (15 args) + elif m.case(( + 'argcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:5] + 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'firstlineno', # args[5:12] + 'lnotab', 'freevars', 'cellvars' # args[12:] + )): + if CODE_VERSION == (3,7): + return CodeType( + *args[:5], + args[5].encode() if hasattr(args[5], 'encode') else args[5], # code + *args[6:12], + args[12].encode() if hasattr(args[12], 'encode') else args[12], # lnotab + args[13], + args[14], + ) + fields = m.fields + # Python 3.11a (20 members) + elif m.case(( + 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6] + 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', 'firstlineno', # args[6:14] + 'linetable', 'endlinetable', 'columntable', 'exceptiontable', 'freevars', 'cellvars' # args[14:] + )): + if CODE_VERSION == (3,11,'a'): + return CodeType( + *args[:6], + args[6].encode() if hasattr(args[6], 'encode') else args[6], # code + *args[7:14], + *(a.encode() if hasattr(a, 'encode') else a for a in args[14:18]), # linetable-exceptiontable + args[18], + args[19], + ) + fields = m.fields else: - argz = args[3:] - return CodeType(args[0], *argz) - elif len(args) == 15: return CodeType(args[0], *args[2:]) # from 3.7 + raise UnpicklingError("pattern match for code object failed") + + # The args format doesn't match this version. + fields.setdefault('posonlyargcount', 0) # from python <= 3.7 + fields.setdefault('lnotab', LNOTAB) # from python >= 3.10 + fields.setdefault('linetable', b'') # from python <= 3.9 + fields.setdefault('qualname', fields['name']) # from python <= 3.10 + fields.setdefault('exceptiontable', b'') # from python <= 3.10 + fields.setdefault('endlinetable', None) # from python != 3.11a + fields.setdefault('columntable', None) # from python != 3.11a + + args = (fields[k].encode() if k in ENCODE_PARAMS and hasattr(fields[k], 'encode') else fields[k] + for k in CODE_PARAMS) return CodeType(*args) def _create_ftype(ftypeobj, func, args, kwds): @@ -963,6 +766,13 @@ def _create_ftype(ftypeobj, func, args, kwds): args = () return ftypeobj(func, *args, **kwds) +def _create_typing_tuple(argz, *args): #NOTE: workaround python/cpython#94245 + if not argz: + return typing.Tuple[()].copy_with(()) + if argz == ((),): + return typing.Tuple[()] + return typing.Tuple[argz] + def _create_lock(locked, *args): #XXX: ignores 'blocking' from threading import Lock lock = Lock() @@ -994,12 +804,9 @@ def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdat import tempfile f = tempfile.TemporaryFile(mode) else: - # treat x mode as w mode - if "x" in mode and sys.hexversion < 0x03030000: - raise ValueError("invalid mode: '%s'" % mode) try: exists = os.path.exists(name) - except: + except Exception: exists = False if not exists: if strictio: @@ -1028,6 +835,7 @@ def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdat elif name == '': # file did not exist import tempfile f = tempfile.TemporaryFile(mode) + # treat x mode as w mode elif fmode == CONTENTS_FMODE \ and ("w" in mode or "x" in mode): # stop truncation when opening @@ -1038,28 +846,9 @@ def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdat flags |= os.O_WRONLY f = os.fdopen(os.open(name, flags), mode) # set name to the correct value - if PY3: - r = getattr(f, "buffer", f) - r = getattr(r, "raw", r) - r.name = name - else: - if not HAS_CTYPES: - raise ImportError("No module named 'ctypes'") - class FILE(ctypes.Structure): - _fields_ = [("refcount", ctypes.c_long), - ("type_obj", ctypes.py_object), - ("file_pointer", ctypes.c_voidp), - ("name", ctypes.py_object)] - - class PyObject(ctypes.Structure): - _fields_ = [ - ("ob_refcnt", ctypes.c_int), - ("ob_type", ctypes.py_object) - ] - #FIXME: CONTENTS_FMODE fails for pypy due to issue #1233 - # https://bitbucket.org/pypy/pypy/issues/1233 - ctypes.cast(id(f), ctypes.POINTER(FILE)).contents.name = name - ctypes.cast(id(name), ctypes.POINTER(PyObject)).contents.ob_refcnt += 1 + r = getattr(f, "buffer", f) + r = getattr(r, "raw", r) + r.name = name assert f.name == name else: f = open(name, mode) @@ -1120,7 +909,7 @@ def __ror__(self, a): # mapping referenced by the proxy. It may work for other implementations, # but is not guaranteed. MAPPING_PROXY_TRICK = __d is (DictProxyType(__d) | _dictproxy_helper_instance) -except: +except Exception: MAPPING_PROXY_TRICK = False del __d @@ -1131,26 +920,15 @@ def __ror__(self, a): _CELL_REF = None _CELL_EMPTY = Sentinel('_CELL_EMPTY') -if PY3: - def _create_cell(contents=None): - if contents is not _CELL_EMPTY: - value = contents - return (lambda: value).__closure__[0] - -else: - def _create_cell(contents=None): - if contents is not _CELL_EMPTY: - value = contents - return (lambda: value).func_closure[0] - +def _create_cell(contents=None): + if contents is not _CELL_EMPTY: + value = contents + return (lambda: value).__closure__[0] def _create_weakref(obj, *args): from weakref import ref if obj is None: # it's dead - if PY3: - from collections import UserDict - else: - from UserDict import UserDict + from collections import UserDict return ref(UserDict(), *args) return ref(obj, *args) @@ -1158,10 +936,7 @@ def _create_weakproxy(obj, callable=False, *args): from weakref import proxy if obj is None: # it's dead if callable: return proxy(lambda x:x, *args) - if PY3: - from collections import UserDict - else: - from UserDict import UserDict + from collections import UserDict return proxy(UserDict(), *args) return proxy(obj, *args) @@ -1182,37 +957,57 @@ def _create_dtypemeta(scalar_type): return NumpyDType return type(NumpyDType(scalar_type)) -if OLD37: - def _create_namedtuple(name, fieldnames, modulename, defaults=None): - class_ = _import_module(modulename + '.' + name, safe=True) - if class_ is not None: - return class_ - import collections - t = collections.namedtuple(name, fieldnames) - t.__module__ = modulename - return t -else: - def _create_namedtuple(name, fieldnames, modulename, defaults=None): - class_ = _import_module(modulename + '.' + name, safe=True) - if class_ is not None: - return class_ - import collections - t = collections.namedtuple(name, fieldnames, defaults=defaults, module=modulename) - return t +def _create_namedtuple(name, fieldnames, modulename, defaults=None): + class_ = _import_module(modulename + '.' + name, safe=True) + if class_ is not None: + return class_ + import collections + t = collections.namedtuple(name, fieldnames, defaults=defaults, module=modulename) + return t + +def _create_capsule(pointer, name, context, destructor): + attr_found = False + try: + # based on https://github.com/python/cpython/blob/f4095e53ab708d95e019c909d5928502775ba68f/Objects/capsule.c#L209-L231 + uname = name.decode('utf8') + for i in range(1, uname.count('.')+1): + names = uname.rsplit('.', i) + try: + module = __import__(names[0]) + except ImportError: + pass + obj = module + for attr in names[1:]: + obj = getattr(obj, attr) + capsule = obj + attr_found = True + break + except Exception: + pass + + if attr_found: + if _PyCapsule_IsValid(capsule, name): + return capsule + raise UnpicklingError("%s object exists at %s but a PyCapsule object was expected." % (type(capsule), name)) + else: + warnings.warn('Creating a new PyCapsule %s for a C data structure that may not be present in memory. Segmentation faults or other memory errors are possible.' % (name,), UnpicklingWarning) + capsule = _PyCapsule_New(pointer, name, destructor) + _PyCapsule_SetContext(capsule, context) + return capsule def _getattr(objclass, name, repr_str): # hack to grab the reference directly try: #XXX: works only for __builtin__ ? attr = repr_str.split("'")[3] return eval(attr+'.__dict__["'+name+'"]') - except: + except Exception: try: attr = objclass.__dict__ if type(attr) is DictProxyType: attr = attr[name] else: attr = getattr(objclass,name) - except: + except (AttributeError, KeyError): attr = getattr(objclass,name) return attr @@ -1220,24 +1015,18 @@ def _get_attr(self, name): # stop recursive pickling return getattr(self, name, None) or getattr(__builtin__, name) -def _dict_from_dictproxy(dictproxy): - # Deprecated. Use _get_typedict_type instead. - _dict = dictproxy.copy() # convert dictproxy to dict - _dict.pop('__dict__', None) - _dict.pop('__weakref__', None) - _dict.pop('__prepare__', None) - return _dict - def _import_module(import_name, safe=False): try: - if '.' in import_name: + if import_name.startswith('__runtime__.'): + return sys.modules[import_name] + elif '.' in import_name: items = import_name.split('.') module = '.'.join(items[:-1]) obj = items[-1] else: return __import__(import_name) return getattr(__import__(module, None, None, [obj]), obj) - except (ImportError, AttributeError): + except (ImportError, AttributeError, KeyError): if safe: return None raise @@ -1266,7 +1055,7 @@ def _locate_function(obj, pickler=None): try: found, _ = _getattribute(module, obj.__qualname__) return found is obj - except: + except AttributeError: return False else: found = _import_module(module_name + '.' + obj.__name__, safe=True) @@ -1321,16 +1110,13 @@ def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO else: pickler.save_reduce(*reduction) # pop None created by calling preprocessing step off stack - if PY3: - pickler.write(bytes('0', 'UTF-8')) - else: - pickler.write('0') + pickler.write(bytes('0', 'UTF-8')) #@register(CodeType) #def save_code(pickler, obj): -# log.info("Co: %s" % obj) +# logger.trace(pickler, "Co: %s", obj) # pickler.save_reduce(_unmarshal, (marshal.dumps(obj),), obj=obj) -# log.info("# Co") +# logger.trace(pickler, "# Co") # return # The following function is based on 'save_codeobject' from 'cloudpickle' @@ -1339,101 +1125,88 @@ def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE @register(CodeType) def save_code(pickler, obj): - log.info("Co: %s" % obj) - if PY3: - if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args) - args = ( - obj.co_lnotab, # for < python 3.10 [not counted in args] - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, - obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable, - obj.co_columntable, obj.co_exceptiontable, obj.co_freevars, - obj.co_cellvars - ) - elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args) - args = ( - obj.co_lnotab, # for < python 3.10 [not counted in args] - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, - obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, - obj.co_freevars, obj.co_cellvars - ) - elif hasattr(obj, "co_linetable"): # python 3.10 (16 args) - args = ( - obj.co_lnotab, # for < python 3.10 [not counted in args] - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_linetable, obj.co_freevars, - obj.co_cellvars - ) - elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args) - args = ( - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, - obj.co_cellvars - ) - else: # python 3.7 (15 args) - args = ( - obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, - obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, - obj.co_names, obj.co_varnames, obj.co_filename, - obj.co_name, obj.co_firstlineno, obj.co_lnotab, - obj.co_freevars, obj.co_cellvars - ) - else: # python 2.7 (14 args) + logger.trace(pickler, "Co: %s", obj) + if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, + obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable, + obj.co_columntable, obj.co_exceptiontable, obj.co_freevars, + obj.co_cellvars + ) + elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args) args = ( - obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, - obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, - obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, + obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, obj.co_freevars, obj.co_cellvars - ) + ) + elif hasattr(obj, "co_linetable"): # python 3.10 (16 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_linetable, obj.co_freevars, + obj.co_cellvars + ) + elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args) + args = ( + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, + obj.co_cellvars + ) + else: # python 3.7 (15 args) + args = ( + obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, + obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, + obj.co_names, obj.co_varnames, obj.co_filename, + obj.co_name, obj.co_firstlineno, obj.co_lnotab, + obj.co_freevars, obj.co_cellvars + ) pickler.save_reduce(_create_code, args, obj=obj) - log.info("# Co") + logger.trace(pickler, "# Co") return +def _repr_dict(obj): + """make a short string representation of a dictionary""" + return "<%s object at %#012x>" % (type(obj).__name__, id(obj)) + @register(dict) def save_module_dict(pickler, obj): if is_dill(pickler, child=False) and obj == pickler._main.__dict__ and \ not (pickler._session and pickler._first_pass): - log.info("D1: " % (obj,)) + logger.trace(pickler, "Dkvi: <%s>", obj) mapping = obj.mapping | _dictproxy_helper_instance pickler.save_reduce(func, (mapping,), obj=obj) - log.info("# Dkvi") + logger.trace(pickler, "# Dkvi") return _save_dict_view return [ (funcname, save_dict_view_for_function(getattr(dicttype, funcname))) @@ -1458,21 +1231,21 @@ def _save_dict_view(pickler, obj): # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE def save_dict_view(dicttype): def save_dict_keys(pickler, obj): - log.info("Dk: <%s>" % (obj,)) + logger.trace(pickler, "Dk: <%s>", obj) dict_constructor = _shims.Reduce(dicttype.fromkeys, (list(obj),)) pickler.save_reduce(dicttype.keys, (dict_constructor,), obj=obj) - log.info("# Dk") + logger.trace(pickler, "# Dk") def save_dict_values(pickler, obj): - log.info("Dv: <%s>" % (obj,)) + logger.trace(pickler, "Dv: <%s>", obj) dict_constructor = _shims.Reduce(dicttype, (enumerate(obj),)) pickler.save_reduce(dicttype.values, (dict_constructor,), obj=obj) - log.info("# Dv") + logger.trace(pickler, "# Dv") def save_dict_items(pickler, obj): - log.info("Di: <%s>" % (obj,)) + logger.trace(pickler, "Di: <%s>", obj) pickler.save_reduce(dicttype.items, (dicttype(obj),), obj=obj) - log.info("# Di") + logger.trace(pickler, "# Di") return ( ('keys', save_dict_keys), @@ -1495,62 +1268,58 @@ def save_dict_items(pickler, obj): @register(ClassType) def save_classobj(pickler, obj): #FIXME: enable pickler._byref if not _locate_function(obj, pickler): - log.info("C1: %s" % obj) + logger.trace(pickler, "C1: %s", obj) pickler.save_reduce(ClassType, (obj.__name__, obj.__bases__, obj.__dict__), obj=obj) #XXX: or obj.__dict__.copy()), obj=obj) ? - log.info("# C1") + logger.trace(pickler, "# C1") else: - log.info("C2: %s" % obj) + logger.trace(pickler, "C2: %s", obj) name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) StockPickler.save_global(pickler, obj, name=name) - log.info("# C2") + logger.trace(pickler, "# C2") + return + +@register(typing._GenericAlias) +def save_generic_alias(pickler, obj): + args = obj.__args__ + if type(obj.__reduce__()) is str: + logger.trace(pickler, "Ga0: %s", obj) + StockPickler.save_global(pickler, obj, name=obj.__reduce__()) + logger.trace(pickler, "# Ga0") + elif obj.__origin__ is tuple and (not args or args == ((),)): + logger.trace(pickler, "Ga1: %s", obj) + pickler.save_reduce(_create_typing_tuple, (args,), obj=obj) + logger.trace(pickler, "# Ga1") + else: + logger.trace(pickler, "Ga2: %s", obj) + StockPickler.save_reduce(pickler, *obj.__reduce__(), obj=obj) + logger.trace(pickler, "# Ga2") return @register(LockType) def save_lock(pickler, obj): - log.info("Lo: %s" % obj) + logger.trace(pickler, "Lo: %s", obj) pickler.save_reduce(_create_lock, (obj.locked(),), obj=obj) - log.info("# Lo") + logger.trace(pickler, "# Lo") return @register(RLockType) def save_rlock(pickler, obj): - log.info("RL: %s" % obj) + logger.trace(pickler, "RL: %s", obj) r = obj.__repr__() # don't use _release_save as it unlocks the lock count = int(r.split('count=')[1].split()[0].rstrip('>')) - owner = int(r.split('owner=')[1].split()[0]) if PY3 else getattr(obj, '_RLock__owner') + owner = int(r.split('owner=')[1].split()[0]) pickler.save_reduce(_create_rlock, (count,owner,), obj=obj) - log.info("# RL") + logger.trace(pickler, "# RL") return -if not IS_PYPY2: - #@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL - def save_socket(pickler, obj): - log.info("So: %s" % obj) - pickler.save_reduce(*reduce_socket(obj)) - log.info("# So") - return - -if sys.hexversion <= 0x3050000: - @register(ItemGetterType) - def save_itemgetter(pickler, obj): - log.info("Ig: %s" % obj) - helper = _itemgetter_helper() - obj(helper) - pickler.save_reduce(type(obj), tuple(helper.items), obj=obj) - log.info("# Ig") - return - - @register(AttrGetterType) - def save_attrgetter(pickler, obj): - log.info("Ag: %s" % obj) - attrs = [] - helper = _attrgetter_helper(attrs) - obj(helper) - pickler.save_reduce(type(obj), tuple(attrs), obj=obj) - log.info("# Ag") - return +#@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL +def save_socket(pickler, obj): + logger.trace(pickler, "So: %s", obj) + pickler.save_reduce(*reduce_socket(obj)) + logger.trace(pickler, "# So") + return def _save_file(pickler, obj, open_): if obj.closed: @@ -1585,9 +1354,9 @@ def _save_file(pickler, obj, open_): @register(BufferedWriterType) @register(TextWrapperType) def save_file(pickler, obj): - log.info("Fi: %s" % obj) + logger.trace(pickler, "Fi: %s", obj) f = _save_file(pickler, obj, open) - log.info("# Fi") + logger.trace(pickler, "# Fi") return f if PyTextWrapperType: @@ -1596,9 +1365,9 @@ def save_file(pickler, obj): @register(PyBufferedWriterType) @register(PyTextWrapperType) def save_file(pickler, obj): - log.info("Fi: %s" % obj) + logger.trace(pickler, "Fi: %s", obj) f = _save_file(pickler, obj, _open) - log.info("# Fi") + logger.trace(pickler, "# Fi") return f # The following two functions are based on 'saveCStringIoInput' @@ -1608,42 +1377,33 @@ def save_file(pickler, obj): if InputType: @register(InputType) def save_stringi(pickler, obj): - log.info("Io: %s" % obj) + logger.trace(pickler, "Io: %s", obj) if obj.closed: value = ''; position = 0 else: value = obj.getvalue(); position = obj.tell() pickler.save_reduce(_create_stringi, (value, position, \ obj.closed), obj=obj) - log.info("# Io") + logger.trace(pickler, "# Io") return @register(OutputType) def save_stringo(pickler, obj): - log.info("Io: %s" % obj) + logger.trace(pickler, "Io: %s", obj) if obj.closed: value = ''; position = 0 else: value = obj.getvalue(); position = obj.tell() pickler.save_reduce(_create_stringo, (value, position, \ obj.closed), obj=obj) - log.info("# Io") - return - -if 0x2050000 <= sys.hexversion < 0x3010000: - @register(PartialType) - def save_functor(pickler, obj): - log.info("Fu: %s" % obj) - pickler.save_reduce(_create_ftype, (type(obj), obj.func, obj.args, - obj.keywords), obj=obj) - log.info("# Fu") + logger.trace(pickler, "# Io") return if LRUCacheType is not None: from functools import lru_cache @register(LRUCacheType) def save_lru_cache(pickler, obj): - log.info("LRU: %s" % obj) + logger.trace(pickler, "LRU: %s", obj) if OLD39: kwargs = obj.cache_info() args = (kwargs.maxsize,) @@ -1655,103 +1415,69 @@ def save_lru_cache(pickler, obj): else: wrapper = lru_cache pickler.save_reduce(wrapper, (obj.__wrapped__,), obj=obj) - log.info("# LRU") + logger.trace(pickler, "# LRU") return @register(SuperType) def save_super(pickler, obj): - log.info("Su: %s" % obj) + logger.trace(pickler, "Su: %s", obj) pickler.save_reduce(super, (obj.__thisclass__, obj.__self__), obj=obj) - log.info("# Su") + logger.trace(pickler, "# Su") return -if OLDER or not PY3: - @register(BuiltinMethodType) - def save_builtin_method(pickler, obj): - if obj.__self__ is not None: - if obj.__self__ is __builtin__: - module = 'builtins' if PY3 else '__builtin__' - _t = "B1" - log.info("%s: %s" % (_t, obj)) - else: - module = obj.__self__ - _t = "B3" - log.info("%s: %s" % (_t, obj)) - if is_dill(pickler, child=True): - _recurse = pickler._recurse - pickler._recurse = False - pickler.save_reduce(_get_attr, (module, obj.__name__), obj=obj) - if is_dill(pickler, child=True): - pickler._recurse = _recurse - log.info("# %s" % _t) - else: - log.info("B2: %s" % obj) - name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) - StockPickler.save_global(pickler, obj, name=name) - log.info("# B2") - return - - @register(MethodType) #FIXME: fails for 'hidden' or 'name-mangled' classes - def save_instancemethod0(pickler, obj):# example: cStringIO.StringI - log.info("Me: %s" % obj) #XXX: obj.__dict__ handled elsewhere? - if PY3: - pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) - else: - pickler.save_reduce(MethodType, (obj.im_func, obj.im_self, - obj.im_class), obj=obj) - log.info("# Me") - return - -if sys.hexversion >= 0x20500f0: - if not IS_PYPY: - @register(MemberDescriptorType) - @register(GetSetDescriptorType) - @register(MethodDescriptorType) - @register(WrapperDescriptorType) - @register(ClassMethodDescriptorType) - def save_wrapper_descriptor(pickler, obj): - log.info("Wr: %s" % obj) - pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, - obj.__repr__()), obj=obj) - log.info("# Wr") - return - else: - @register(MemberDescriptorType) - @register(GetSetDescriptorType) - def save_wrapper_descriptor(pickler, obj): - log.info("Wr: %s" % obj) - pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, - obj.__repr__()), obj=obj) - log.info("# Wr") +if IS_PYPY: + @register(MethodType) + def save_instancemethod0(pickler, obj): + code = getattr(obj.__func__, '__code__', None) + if code is not None and type(code) is not CodeType \ + and getattr(obj.__self__, obj.__name__) == obj: + # Some PyPy builtin functions have no module name + logger.trace(pickler, "Me2: %s", obj) + # TODO: verify that this works for all PyPy builtin methods + pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) + logger.trace(pickler, "# Me2") return - @register(MethodWrapperType) - def save_instancemethod(pickler, obj): - log.info("Mw: %s" % obj) - if IS_PYPY2 and obj.__self__ is None and obj.im_class: - # Can be a class method in PYPY2 if __self__ is none - pickler.save_reduce(getattr, (obj.im_class, obj.__name__), obj=obj) - return - pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) - log.info("# Mw") + logger.trace(pickler, "Me1: %s", obj) + pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) + logger.trace(pickler, "# Me1") + return +else: + @register(MethodType) + def save_instancemethod0(pickler, obj): + logger.trace(pickler, "Me1: %s", obj) + pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) + logger.trace(pickler, "# Me1") return -elif not IS_PYPY: +if not IS_PYPY: + @register(MemberDescriptorType) + @register(GetSetDescriptorType) @register(MethodDescriptorType) @register(WrapperDescriptorType) + @register(ClassMethodDescriptorType) + def save_wrapper_descriptor(pickler, obj): + logger.trace(pickler, "Wr: %s", obj) + pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, + obj.__repr__()), obj=obj) + logger.trace(pickler, "# Wr") + return +else: + @register(MemberDescriptorType) + @register(GetSetDescriptorType) def save_wrapper_descriptor(pickler, obj): - log.info("Wr: %s" % obj) + logger.trace(pickler, "Wr: %s", obj) pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, obj.__repr__()), obj=obj) - log.info("# Wr") + logger.trace(pickler, "# Wr") return @register(CellType) def save_cell(pickler, obj): try: f = obj.cell_contents - except: - log.info("Ce3: %s" % obj) + except ValueError: # cell is empty + logger.trace(pickler, "Ce3: %s", obj) # _shims._CELL_EMPTY is defined in _shims.py to support PyPy 2.7. # It unpickles to a sentinel object _dill._CELL_EMPTY, also created in # _shims.py. This object is not present in Python 3 because the cell's @@ -1766,11 +1492,8 @@ def save_cell(pickler, obj): # The result of this function call will be None pickler.save_reduce(_shims._delattr, (obj, 'cell_contents')) # pop None created by calling _delattr off stack - if PY3: - pickler.write(bytes('0', 'UTF-8')) - else: - pickler.write('0') - log.info("# Ce3") + pickler.write(bytes('0', 'UTF-8')) + logger.trace(pickler, "# Ce3") return if is_dill(pickler, child=True): if id(f) in pickler._postproc: @@ -1781,66 +1504,49 @@ def save_cell(pickler, obj): # value as late as possible to prevent cycle. postproc = next(iter(pickler._postproc.values()), None) if postproc is not None: - log.info("Ce2: %s" % obj) + logger.trace(pickler, "Ce2: %s", obj) # _CELL_REF is defined in _shims.py to support older versions of # dill. When breaking changes are made to dill, (_CELL_REF,) can # be replaced by () pickler.save_reduce(_create_cell, (_CELL_REF,), obj=obj) postproc.append((_shims._setattr, (obj, 'cell_contents', f))) - log.info("# Ce2") + logger.trace(pickler, "# Ce2") return - log.info("Ce1: %s" % obj) + logger.trace(pickler, "Ce1: %s", obj) pickler.save_reduce(_create_cell, (f,), obj=obj) - log.info("# Ce1") + logger.trace(pickler, "# Ce1") return if MAPPING_PROXY_TRICK: @register(DictProxyType) def save_dictproxy(pickler, obj): - log.info("Mp: %s" % obj) + logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj mapping = obj | _dictproxy_helper_instance pickler.save_reduce(DictProxyType, (mapping,), obj=obj) - log.info("# Mp") + logger.trace(pickler, "# Mp") + return +else: + @register(DictProxyType) + def save_dictproxy(pickler, obj): + logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj + pickler.save_reduce(DictProxyType, (obj.copy(),), obj=obj) + logger.trace(pickler, "# Mp") return -elif not IS_PYPY: - if not OLD33: - @register(DictProxyType) - def save_dictproxy(pickler, obj): - log.info("Mp: %s" % obj) - pickler.save_reduce(DictProxyType, (obj.copy(),), obj=obj) - log.info("# Mp") - return - else: - # The following function is based on 'saveDictProxy' from spickle - # Copyright (c) 2011 by science+computing ag - # License: http://www.apache.org/licenses/LICENSE-2.0 - @register(DictProxyType) - def save_dictproxy(pickler, obj): - log.info("Dp: %s" % obj) - attr = obj.get('__dict__') - #pickler.save_reduce(_create_dictproxy, (attr,'nested'), obj=obj) - if type(attr) == GetSetDescriptorType and attr.__name__ == "__dict__" \ - and getattr(attr.__objclass__, "__dict__", None) == obj: - pickler.save_reduce(getattr, (attr.__objclass__,"__dict__"),obj=obj) - log.info("# Dp") - return - # all bad below... so throw ReferenceError or TypeError - raise ReferenceError("%s does not reference a class __dict__" % obj) @register(SliceType) def save_slice(pickler, obj): - log.info("Sl: %s" % obj) + logger.trace(pickler, "Sl: %s", obj) pickler.save_reduce(slice, (obj.start, obj.stop, obj.step), obj=obj) - log.info("# Sl") + logger.trace(pickler, "# Sl") return @register(XRangeType) @register(EllipsisType) @register(NotImplementedType) def save_singleton(pickler, obj): - log.info("Si: %s" % obj) + logger.trace(pickler, "Si: %s", obj) pickler.save_reduce(_eval_repr, (obj.__repr__(),), obj=obj) - log.info("# Si") + logger.trace(pickler, "# Si") return def _proxy_helper(obj): # a dead proxy returns a reference to None @@ -1870,10 +1576,7 @@ def _locate_object(address, module=None): for obj in special: if address == id(obj): return obj if module: - if PY3: - objects = iter(module.__dict__.values()) - else: - objects = module.__dict__.itervalues() + objects = iter(module.__dict__.values()) else: objects = iter(gc.get_objects()) for obj in objects: if address == id(obj): return obj @@ -1886,10 +1589,10 @@ def _locate_object(address, module=None): @register(ReferenceType) def save_weakref(pickler, obj): refobj = obj() - log.info("R1: %s" % obj) + logger.trace(pickler, "R1: %s", obj) #refobj = ctypes.pythonapi.PyWeakref_GetObject(obj) # dead returns "None" pickler.save_reduce(_create_weakref, (refobj,), obj=obj) - log.info("# R1") + logger.trace(pickler, "# R1") return @register(ProxyType) @@ -1898,15 +1601,15 @@ def save_weakproxy(pickler, obj): refobj = _locate_object(_proxy_helper(obj)) try: _t = "R2" - log.info("%s: %s" % (_t, obj)) + logger.trace(pickler, "%s: %s", _t, obj) except ReferenceError: _t = "R3" - log.info("%s: %s" % (_t, sys.exc_info()[1])) + logger.trace(pickler, "%s: %s", _t, sys.exc_info()[1]) #callable = bool(getattr(refobj, '__call__', None)) if type(obj) is CallableProxyType: callable = True else: callable = False pickler.save_reduce(_create_weakproxy, (refobj, callable), obj=obj) - log.info("# %s" % _t) + logger.trace(pickler, "# %s", _t) return def _is_builtin_module(module): @@ -1919,6 +1622,9 @@ def _is_builtin_module(module): module.__file__.endswith(EXTENSION_SUFFIXES) or \ 'site-packages' in module.__file__ +def _is_imported_module(module): + return getattr(module, '__loader__', None) is not None or module in sys.modules.values() + @register(ModuleType) def save_module(pickler, obj): if False: #_use_diff: @@ -1928,35 +1634,36 @@ def save_module(pickler, obj): except RuntimeError: # not memorised module, probably part of dill pass else: - log.info("M2: %s with diff" % obj) - log.info("Diff: %s", changed.keys()) + logger.trace(pickler, "M2: %s with diff", obj) + logger.trace(pickler, "Diff: %s", changed.keys()) pickler.save_reduce(_import_module, (obj.__name__,), obj=obj, state=changed) - log.info("# M2") + logger.trace(pickler, "# M2") return - log.info("M1: %s" % obj) + logger.trace(pickler, "M1: %s", obj) pickler.save_reduce(_import_module, (obj.__name__,), obj=obj) - log.info("# M1") + logger.trace(pickler, "# M1") else: builtin_mod = _is_builtin_module(obj) if obj.__name__ not in ("builtins", "dill", "dill._dill") and not builtin_mod or \ is_dill(pickler, child=True) and obj is pickler._main: - log.info("M1: %s" % obj) + logger.trace(pickler, "M1: %s", obj) _main_dict = obj.__dict__.copy() #XXX: better no copy? option to copy? [_main_dict.pop(item, None) for item in singletontypes + ["__builtins__", "__loader__"]] - pickler.save_reduce(_import_module, (obj.__name__,), obj=obj, + mod_name = obj.__name__ if _is_imported_module(obj) else '__runtime__.%s' % obj.__name__ + pickler.save_reduce(_import_module, (mod_name,), obj=obj, state=_main_dict) - log.info("# M1") - elif PY3 and obj.__name__ == "dill._dill": - log.info("M2: %s" % obj) + logger.trace(pickler, "# M1") + elif obj.__name__ == "dill._dill": + logger.trace(pickler, "M2: %s", obj) pickler.save_global(obj, name="_dill") - log.info("# M2") + logger.trace(pickler, "# M2") else: - log.info("M2: %s" % obj) + logger.trace(pickler, "M2: %s", obj) pickler.save_reduce(_import_module, (obj.__name__,), obj=obj) - log.info("# M2") + logger.trace(pickler, "# M2") return return @@ -1988,10 +1695,8 @@ def _get_typedict_type(cls, clsdict, postproc_list): clsdict.pop('__weakref__', None) # clsdict.pop('__prepare__', None) return clsdict - # return _dict_from_dictproxy(cls.__dict__) def _get_typedict_abc(obj, _dict, attrs, postproc_list): - log.info("ABC: %s" % obj) if hasattr(abc, '_get_dump'): (registry, _, _, _) = abc._get_dump(obj) register = obj.register @@ -2010,13 +1715,11 @@ def _get_typedict_abc(obj, _dict, attrs, postproc_list): # del _dict['_abc_negative_cache_version'] else: del _dict['_abc_impl'] - log.info("# ABC") return _dict, attrs -CORE_CLASSES = (int, float, type(None), str, dict, tuple, set, list, frozenset) +CORE_CLASSES = {int, float, type(None), str, dict, tuple, set, list, frozenset} def _get_typedict_enum(obj, _dict, attrs, postproc_list): - log.info("E: %s" % obj) base = None metacls = type(obj) @@ -2049,50 +1752,48 @@ def _get_typedict_enum(obj, _dict, attrs, postproc_list): attrs.update(_dict) _dict = attrs - log.info("# E") return original_dict, _dict @register(TypeType) def save_type(pickler, obj, postproc_list=None): if obj in _typemap: - log.info("T1: %s" % obj) + logger.trace(pickler, "T1: %s", obj) + # if obj in _incedental_types: + # warnings.warn('Type %r may only exist on this implementation of Python and cannot be unpickled in other implementations.' % (obj,), PicklingWarning) pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj) - log.info("# T1") + logger.trace(pickler, "# T1") elif obj.__bases__ == (tuple,) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]): # special case: namedtuples - log.info("T6: %s" % obj) + logger.trace(pickler, "T6: %s", obj) obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) - if PY3 and obj.__name__ != obj_name: + if obj.__name__ != obj_name: if postproc_list is None: postproc_list = [] postproc_list.append((setattr, (obj, '__qualname__', obj_name))) - if OLD37 or (not obj._field_defaults): + if not obj._field_defaults: _save_with_postproc(pickler, (_create_namedtuple, (obj.__name__, obj._fields, obj.__module__)), obj=obj, postproc_list=postproc_list) else: defaults = [obj._field_defaults[field] for field in obj._fields if field in obj._field_defaults] _save_with_postproc(pickler, (_create_namedtuple, (obj.__name__, obj._fields, obj.__module__, defaults)), obj=obj, postproc_list=postproc_list) - log.info("# T6") + logger.trace(pickler, "# T6") return # special cases: NoneType, NotImplementedType, EllipsisType elif obj is type(None): - log.info("T7: %s" % obj) + logger.trace(pickler, "T7: %s", obj) #XXX: pickler.save_reduce(type, (None,), obj=obj) - if PY3: - pickler.write(bytes('c__builtin__\nNoneType\n', 'UTF-8')) - else: - pickler.write('c__builtin__\nNoneType\n') - log.info("# T7") + pickler.write(bytes('c__builtin__\nNoneType\n', 'UTF-8')) + logger.trace(pickler, "# T7") elif obj is NotImplementedType: - log.info("T7: %s" % obj) + logger.trace(pickler, "T7: %s", obj) pickler.save_reduce(type, (NotImplemented,), obj=obj) - log.info("# T7") + logger.trace(pickler, "# T7") elif obj is EllipsisType: - log.info("T7: %s" % obj) + logger.trace(pickler, "T7: %s", obj) pickler.save_reduce(type, (Ellipsis,), obj=obj) - log.info("# T7") + logger.trace(pickler, "# T7") else: _byref = getattr(pickler, '_byref', None) @@ -2103,18 +1804,26 @@ def save_type(pickler, obj, postproc_list=None): postproc_list = [] # thanks to Tom Stepleton pointing out pickler._session unneeded - log.info("T3: %s" % obj) + logger.trace(pickler, "T2: %s", obj) _dict = _get_typedict_type(obj, obj.__dict__.copy(), postproc_list) # copy dict proxy to a dict attrs = None - for name in _dict.get("__slots__", []): - del _dict[name] + slots = _dict.get('__slots__', ()) + if type(slots) == str: + # __slots__ accepts a single string + slots = (slots,) + for name in slots: + _dict.pop(name, None) if isinstance(obj, abc.ABCMeta): + logger.trace(pickler, "ABC: %s", obj) _dict, attrs = _get_typedict_abc(obj, _dict, attrs, postproc_list) + logger.trace(pickler, "# ABC") if EnumMeta and isinstance(obj, EnumMeta): + logger.trace(pickler, "E: %s", obj) _dict, attrs = _get_typedict_enum(obj, _dict, attrs, postproc_list) + logger.trace(pickler, "# E") qualname = getattr(obj, '__qualname__', None) if attrs is not None: @@ -2126,7 +1835,11 @@ def save_type(pickler, obj, postproc_list=None): elif qualname is not None: postproc_list.append((setattr, (obj, '__qualname__', qualname))) - if PY3: # and type(obj) is not type or hasattr(obj, '__orig_bases__'): + if False: # not hasattr(obj, '__orig_bases__'): + _save_with_postproc(pickler, (_create_type, ( + type(obj), obj.__name__, obj.__bases__, _dict + )), obj=obj, postproc_list=postproc_list) + else: # This case will always work, but might be overkill. from types import new_class _metadict = { @@ -2142,73 +1855,70 @@ def save_type(pickler, obj, postproc_list=None): _save_with_postproc(pickler, (new_class, ( obj.__name__, bases, _metadict, _dict_update )), obj=obj, postproc_list=postproc_list) - else: - _save_with_postproc(pickler, (_create_type, ( - type(obj), obj.__name__, obj.__bases__, _dict - )), obj=obj, postproc_list=postproc_list) - log.info("# T3") + logger.trace(pickler, "# T2") else: obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) - log.info("T4: %s" % obj) + logger.trace(pickler, "T4: %s", obj) if incorrectly_named: warnings.warn('Cannot locate reference to %r.' % (obj,), PicklingWarning) if obj_recursive: warnings.warn('Cannot pickle %r: %s.%s has recursive self-references that trigger a RecursionError.' % (obj, obj.__module__, obj_name), PicklingWarning) - #print (obj.__dict__) - #print ("%s\n%s" % (type(obj), obj.__name__)) - #print ("%s\n%s" % (obj.__bases__, obj.__dict__)) StockPickler.save_global(pickler, obj, name=obj_name) - log.info("# T4") + logger.trace(pickler, "# T4") return -# Error in PyPy 2.7 when adding ABC support -if IS_PYPY2: - @register(FrameType) - def save_frame(pickler, obj): - raise PicklingError('Cannot pickle a Python stack frame') - @register(property) @register(abc.abstractproperty) def save_property(pickler, obj): - log.info("Pr: %s" % obj) + logger.trace(pickler, "Pr: %s", obj) pickler.save_reduce(type(obj), (obj.fget, obj.fset, obj.fdel, obj.__doc__), obj=obj) - log.info("# Pr") + logger.trace(pickler, "# Pr") @register(staticmethod) @register(classmethod) +@register(abc.abstractstaticmethod) +@register(abc.abstractclassmethod) def save_classmethod(pickler, obj): - log.info("Cm: %s" % obj) - im_func = '__func__' if PY3 else 'im_func' - try: - orig_func = getattr(obj, im_func) - except AttributeError: # Python 2.6 - orig_func = obj.__get__(None, object) - if isinstance(obj, classmethod): - orig_func = getattr(orig_func, im_func) # Unbind - - # if PY3: - # if type(obj.__dict__) is dict: - # if obj.__dict__: - # state = obj.__dict__ - # else: - # state = None + logger.trace(pickler, "Cm: %s", obj) + orig_func = obj.__func__ + + # if type(obj.__dict__) is dict: + # if obj.__dict__: + # state = obj.__dict__ # else: - # state = (None, {'__dict__', obj.__dict__}) + # state = None # else: - # state = None + # state = (None, {'__dict__', obj.__dict__}) pickler.save_reduce(type(obj), (orig_func,), obj=obj) - log.info("# Cm") - -if sys.hexversion >= 0x03020000: - register(abc.abstractstaticmethod)(save_classmethod) - register(abc.abstractclassmethod)(save_classmethod) + logger.trace(pickler, "# Cm") @register(FunctionType) def save_function(pickler, obj): if not _locate_function(obj, pickler): - log.info("F1: %s" % obj) + if type(obj.__code__) is not CodeType: + # Some PyPy builtin functions have no module name, and thus are not + # able to be located + module_name = getattr(obj, '__module__', None) + if module_name is None: + module_name = __builtin__.__name__ + module = _import_module(module_name, safe=True) + _pypy_builtin = False + try: + found, _ = _getattribute(module, obj.__qualname__) + if getattr(found, '__func__', None) is obj: + _pypy_builtin = True + except AttributeError: + pass + + if _pypy_builtin: + logger.trace(pickler, "F3: %s", obj) + pickler.save_reduce(getattr, (found, '__func__'), obj=obj) + logger.trace(pickler, "# F3") + return + + logger.trace(pickler, "F1: %s", obj) _recurse = getattr(pickler, '_recurse', None) _postproc = getattr(pickler, '_postproc', None) _main_modified = getattr(pickler, '_main_modified', None) @@ -2225,7 +1935,7 @@ def save_function(pickler, obj): # is created to correctly handle recursion. globs = {'__name__': obj.__module__} else: - globs_copy = obj.__globals__ if PY3 else obj.func_globals + globs_copy = obj.__globals__ # If the globals is the __dict__ from the module being saved as a # session, substitute it by the dictionary being actually saved. @@ -2243,11 +1953,7 @@ def save_function(pickler, obj): # In the case that the globals are copied, we need to ensure that # the globals dictionary is updated when all objects in the # dictionary are already created. - if PY3: - glob_ids = {id(g) for g in globs_copy.values()} - else: - glob_ids = {id(g) for g in globs_copy.itervalues()} - + glob_ids = {id(g) for g in globs_copy.values()} for stack_element in _postproc: if stack_element in glob_ids: _postproc[stack_element].append((_setitems, (globs, globs_copy))) @@ -2255,42 +1961,28 @@ def save_function(pickler, obj): else: postproc_list.append((_setitems, (globs, globs_copy))) - if PY3: - closure = obj.__closure__ - state_dict = {} - for fattrname in ('__doc__', '__kwdefaults__', '__annotations__'): - fattr = getattr(obj, fattrname, None) - if fattr is not None: - state_dict[fattrname] = fattr - if obj.__qualname__ != obj.__name__: - state_dict['__qualname__'] = obj.__qualname__ - if '__name__' not in globs or obj.__module__ != globs['__name__']: - state_dict['__module__'] = obj.__module__ - - state = obj.__dict__ - if type(state) is not dict: - state_dict['__dict__'] = state - state = None - if state_dict: - state = state, state_dict - - _save_with_postproc(pickler, (_create_function, ( - obj.__code__, globs, obj.__name__, obj.__defaults__, - closure - ), state), obj=obj, postproc_list=postproc_list) - else: - closure = obj.func_closure - if obj.__doc__ is not None: - postproc_list.append((setattr, (obj, '__doc__', obj.__doc__))) - if '__name__' not in globs or obj.__module__ != globs['__name__']: - postproc_list.append((setattr, (obj, '__module__', obj.__module__))) - if obj.__dict__: - postproc_list.append((setattr, (obj, '__dict__', obj.__dict__))) - - _save_with_postproc(pickler, (_create_function, ( - obj.func_code, globs, obj.func_name, obj.func_defaults, + closure = obj.__closure__ + state_dict = {} + for fattrname in ('__doc__', '__kwdefaults__', '__annotations__'): + fattr = getattr(obj, fattrname, None) + if fattr is not None: + state_dict[fattrname] = fattr + if obj.__qualname__ != obj.__name__: + state_dict['__qualname__'] = obj.__qualname__ + if '__name__' not in globs or obj.__module__ != globs['__name__']: + state_dict['__module__'] = obj.__module__ + + state = obj.__dict__ + if type(state) is not dict: + state_dict['__dict__'] = state + state = None + if state_dict: + state = state, state_dict + + _save_with_postproc(pickler, (_create_function, ( + obj.__code__, globs, obj.__name__, obj.__defaults__, closure - )), obj=obj, postproc_list=postproc_list) + ), state), obj=obj, postproc_list=postproc_list) # Lift closure cell update to earliest function (#458) if _postproc: @@ -2306,19 +1998,100 @@ def save_function(pickler, obj): # Change the value of the cell pickler.save_reduce(*possible_postproc) # pop None created by calling preprocessing step off stack - if PY3: - pickler.write(bytes('0', 'UTF-8')) - else: - pickler.write('0') + pickler.write(bytes('0', 'UTF-8')) - log.info("# F1") + logger.trace(pickler, "# F1") else: - log.info("F2: %s" % obj) + logger.trace(pickler, "F2: %s", obj) name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) StockPickler.save_global(pickler, obj, name=name) - log.info("# F2") + logger.trace(pickler, "# F2") return +if HAS_CTYPES and hasattr(ctypes, 'pythonapi'): + _PyCapsule_New = ctypes.pythonapi.PyCapsule_New + _PyCapsule_New.argtypes = (ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p) + _PyCapsule_New.restype = ctypes.py_object + _PyCapsule_GetPointer = ctypes.pythonapi.PyCapsule_GetPointer + _PyCapsule_GetPointer.argtypes = (ctypes.py_object, ctypes.c_char_p) + _PyCapsule_GetPointer.restype = ctypes.c_void_p + _PyCapsule_GetDestructor = ctypes.pythonapi.PyCapsule_GetDestructor + _PyCapsule_GetDestructor.argtypes = (ctypes.py_object,) + _PyCapsule_GetDestructor.restype = ctypes.c_void_p + _PyCapsule_GetContext = ctypes.pythonapi.PyCapsule_GetContext + _PyCapsule_GetContext.argtypes = (ctypes.py_object,) + _PyCapsule_GetContext.restype = ctypes.c_void_p + _PyCapsule_GetName = ctypes.pythonapi.PyCapsule_GetName + _PyCapsule_GetName.argtypes = (ctypes.py_object,) + _PyCapsule_GetName.restype = ctypes.c_char_p + _PyCapsule_IsValid = ctypes.pythonapi.PyCapsule_IsValid + _PyCapsule_IsValid.argtypes = (ctypes.py_object, ctypes.c_char_p) + _PyCapsule_IsValid.restype = ctypes.c_bool + _PyCapsule_SetContext = ctypes.pythonapi.PyCapsule_SetContext + _PyCapsule_SetContext.argtypes = (ctypes.py_object, ctypes.c_void_p) + _PyCapsule_SetDestructor = ctypes.pythonapi.PyCapsule_SetDestructor + _PyCapsule_SetDestructor.argtypes = (ctypes.py_object, ctypes.c_void_p) + _PyCapsule_SetName = ctypes.pythonapi.PyCapsule_SetName + _PyCapsule_SetName.argtypes = (ctypes.py_object, ctypes.c_char_p) + _PyCapsule_SetPointer = ctypes.pythonapi.PyCapsule_SetPointer + _PyCapsule_SetPointer.argtypes = (ctypes.py_object, ctypes.c_void_p) + _testcapsule = _PyCapsule_New( + ctypes.cast(_PyCapsule_New, ctypes.c_void_p), + ctypes.create_string_buffer(b'dill._dill._testcapsule'), + None + ) + PyCapsuleType = type(_testcapsule) + @register(PyCapsuleType) + def save_capsule(pickler, obj): + logger.trace(pickler, "Cap: %s", obj) + name = _PyCapsule_GetName(obj) + warnings.warn('Pickling a PyCapsule (%s) does not pickle any C data structures and could cause segmentation faults or other memory errors when unpickling.' % (name,), PicklingWarning) + pointer = _PyCapsule_GetPointer(obj, name) + context = _PyCapsule_GetContext(obj) + destructor = _PyCapsule_GetDestructor(obj) + pickler.save_reduce(_create_capsule, (pointer, name, context, destructor), obj=obj) + logger.trace(pickler, "# Cap") + _incedental_reverse_typemap['PyCapsuleType'] = PyCapsuleType + _reverse_typemap['PyCapsuleType'] = PyCapsuleType + _incedental_types.add(PyCapsuleType) +else: + _testcapsule = None + + +############################# +# A quick fix for issue #500 +# This should be removed when a better solution is found. + +if hasattr(dataclasses, "_HAS_DEFAULT_FACTORY_CLASS"): + @register(dataclasses._HAS_DEFAULT_FACTORY_CLASS) + def save_dataclasses_HAS_DEFAULT_FACTORY_CLASS(pickler, obj): + logger.trace(pickler, "DcHDF: %s", obj) + pickler.write(GLOBAL + b"dataclasses\n_HAS_DEFAULT_FACTORY\n") + logger.trace(pickler, "# DcHDF") + +if hasattr(dataclasses, "MISSING"): + @register(type(dataclasses.MISSING)) + def save_dataclasses_MISSING_TYPE(pickler, obj): + logger.trace(pickler, "DcM: %s", obj) + pickler.write(GLOBAL + b"dataclasses\nMISSING\n") + logger.trace(pickler, "# DcM") + +if hasattr(dataclasses, "KW_ONLY"): + @register(type(dataclasses.KW_ONLY)) + def save_dataclasses_KW_ONLY_TYPE(pickler, obj): + logger.trace(pickler, "DcKWO: %s", obj) + pickler.write(GLOBAL + b"dataclasses\nKW_ONLY\n") + logger.trace(pickler, "# DcKWO") + +if hasattr(dataclasses, "_FIELD_BASE"): + @register(dataclasses._FIELD_BASE) + def save_dataclasses_FIELD_BASE(pickler, obj): + logger.trace(pickler, "DcFB: %s", obj) + pickler.write(GLOBAL + b"dataclasses\n" + obj.name.encode() + b"\n") + logger.trace(pickler, "# DcFB") + +############################# + # quick sanity checking def pickles(obj,exact=False,safe=False,**kwds): """ @@ -2334,7 +2107,7 @@ def pickles(obj,exact=False,safe=False,**kwds): """ if safe: exceptions = (Exception,) # RuntimeError, ValueError else: - exceptions = (TypeError, AssertionError, PicklingError, UnpicklingError) + exceptions = (TypeError, AssertionError, NotImplementedError, PicklingError, UnpicklingError) try: pik = copy(obj, **kwds) #FIXME: should check types match first, then check content if "exact" @@ -2401,7 +2174,7 @@ def check(obj, *args, **kwds): # use to protect against missing attributes def is_dill(pickler, child=None): "check the dill-ness of your pickler" - if (child is False) or PY34 or (not hasattr(pickler.__class__, 'mro')): + if child is False or not hasattr(pickler.__class__, 'mro'): return 'dill' in pickler.__module__ return Pickler in pickler.__class__.mro() @@ -2411,9 +2184,8 @@ def _extend(): for t,func in Pickler.dispatch.items(): try: StockPickler.dispatch[t] = func - except: #TypeError, PicklingError, UnpicklingError - log.info("skip: %s" % t) - else: pass + except Exception: #TypeError, PicklingError, UnpicklingError + logger.trace(pickler, "skip: %s", t) return del diff, _use_diff, use_diff diff --git a/dill/_objects.py b/dill/_objects.py index 8b1cb65c..d9f9fe57 100644 --- a/dill/_objects.py +++ b/dill/_objects.py @@ -15,32 +15,21 @@ # helper imports import warnings; warnings.filterwarnings("ignore", category=DeprecationWarning) import sys -PY3 = (hex(sys.hexversion) >= '0x30000f0') -if PY3: - import queue as Queue - import dbm as anydbm -else: - import Queue - import anydbm - import sets # deprecated/removed - import mutex # removed -try: - from cStringIO import StringIO # has StringI and StringO types -except ImportError: # only has StringIO type - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO +import queue as Queue +import dbm as anydbm +from io import BytesIO as StringIO import re import array import collections import codecs import struct +import dataclasses import datetime import calendar import weakref import pprint import decimal +import numbers import functools import itertools import operator @@ -56,6 +45,7 @@ import hmac import os import logging +import logging.handlers import optparse #import __hello__ import threading @@ -64,8 +54,7 @@ try: import bz2 import sqlite3 - if PY3: import dbm.ndbm as dbm - else: import dbm + import dbm.ndbm as dbm HAS_ALL = True except ImportError: # Ubuntu HAS_ALL = False @@ -112,7 +101,7 @@ class _newclass2(object): def _function(x): yield x def _function2(): try: raise - except: + except Exception: from sys import exc_info e, er, tb = exc_info() return er, tb @@ -123,20 +112,12 @@ class _Struct(ctypes.Structure): _filedescrip, _tempfile = tempfile.mkstemp('r') # deleted in cleanup _tmpf = tempfile.TemporaryFile('w') -# put the objects in order, if possible -try: - from collections import OrderedDict as odict -except ImportError: - try: - from ordereddict import OrderedDict as odict - except ImportError: - odict = dict # objects used by dill for type declaration -registered = d = odict() +registered = d = {} # objects dill fails to pickle -failures = x = odict() +failures = x = {} # all other type objects -succeeds = a = odict() +succeeds = a = {} # types module (part of CH 8) a['BooleanType'] = bool(1) @@ -157,12 +138,8 @@ class _Struct(ctypes.Structure): a['StringType'] = _str = str(1) a['TupleType'] = _tuple = () a['TypeType'] = type -if PY3: - a['LongType'] = _int - a['UnicodeType'] = _str -else: - a['LongType'] = long(1) - a['UnicodeType'] = unicode(1) +a['LongType'] = _int +a['UnicodeType'] = _str # built-in constants (CH 4) a['CopyrightType'] = copyright # built-in types (CH 5) @@ -181,17 +158,13 @@ class _Struct(ctypes.Structure): a['TZInfoType'] = datetime.tzinfo() a['DateTimeType'] = datetime.datetime.today() a['CalendarType'] = calendar.Calendar() -if not PY3: - a['SetsType'] = sets.Set() - a['ImmutableSetType'] = sets.ImmutableSet() - a['MutexType'] = mutex.mutex() # numeric and mathematical types (CH 9) a['DecimalType'] = decimal.Decimal(1) a['CountType'] = itertools.count(0) # data compression and archiving (CH 12) a['TarInfoType'] = tarfile.TarInfo() # generic operating system services (CH 15) -a['LoggerType'] = logging.getLogger() +a['LoggerType'] = _logger = logging.getLogger() a['FormatterType'] = logging.Formatter() # pickle ok a['FilterType'] = logging.Filter() # pickle ok a['LogRecordType'] = logging.makeLogRecord(_dict) # pickle ok @@ -199,22 +172,26 @@ class _Struct(ctypes.Structure): a['OptionGroupType'] = optparse.OptionGroup(_oparser,"foo") # pickle ok a['OptionType'] = optparse.Option('--foo') # pickle ok if HAS_CTYPES: - a['CCharType'] = _cchar = ctypes.c_char() - a['CWCharType'] = ctypes.c_wchar() # fail == 2.6 - a['CByteType'] = ctypes.c_byte() - a['CUByteType'] = ctypes.c_ubyte() - a['CShortType'] = ctypes.c_short() - a['CUShortType'] = ctypes.c_ushort() - a['CIntType'] = ctypes.c_int() - a['CUIntType'] = ctypes.c_uint() - a['CLongType'] = ctypes.c_long() - a['CULongType'] = ctypes.c_ulong() - a['CLongLongType'] = ctypes.c_longlong() - a['CULongLongType'] = ctypes.c_ulonglong() - a['CFloatType'] = ctypes.c_float() - a['CDoubleType'] = ctypes.c_double() - a['CSizeTType'] = ctypes.c_size_t() - a['CLibraryLoaderType'] = ctypes.cdll + z = x if IS_PYPY else a + z['CCharType'] = _cchar = ctypes.c_char() + z['CWCharType'] = ctypes.c_wchar() # fail == 2.6 + z['CByteType'] = ctypes.c_byte() + z['CUByteType'] = ctypes.c_ubyte() + z['CShortType'] = ctypes.c_short() + z['CUShortType'] = ctypes.c_ushort() + z['CIntType'] = ctypes.c_int() + z['CUIntType'] = ctypes.c_uint() + z['CLongType'] = ctypes.c_long() + z['CULongType'] = ctypes.c_ulong() + z['CLongLongType'] = ctypes.c_longlong() + z['CULongLongType'] = ctypes.c_ulonglong() + z['CFloatType'] = ctypes.c_float() + z['CDoubleType'] = ctypes.c_double() + z['CSizeTType'] = ctypes.c_size_t() + z = (sys.platform[:3] == 'win' or sys.platform[:6] == 'darwin') # non-'nux + z = a if (sys.hexversion >= 0x30b00b3 and not z) else x + z['CLibraryLoaderType'] = ctypes.cdll + del z a['StructureType'] = _Struct # if not IS_PYPY: # a['BigEndianStructureType'] = ctypes.BigEndianStructure() @@ -223,40 +200,37 @@ class _Struct(ctypes.Structure): #NOTE: ctypes.c_int._objects is memberdescriptor for object's __dict__ #NOTE: base class of all ctypes data types is non-public _CData -try: # python 2.6 - import fractions - import number - import io - from io import StringIO as TextIO - # built-in functions (CH 2) - a['ByteArrayType'] = bytearray([1]) - # numeric and mathematical types (CH 9) - a['FractionType'] = fractions.Fraction() - a['NumberType'] = numbers.Number() - # generic operating system services (CH 15) - a['IOBaseType'] = io.IOBase() - a['RawIOBaseType'] = io.RawIOBase() - a['TextIOBaseType'] = io.TextIOBase() - a['BufferedIOBaseType'] = io.BufferedIOBase() - a['UnicodeIOType'] = TextIO() # the new StringIO - a['LoggingAdapterType'] = logging.LoggingAdapter(_logger,_dict) # pickle ok - if HAS_CTYPES: - a['CBoolType'] = ctypes.c_bool(1) - a['CLongDoubleType'] = ctypes.c_longdouble() -except ImportError: - pass -try: # python 2.7 - import argparse - # data types (CH 8) - a['OrderedDictType'] = collections.OrderedDict(_dict) - a['CounterType'] = collections.Counter(_dict) - if HAS_CTYPES: - a['CSSizeTType'] = ctypes.c_ssize_t() - # generic operating system services (CH 15) - a['NullHandlerType'] = logging.NullHandler() # pickle ok # new 2.7 - a['ArgParseFileType'] = argparse.FileType() # pickle ok -except (AttributeError, ImportError): - pass +import fractions +import io +from io import StringIO as TextIO +# built-in functions (CH 2) +a['ByteArrayType'] = bytearray([1]) +# numeric and mathematical types (CH 9) +a['FractionType'] = fractions.Fraction() +a['NumberType'] = numbers.Number() +# generic operating system services (CH 15) +a['IOBaseType'] = io.IOBase() +a['RawIOBaseType'] = io.RawIOBase() +a['TextIOBaseType'] = io.TextIOBase() +a['BufferedIOBaseType'] = io.BufferedIOBase() +a['UnicodeIOType'] = TextIO() # the new StringIO +a['LoggerAdapterType'] = logging.LoggerAdapter(_logger,_dict) # pickle ok +if HAS_CTYPES: + z = x if IS_PYPY else a + z['CBoolType'] = ctypes.c_bool(1) + z['CLongDoubleType'] = ctypes.c_longdouble() + del z +import argparse +# data types (CH 8) +a['OrderedDictType'] = collections.OrderedDict(_dict) +a['CounterType'] = collections.Counter(_dict) +if HAS_CTYPES: + z = x if IS_PYPY else a + z['CSSizeTType'] = ctypes.c_ssize_t() + del z +# generic operating system services (CH 15) +a['NullHandlerType'] = logging.NullHandler() # pickle ok # new 2.7 +a['ArgParseFileType'] = argparse.FileType() # pickle ok # -- pickle fails on all below here ----------------------------------------- # types module (part of CH 8) @@ -275,45 +249,36 @@ class _Struct(ctypes.Structure): a['NotImplementedType'] = NotImplemented a['SliceType'] = slice(1) a['UnboundMethodType'] = _class._method #XXX: works when not imported! -a['TextWrapperType'] = open(os.devnull, 'r') # same as mode='w','w+','r+' -a['BufferedRandomType'] = open(os.devnull, 'r+b') # same as mode='w+b' -a['BufferedReaderType'] = open(os.devnull, 'rb') # (default: buffering=-1) -a['BufferedWriterType'] = open(os.devnull, 'wb') +d['TextWrapperType'] = open(os.devnull, 'r') # same as mode='w','w+','r+' +d['BufferedRandomType'] = open(os.devnull, 'r+b') # same as mode='w+b' +d['BufferedReaderType'] = open(os.devnull, 'rb') # (default: buffering=-1) +d['BufferedWriterType'] = open(os.devnull, 'wb') try: # oddities: deprecated from _pyio import open as _open - a['PyTextWrapperType'] = _open(os.devnull, 'r', buffering=-1) - a['PyBufferedRandomType'] = _open(os.devnull, 'r+b', buffering=-1) - a['PyBufferedReaderType'] = _open(os.devnull, 'rb', buffering=-1) - a['PyBufferedWriterType'] = _open(os.devnull, 'wb', buffering=-1) + d['PyTextWrapperType'] = _open(os.devnull, 'r', buffering=-1) + d['PyBufferedRandomType'] = _open(os.devnull, 'r+b', buffering=-1) + d['PyBufferedReaderType'] = _open(os.devnull, 'rb', buffering=-1) + d['PyBufferedWriterType'] = _open(os.devnull, 'wb', buffering=-1) except ImportError: pass # other (concrete) object types -if PY3: - d['CellType'] = (_lambda)(0).__closure__[0] - a['XRangeType'] = _xrange = range(1) -else: - d['CellType'] = (_lambda)(0).func_closure[0] - a['XRangeType'] = _xrange = xrange(1) -if not IS_PYPY: - d['MethodDescriptorType'] = type.__dict__['mro'] - d['WrapperDescriptorType'] = type.__repr__ - a['WrapperDescriptorType2'] = type.__dict__['__module__'] - d['ClassMethodDescriptorType'] = type.__dict__['__prepare__' if PY3 else 'mro'] +z = d if sys.hexversion < 0x30800a2 else a +z['CellType'] = (_lambda)(0).__closure__[0] +del z +a['XRangeType'] = _xrange = range(1) +a['MethodDescriptorType'] = type.__dict__['mro'] +a['WrapperDescriptorType'] = type.__repr__ +#a['WrapperDescriptorType2'] = type.__dict__['__module__']#XXX: GetSetDescriptor +a['ClassMethodDescriptorType'] = type.__dict__['__prepare__'] # built-in functions (CH 2) -if PY3 or IS_PYPY: - _methodwrap = (1).__lt__ -else: - _methodwrap = (1).__cmp__ -d['MethodWrapperType'] = _methodwrap +_methodwrap = (1).__lt__ +a['MethodWrapperType'] = _methodwrap a['StaticMethodType'] = staticmethod(_method) a['ClassMethodType'] = classmethod(_method) a['PropertyType'] = property() d['SuperType'] = super(Exception, _exception) # string services (CH 7) -if PY3: - _in = _bytes -else: - _in = _str +_in = _bytes a['InputType'] = _cstrI = StringIO(_in) a['OutputType'] = _cstrO = StringIO() # data types (CH 8) @@ -328,52 +293,40 @@ class _Struct(ctypes.Structure): a['QueueType'] = Queue.Queue() # numeric and mathematical types (CH 9) d['PartialType'] = functools.partial(int,base=2) -if PY3: - a['IzipType'] = zip('0','1') -else: - a['IzipType'] = itertools.izip('0','1') +a['IzipType'] = zip('0','1') a['ChainType'] = itertools.chain('0','1') d['ItemGetterType'] = operator.itemgetter(0) d['AttrGetterType'] = operator.attrgetter('__repr__') # file and directory access (CH 10) -if PY3: _fileW = _cstrO -else: _fileW = _tmpf +_fileW = _cstrO # data persistence (CH 11) if HAS_ALL: - a['ConnectionType'] = _conn = sqlite3.connect(':memory:') - a['CursorType'] = _conn.cursor() + x['ConnectionType'] = _conn = sqlite3.connect(':memory:') + x['CursorType'] = _conn.cursor() a['ShelveType'] = shelve.Shelf({}) # data compression and archiving (CH 12) if HAS_ALL: - if (hex(sys.hexversion) < '0x2070ef0') or PY3: - a['BZ2FileType'] = bz2.BZ2File(os.devnull) #FIXME: fail >= 3.3, 2.7.14 - a['BZ2CompressorType'] = bz2.BZ2Compressor() - a['BZ2DecompressorType'] = bz2.BZ2Decompressor() -#a['ZipFileType'] = _zip = zipfile.ZipFile(os.devnull,'w') #FIXME: fail >= 3.2 + x['BZ2FileType'] = bz2.BZ2File(os.devnull) + x['BZ2CompressorType'] = bz2.BZ2Compressor() + x['BZ2DecompressorType'] = bz2.BZ2Decompressor() +#x['ZipFileType'] = _zip = zipfile.ZipFile(os.devnull,'w') #_zip.write(_tempfile,'x') [causes annoying warning/error printed on import] #a['ZipInfoType'] = _zip.getinfo('x') a['TarFileType'] = tarfile.open(fileobj=_fileW,mode='w') # file formats (CH 13) -a['DialectType'] = csv.get_dialect('excel') +x['DialectType'] = csv.get_dialect('excel') a['PackerType'] = xdrlib.Packer() # optional operating system services (CH 16) a['LockType'] = threading.Lock() a['RLockType'] = threading.RLock() # generic operating system services (CH 15) # also closed/open and r/w/etc... -a['NamedLoggerType'] = _logger = logging.getLogger(__name__) #FIXME: fail >= 3.2 and <= 2.6 +a['NamedLoggerType'] = _logger = logging.getLogger(__name__) #a['FrozenModuleType'] = __hello__ #FIXME: prints "Hello world..." # interprocess communication (CH 17) -if PY3: - a['SocketType'] = _socket = socket.socket() #FIXME: fail >= 3.3 - a['SocketPairType'] = socket.socketpair()[0] #FIXME: fail >= 3.3 -else: - a['SocketType'] = _socket = socket.socket() - a['SocketPairType'] = _socket._sock +x['SocketType'] = _socket = socket.socket() +x['SocketPairType'] = socket.socketpair()[0] # python runtime services (CH 27) -if PY3: - a['GeneratorContextManagerType'] = contextlib.contextmanager(max)([1]) -else: - a['GeneratorContextManagerType'] = contextlib.GeneratorContextManager(max) +a['GeneratorContextManagerType'] = contextlib.contextmanager(max)([1]) try: # ipython __IPYTHON__ is True # is ipython @@ -390,59 +343,84 @@ class _Struct(ctypes.Structure): a['NumpyInt32Type'] = _numpy_int32 except ImportError: pass -try: # python 2.6 - # numeric and mathematical types (CH 9) - a['ProductType'] = itertools.product('0','1') - # generic operating system services (CH 15) - a['FileHandlerType'] = logging.FileHandler(os.devnull) #FIXME: fail >= 3.2 and <= 2.6 - a['RotatingFileHandlerType'] = logging.handlers.RotatingFileHandler(os.devnull) - a['SocketHandlerType'] = logging.handlers.SocketHandler('localhost',514) - a['MemoryHandlerType'] = logging.handlers.MemoryHandler(1) -except AttributeError: - pass -try: # python 2.7 - # data types (CH 8) - a['WeakSetType'] = weakref.WeakSet() # 2.7 -# # generic operating system services (CH 15) [errors when dill is imported] -# a['ArgumentParserType'] = _parser = argparse.ArgumentParser('PROG') -# a['NamespaceType'] = _parser.parse_args() # pickle ok -# a['SubParsersActionType'] = _parser.add_subparsers() -# a['MutuallyExclusiveGroupType'] = _parser.add_mutually_exclusive_group() -# a['ArgumentGroupType'] = _parser.add_argument_group() -except AttributeError: - pass +# numeric and mathematical types (CH 9) +a['ProductType'] = itertools.product('0','1') +# generic operating system services (CH 15) +a['FileHandlerType'] = logging.FileHandler(os.devnull) +a['RotatingFileHandlerType'] = logging.handlers.RotatingFileHandler(os.devnull) +a['SocketHandlerType'] = logging.handlers.SocketHandler('localhost',514) +a['MemoryHandlerType'] = logging.handlers.MemoryHandler(1) +# data types (CH 8) +a['WeakSetType'] = weakref.WeakSet() # 2.7 +# generic operating system services (CH 15) [errors when dill is imported] +#a['ArgumentParserType'] = _parser = argparse.ArgumentParser('PROG') +#a['NamespaceType'] = _parser.parse_args() # pickle ok +#a['SubParsersActionType'] = _parser.add_subparsers() +#a['MutuallyExclusiveGroupType'] = _parser.add_mutually_exclusive_group() +#a['ArgumentGroupType'] = _parser.add_argument_group() # -- dill fails in some versions below here --------------------------------- # types module (part of CH 8) -a['FileType'] = open(os.devnull, 'rb', buffering=0) # same 'wb','wb+','rb+' -# FIXME: FileType fails >= 3.1 +d['FileType'] = open(os.devnull, 'rb', buffering=0) # same 'wb','wb+','rb+' # built-in functions (CH 2) -a['ListIteratorType'] = iter(_list) # empty vs non-empty FIXME: fail < 3.2 -a['TupleIteratorType']= iter(_tuple) # empty vs non-empty FIXME: fail < 3.2 -a['XRangeIteratorType'] = iter(_xrange) # empty vs non-empty FIXME: fail < 3.2 +# Iterators: +a['ListIteratorType'] = iter(_list) # empty vs non-empty +a['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty #FIXME: list_iterator +a['TupleIteratorType']= iter(_tuple) # empty vs non-empty +a['XRangeIteratorType'] = iter(_xrange) # empty vs non-empty +a["BytesIteratorType"] = iter(b'') +a["BytearrayIteratorType"] = iter(bytearray(b'')) +z = x if IS_PYPY else a +z["CallableIteratorType"] = iter(iter, None) +del z +x["MemoryIteratorType"] = iter(memoryview(b'')) +a["ListReverseiteratorType"] = reversed([]) +X = a['OrderedDictType'] +d["OdictKeysType"] = X.keys() +d["OdictValuesType"] = X.values() +d["OdictItemsType"] = X.items() +a["OdictIteratorType"] = iter(X.keys()) #FIXME: list_iterator +del X +#FIXME: list_iterator +a['DictionaryItemIteratorType'] = iter(type.__dict__.items()) +a['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) +a['DictionaryValueIteratorType'] = iter(type.__dict__.values()) +if sys.hexversion >= 0x30800a0: + a["DictReversekeyiteratorType"] = reversed({}.keys()) + a["DictReversevalueiteratorType"] = reversed({}.values()) + a["DictReverseitemiteratorType"] = reversed({}.items()) + +try: + import symtable + #FIXME: fails to pickle + x["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table +except ImportError: + pass + +if sys.hexversion >= 0x30a00a0: + x['LineIteratorType'] = compile('3', '', 'eval').co_lines() + +if sys.hexversion >= 0x30b00b0: + from types import GenericAlias + d["GenericAliasIteratorType"] = iter(GenericAlias(list, (int,))) + x['PositionsIteratorType'] = compile('3', '', 'eval').co_positions() + # data types (CH 8) -a['PrettyPrinterType'] = pprint.PrettyPrinter() #FIXME: fail >= 3.2 and == 2.5 +a['PrettyPrinterType'] = pprint.PrettyPrinter() # numeric and mathematical types (CH 9) -a['CycleType'] = itertools.cycle('0') #FIXME: fail < 3.2 +a['CycleType'] = itertools.cycle('0') # file and directory access (CH 10) -a['TemporaryFileType'] = _tmpf #FIXME: fail >= 3.2 and == 2.5 +a['TemporaryFileType'] = _tmpf # data compression and archiving (CH 12) -a['GzipFileType'] = gzip.GzipFile(fileobj=_fileW) #FIXME: fail > 3.2 and <= 2.6 +x['GzipFileType'] = gzip.GzipFile(fileobj=_fileW) # generic operating system services (CH 15) -a['StreamHandlerType'] = logging.StreamHandler() #FIXME: fail >= 3.2 and == 2.5 -try: # python 2.6 - # numeric and mathematical types (CH 9) - a['PermutationsType'] = itertools.permutations('0') #FIXME: fail < 3.2 - a['CombinationsType'] = itertools.combinations('0',1) #FIXME: fail < 3.2 -except AttributeError: - pass -try: # python 2.7 - # numeric and mathematical types (CH 9) - a['RepeatType'] = itertools.repeat(0) #FIXME: fail < 3.2 - a['CompressType'] = itertools.compress('0',[1]) #FIXME: fail < 3.2 - #XXX: ...and etc -except AttributeError: - pass +a['StreamHandlerType'] = logging.StreamHandler() +# numeric and mathematical types (CH 9) +a['PermutationsType'] = itertools.permutations('0') +a['CombinationsType'] = itertools.combinations('0',1) +a['RepeatType'] = itertools.repeat(0) +a['CompressType'] = itertools.compress('0',[1]) +#XXX: ...and etc # -- dill fails on all below here ------------------------------------------- # types module (part of CH 8) @@ -452,16 +430,7 @@ class _Struct(ctypes.Structure): # other (concrete) object types # (also: Capsule / CObject ?) # built-in functions (CH 2) -x['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty # built-in types (CH 5) -if PY3: - x['DictionaryItemIteratorType'] = iter(type.__dict__.items()) - x['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) - x['DictionaryValueIteratorType'] = iter(type.__dict__.values()) -else: - x['DictionaryItemIteratorType'] = type.__dict__.iteritems() - x['DictionaryKeyIteratorType'] = type.__dict__.iterkeys() - x['DictionaryValueIteratorType'] = type.__dict__.itervalues() # string services (CH 7) x['StructType'] = struct.Struct('c') x['CallableIteratorType'] = _srepattern.finditer('') @@ -471,7 +440,9 @@ class _Struct(ctypes.Structure): # python object persistence (CH 11) # x['DbShelveType'] = shelve.open('foo','n')#,protocol=2) #XXX: delete foo if HAS_ALL: - x['DbmType'] = dbm.open(_tempfile,'n') + z = a if IS_PYPY else x + z['DbmType'] = dbm.open(_tempfile,'n') + del z # x['DbCursorType'] = _dbcursor = anydbm.open('foo','n') #XXX: delete foo # x['DbType'] = _dbcursor.db # data compression and archiving (CH 12) @@ -484,7 +455,7 @@ class _Struct(ctypes.Structure): x['CSVDictWriterType'] = csv.DictWriter(_cstrO,{}) # cryptographic services (CH 14) x['HashType'] = hashlib.md5() -if (hex(sys.hexversion) < '0x30800a1'): +if (sys.hexversion < 0x30800a1): x['HMACType'] = hmac.new(_in) else: x['HMACType'] = hmac.new(_in, digestmod='md5') @@ -512,41 +483,46 @@ class _Struct(ctypes.Structure): x['NullPtrType'] = _lpchar() x['NullPyObjectType'] = ctypes.py_object() x['PyObjectType'] = ctypes.py_object(lambda :None) - x['FieldType'] = _field = _Struct._field - x['CFUNCTYPEType'] = _cfunc = ctypes.CFUNCTYPE(ctypes.c_char) + z = a if IS_PYPY else x + z['FieldType'] = _field = _Struct._field + z['CFUNCTYPEType'] = _cfunc = ctypes.CFUNCTYPE(ctypes.c_char) x['CFunctionType'] = _cfunc(str) -try: # python 2.6 - # numeric and mathematical types (CH 9) - x['MethodCallerType'] = operator.methodcaller('mro') # 2.6 -except AttributeError: - pass -try: # python 2.7 - # built-in types (CH 5) - x['MemoryType'] = memoryview(_in) # 2.7 - x['MemoryType2'] = memoryview(bytearray(_in)) # 2.7 - if PY3: - x['DictItemsType'] = _dict.items() # 2.7 - x['DictKeysType'] = _dict.keys() # 2.7 - x['DictValuesType'] = _dict.values() # 2.7 - else: - x['DictItemsType'] = _dict.viewitems() # 2.7 - x['DictKeysType'] = _dict.viewkeys() # 2.7 - x['DictValuesType'] = _dict.viewvalues() # 2.7 - # generic operating system services (CH 15) - x['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG') - x['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG') - x['ArgDefaultsHelpFormatterType'] = argparse.ArgumentDefaultsHelpFormatter('PROG') -except NameError: - pass -try: # python 2.7 (and not 3.1) - x['CmpKeyType'] = _cmpkey = functools.cmp_to_key(_methodwrap) # 2.7, >=3.2 - x['CmpKeyObjType'] = _cmpkey('0') #2.7, >=3.2 -except AttributeError: - pass -if PY3: # oddities: removed, etc - x['BufferType'] = x['MemoryType'] -else: - x['BufferType'] = buffer('') + del z +# numeric and mathematical types (CH 9) +a['MethodCallerType'] = operator.methodcaller('mro') # 2.6 +# built-in types (CH 5) +x['MemoryType'] = memoryview(_in) # 2.7 +x['MemoryType2'] = memoryview(bytearray(_in)) # 2.7 +d['DictItemsType'] = _dict.items() # 2.7 +d['DictKeysType'] = _dict.keys() # 2.7 +d['DictValuesType'] = _dict.values() # 2.7 +# generic operating system services (CH 15) +a['RawTextHelpFormatterType'] = argparse.RawTextHelpFormatter('PROG') +a['RawDescriptionHelpFormatterType'] = argparse.RawDescriptionHelpFormatter('PROG') +a['ArgDefaultsHelpFormatterType'] = argparse.ArgumentDefaultsHelpFormatter('PROG') +z = a if IS_PYPY else x +z['CmpKeyType'] = _cmpkey = functools.cmp_to_key(_methodwrap) # 2.7, >=3.2 +z['CmpKeyObjType'] = _cmpkey('0') #2.7, >=3.2 +del z +# oddities: removed, etc +x['BufferType'] = x['MemoryType'] + +from dill._dill import _testcapsule +if _testcapsule is not None: + d['PyCapsuleType'] = _testcapsule +del _testcapsule + +if hasattr(dataclasses, '_HAS_DEFAULT_FACTORY'): + a['DataclassesHasDefaultFactoryType'] = dataclasses._HAS_DEFAULT_FACTORY + +if hasattr(dataclasses, 'MISSING'): + a['DataclassesMissingType'] = dataclasses.MISSING + +if hasattr(dataclasses, 'KW_ONLY'): + a['DataclassesKWOnlyType'] = dataclasses.KW_ONLY + +if hasattr(dataclasses, '_FIELD_BASE'): + a['DataclassesFieldBaseType'] = dataclasses._FIELD # -- cleanup ---------------------------------------------------------------- a.update(d) # registered also succeed diff --git a/dill/_shims.py b/dill/_shims.py index 6bda5136..2e6641ca 100644 --- a/dill/_shims.py +++ b/dill/_shims.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) -# Author: Anirudh Vegesana (avegesan@stanford.edu) +# Author: Anirudh Vegesana (avegesan@cs.stanford.edu) # Copyright (c) 2021-2022 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/dill/blob/master/LICENSE @@ -49,7 +49,8 @@ def _setattr(object, name, value): https://github.com/uqfoundation/dill/pull/443 """ -import inspect, sys +import inspect +import sys _dill = sys.modules['dill._dill'] @@ -150,117 +151,43 @@ def decorator(func): return func return decorator -###################### -## Compatibility Shims are defined below -###################### - -_CELL_EMPTY = Getattr(_dill, '_CELL_EMPTY', None) - -if _dill.OLD37: - if _dill.HAS_CTYPES and hasattr(_dill.ctypes, 'pythonapi') and hasattr(_dill.ctypes.pythonapi, 'PyCell_Set'): - # CPython - ctypes = _dill.ctypes +def register_shim(name, default): + """ + A easier to understand and more compact way of "softly" defining a function. + These two pieces of code are equivalent: - _PyCell_Set = ctypes.pythonapi.PyCell_Set + if _dill.OLD3X: + def _create_class(): + ... + _create_class = register_shim('_create_class', types.new_class) + if _dill.OLD3X: @move_to(_dill) - def _setattr(object, name, value): - if type(object) is _dill.CellType and name == 'cell_contents': - _PyCell_Set.argtypes = (ctypes.py_object, ctypes.py_object) - _PyCell_Set(object, value) - else: - setattr(object, name, value) + def _create_class(): + ... + _create_class = Getattr(_dill, '_create_class', types.new_class) - @move_to(_dill) - def _delattr(object, name): - if type(object) is _dill.CellType and name == 'cell_contents': - _PyCell_Set.argtypes = (ctypes.py_object, ctypes.c_void_p) - _PyCell_Set(object, None) - else: - delattr(object, name) - - # General Python (not CPython) up to 3.6 is in a weird case, where it is - # possible to pickle recursive cells, but we can't assign directly to the - # cell. - elif _dill.PY3: - # Use nonlocal variables to reassign the cell value. - # https://stackoverflow.com/a/59276835 - __nonlocal = ('nonlocal cell',) - exec('''def _setattr(cell, name, value): - if type(cell) is _dill.CellType and name == 'cell_contents': - def cell_setter(value): - %s - cell = value # pylint: disable=unused-variable - func = _dill.FunctionType(cell_setter.__code__, globals(), "", None, (cell,)) # same as cell_setter, but with cell being the cell's contents - func(value) - else: - setattr(cell, name, value)''' % __nonlocal) - move_to(_dill)(_setattr) - - exec('''def _delattr(cell, name): - if type(cell) is _dill.CellType and name == 'cell_contents': - try: - cell.cell_contents - except: - return - def cell_deleter(): - %s - del cell # pylint: disable=unused-variable - func = _dill.FunctionType(cell_deleter.__code__, globals(), "", None, (cell,)) # same as cell_deleter, but with cell being the cell's contents - func() - else: - delattr(cell, name)''' % __nonlocal) - move_to(_dill)(_delattr) + Intuitively, it creates a function or object in the versions of dill/python + that require special reimplementations, and use a core library or default + implementation if that function or object does not exist. + """ + func = globals().get(name) + if func is not None: + _dill.__dict__[name] = func + func.__module__ = _dill.__name__ + if default is Getattr.NO_DEFAULT: + reduction = (getattr, (_dill, name)) else: - # Likely PyPy 2.7. Simulate the nonlocal keyword with bytecode - # manipulation. - - # The following function is based on 'cell_set' from 'cloudpickle' - # https://github.com/cloudpipe/cloudpickle/blob/5d89947288a18029672596a4d719093cc6d5a412/cloudpickle/cloudpickle.py#L393-L482 - # Copyright (c) 2012, Regents of the University of California. - # Copyright (c) 2009 `PiCloud, Inc. `_. - # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE - @move_to(_dill) - def _setattr(cell, name, value): - if type(cell) is _dill.CellType and name == 'cell_contents': - _cell_set = _dill.FunctionType( - _cell_set_template_code, {}, '_cell_set', (), (cell,),) - _cell_set(value) - else: - setattr(cell, name, value) - - def _cell_set_factory(value): - lambda: cell - cell = value - - co = _cell_set_factory.__code__ - - _cell_set_template_code = _dill.CodeType( - co.co_argcount, - co.co_nlocals, - co.co_stacksize, - co.co_flags, - co.co_code, - co.co_consts, - co.co_names, - co.co_varnames, - co.co_filename, - co.co_name, - co.co_firstlineno, - co.co_lnotab, - co.co_cellvars, # co_freevars is initialized with co_cellvars - (), # co_cellvars is made empty - ) - - del co + reduction = (getattr, (_dill, name, default)) - @move_to(_dill) - def _delattr(cell, name): - if type(cell) is _dill.CellType and name == 'cell_contents': - pass - else: - delattr(cell, name) + return Reduce(*reduction, is_callable=callable(default)) -_setattr = Getattr(_dill, '_setattr', setattr) -_delattr = Getattr(_dill, '_delattr', delattr) +###################### +## Compatibility Shims are defined below +###################### + +_CELL_EMPTY = register_shim('_CELL_EMPTY', None) + +_setattr = register_shim('_setattr', setattr) +_delattr = register_shim('_delattr', delattr) diff --git a/dill/detect.py b/dill/detect.py index 41575205..b6a6cb76 100644 --- a/dill/detect.py +++ b/dill/detect.py @@ -11,10 +11,9 @@ import dis from inspect import ismethod, isfunction, istraceback, isframe, iscode -from .pointers import parent, reference, at, parents, children -from ._dill import _trace as trace -from ._dill import PY3 +from .pointers import parent, reference, at, parents, children +from .logger import trace __all__ = ['baditems','badobjects','badtypes','code','errors','freevars', 'getmodule','globalvars','nestedcode','nestedglobals','outermost', @@ -25,9 +24,7 @@ def getmodule(object, _filename=None, force=False): from inspect import getmodule as getmod module = getmod(object, _filename) if module or not force: return module - if PY3: builtins = 'builtins' - else: builtins = '__builtin__' - builtins = __import__(builtins) + import builtins from .source import getname name = getname(object, force=True) return builtins if name in vars(builtins).keys() else None @@ -37,26 +34,17 @@ def outermost(func): # is analogous to getsource(func,enclosing=True) NOTE: this is the object-equivalent of getsource(func, enclosing=True) """ - if PY3: - if ismethod(func): - _globals = func.__func__.__globals__ or {} - elif isfunction(func): - _globals = func.__globals__ or {} - else: - return #XXX: or raise? no matches - _globals = _globals.items() + if ismethod(func): + _globals = func.__func__.__globals__ or {} + elif isfunction(func): + _globals = func.__globals__ or {} else: - if ismethod(func): - _globals = func.im_func.func_globals or {} - elif isfunction(func): - _globals = func.func_globals or {} - else: - return #XXX: or raise? no matches - _globals = _globals.iteritems() + return #XXX: or raise? no matches + _globals = _globals.items() # get the enclosing source from .source import getsourcelines try: lines,lnum = getsourcelines(func, enclosing=True) - except: #TypeError, IOError + except Exception: #TypeError, IOError lines,lnum = [],None code = ''.join(lines) # get all possible names,objects that are named in the enclosing source @@ -65,7 +53,7 @@ def outermost(func): # is analogous to getsource(func,enclosing=True) for name,obj in _locals: #XXX: don't really need 'name' try: if getsourcelines(obj) == (lines,lnum): return obj - except: #TypeError, IOError + except Exception: #TypeError, IOError pass return #XXX: or raise? no matches @@ -83,18 +71,12 @@ def nestedcode(func, recurse=True): #XXX: or return dict of {co_name: co} ? return list(nested) def code(func): - '''get the code object for the given function or method + """get the code object for the given function or method NOTE: use dill.source.getsource(CODEOBJ) to get the source code - ''' - if PY3: - im_func = '__func__' - func_code = '__code__' - else: - im_func = 'im_func' - func_code = 'func_code' - if ismethod(func): func = getattr(func, im_func) - if isfunction(func): func = getattr(func, func_code) + """ + if ismethod(func): func = func.__func__ + if isfunction(func): func = func.__code__ if istraceback(func): func = func.tb_frame if isframe(func): func = func.f_code if iscode(func): return func @@ -109,13 +91,6 @@ def referrednested(func, recurse=True): #XXX: return dict of {__name__: obj} ? If possible, python builds code objects, but delays building functions until func() is called. """ - if PY3: - att1 = '__code__' - att0 = '__func__' - else: - att1 = 'func_code' # functions - att0 = 'im_func' # methods - import gc funcs = set() # get the code objects, and try to track down by referrence @@ -123,16 +98,16 @@ def referrednested(func, recurse=True): #XXX: return dict of {__name__: obj} ? # look for function objects that refer to the code object for obj in gc.get_referrers(co): # get methods - _ = getattr(obj, att0, None) # ismethod - if getattr(_, att1, None) is co: funcs.add(obj) + _ = getattr(obj, '__func__', None) # ismethod + if getattr(_, '__code__', None) is co: funcs.add(obj) # get functions - elif getattr(obj, att1, None) is co: funcs.add(obj) + elif getattr(obj, '__code__', None) is co: funcs.add(obj) # get frame objects elif getattr(obj, 'f_code', None) is co: funcs.add(obj) # get code objects elif hasattr(obj, 'co_code') and obj is co: funcs.add(obj) -# frameobjs => func.func_code.co_varnames not in func.func_code.co_cellvars -# funcobjs => func.func_code.co_cellvars not in func.func_code.co_varnames +# frameobjs => func.__code__.co_varnames not in func.__code__.co_cellvars +# funcobjs => func.__code__.co_cellvars not in func.__code__.co_varnames # frameobjs are not found, however funcobjs are... # (see: test_mixins.quad ... and test_mixins.wtf) # after execution, code objects get compiled, and then may be found by gc @@ -143,28 +118,20 @@ def freevars(func): """get objects defined in enclosing code that are referred to by func returns a dict of {name:object}""" - if PY3: - im_func = '__func__' - func_code = '__code__' - func_closure = '__closure__' - else: - im_func = 'im_func' - func_code = 'func_code' - func_closure = 'func_closure' - if ismethod(func): func = getattr(func, im_func) + if ismethod(func): func = func.__func__ if isfunction(func): - closures = getattr(func, func_closure) or () - func = getattr(func, func_code).co_freevars # get freevars + closures = func.__closure__ or () + func = func.__code__.co_freevars # get freevars else: return {} def get_cell_contents(): - for (name,c) in zip(func,closures): + for name, c in zip(func, closures): try: cell_contents = c.cell_contents - except: + except ValueError: # cell is empty continue - yield (name,c.cell_contents) + yield name, c.cell_contents return dict(get_cell_contents()) @@ -175,7 +142,7 @@ def nestedglobals(func, recurse=True): if func is None: return list() import sys from .temp import capture - CAN_NULL = sys.hexversion >= 51052711 #NULL may be prepended >= 3.11a7 + CAN_NULL = sys.hexversion >= 0x30b00a7 # NULL may be prepended >= 3.11a7 names = set() with capture('stdout') as out: dis.dis(func) #XXX: dis.dis(None) disassembles last traceback @@ -199,37 +166,27 @@ def globalvars(func, recurse=True, builtin=False): """get objects defined in global scope that are referred to by func return a dict of {name:object}""" - if PY3: - im_func = '__func__' - func_code = '__code__' - func_globals = '__globals__' - func_closure = '__closure__' - else: - im_func = 'im_func' - func_code = 'func_code' - func_globals = 'func_globals' - func_closure = 'func_closure' - if ismethod(func): func = getattr(func, im_func) + if ismethod(func): func = func.__func__ if isfunction(func): globs = vars(getmodule(sum)).copy() if builtin else {} # get references from within closure orig_func, func = func, set() - for obj in getattr(orig_func, func_closure) or {}: + for obj in orig_func.__closure__ or {}: try: cell_contents = obj.cell_contents - except: + except ValueError: # cell is empty pass else: _vars = globalvars(cell_contents, recurse, builtin) or {} func.update(_vars) #XXX: (above) be wary of infinte recursion? globs.update(_vars) # get globals - globs.update(getattr(orig_func, func_globals) or {}) + globs.update(orig_func.__globals__ or {}) # get names of references if not recurse: - func.update(getattr(orig_func, func_code).co_names) + func.update(orig_func.__code__.co_names) else: - func.update(nestedglobals(getattr(orig_func, func_code))) + func.update(nestedglobals(orig_func.__code__)) # find globals for all entries of func for key in func.copy(): #XXX: unnecessary...? nested_func = globs.get(key) @@ -254,7 +211,7 @@ def globalvars(func, recurse=True, builtin=False): func.update(globalvars(nested_func, True, builtin)) else: return {} - #NOTE: if name not in func_globals, then we skip it... + #NOTE: if name not in __globals__, then we skip it... return dict((name,globs[name]) for name in func if name in globs) diff --git a/dill/logger.py b/dill/logger.py new file mode 100644 index 00000000..e4ac213f --- /dev/null +++ b/dill/logger.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Author: Leonardo Gama (@leogama) +# Copyright (c) 2022 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +""" +Logging utilities for dill. + +The 'logger' object is dill's top-level logger. + +The 'adapter' object wraps the logger and implements a 'trace()' method that +generates a detailed tree-style trace for the pickling call at log level INFO. + +The 'trace()' function sets and resets dill's logger log level, enabling and +disabling the pickling trace. + +The trace shows a tree structure depicting the depth of each object serialized +*with dill save functions*, but not the ones that use save functions from +'pickle._Pickler.dispatch'. If the information is available, it also displays +the size in bytes that the object contributed to the pickle stream (including +its child objects). Sample trace output: + + >>> import dill, dill.tests + >>> dill.detect.trace(True) + >>> dill.dump_session(main=dill.tests) + ┬ M1: + ├┬ F2: + │└ # F2 [32 B] + ├┬ D2: + │├┬ T4: + ││└ # T4 [35 B] + │├┬ D2: + ││├┬ T4: + │││└ # T4 [50 B] + ││├┬ D2: + │││└ # D2 [84 B] + ││└ # D2 [413 B] + │└ # D2 [763 B] + └ # M1 [813 B] +""" + +__all__ = ['adapter', 'logger', 'trace'] + +import codecs +import contextlib +import locale +import logging +import math +import os +from functools import partial +from typing import NoReturn, TextIO, Union + +import dill + +# Tree drawing characters: Unicode to ASCII map. +ASCII_MAP = str.maketrans({"│": "|", "├": "|", "┬": "+", "└": "`"}) + +## Notes about the design choices ## + +# Here is some domumentation of the Standard Library's logging internals that +# can't be found completely in the official documentation. dill's logger is +# obtained by calling logging.getLogger('dill') and therefore is an instance of +# logging.getLoggerClass() at the call time. As this is controlled by the user, +# in order to add some functionality to it it's necessary to use a LoggerAdapter +# to wrap it, overriding some of the adapter's methods and creating new ones. +# +# Basic calling sequence +# ====================== +# +# Python's logging functionality can be conceptually divided into five steps: +# 0. Check logging level -> abort if call level is greater than logger level +# 1. Gather information -> construct a LogRecord from passed arguments and context +# 2. Filter (optional) -> discard message if the record matches a filter +# 3. Format -> format message with args, then format output string with message plus record +# 4. Handle -> write the formatted string to output as defined in the handler +# +# dill.logging.logger.log -> # or logger.info, etc. +# Logger.log -> \ +# Logger._log -> }- accept 'extra' parameter for custom record entries +# Logger.makeRecord -> / +# LogRecord.__init__ +# Logger.handle -> +# Logger.callHandlers -> +# Handler.handle -> +# Filterer.filter -> +# Filter.filter +# StreamHandler.emit -> +# Handler.format -> +# Formatter.format -> +# LogRecord.getMessage # does: record.message = msg % args +# Formatter.formatMessage -> +# PercentStyle.format # does: self._fmt % vars(record) +# +# NOTE: All methods from the second line on are from logging.__init__.py + +class TraceAdapter(logging.LoggerAdapter): + """ + Tracks object tree depth and calculates pickled object size. + + A single instance of this wraps the module's logger, as the logging API + doesn't allow setting it directly with a custom Logger subclass. The added + 'trace()' method receives a pickle instance as the first argument and + creates extra values to be added in the LogRecord from it, then calls + 'info()'. + + Usage of logger with 'trace()' method: + + >>> from dill.logger import adapter as logger #NOTE: not dill.logger.logger + >>> ... + >>> def save_atype(pickler, obj): + >>> logger.trace(pickler, "Message with %s and %r etc. placeholders", 'text', obj) + >>> ... + """ + def __init__(self, logger): + self.logger = logger + def addHandler(self, handler): + formatter = TraceFormatter("%(prefix)s%(message)s%(suffix)s", handler=handler) + handler.setFormatter(formatter) + self.logger.addHandler(handler) + def removeHandler(self, handler): + self.logger.removeHandler(handler) + def process(self, msg, kwargs): + # A no-op override, as we don't have self.extra. + return msg, kwargs + def trace_setup(self, pickler): + # Called by Pickler.dump(). + if not dill._dill.is_dill(pickler, child=False): + return + if self.isEnabledFor(logging.INFO): + pickler._trace_depth = 1 + pickler._size_stack = [] + else: + pickler._trace_depth = None + def trace(self, pickler, msg, *args, **kwargs): + if not hasattr(pickler, '_trace_depth'): + logger.info(msg, *args, **kwargs) + return + if pickler._trace_depth is None: + return + extra = kwargs.get('extra', {}) + pushed_obj = msg.startswith('#') + size = None + try: + # Streams are not required to be tellable. + size = pickler._file.tell() + frame = pickler.framer.current_frame + try: + size += frame.tell() + except AttributeError: + # PyPy may use a BytesBuilder as frame + size += len(frame) + except (AttributeError, TypeError): + pass + if size is not None: + if not pushed_obj: + pickler._size_stack.append(size) + else: + size -= pickler._size_stack.pop() + extra['size'] = size + if pushed_obj: + pickler._trace_depth -= 1 + extra['depth'] = pickler._trace_depth + kwargs['extra'] = extra + self.info(msg, *args, **kwargs) + if not pushed_obj: + pickler._trace_depth += 1 + +class TraceFormatter(logging.Formatter): + """ + Generates message prefix and suffix from record. + + This Formatter adds prefix and suffix strings to the log message in trace + mode (an also provides empty string defaults for normal logs). + """ + def __init__(self, *args, handler=None, **kwargs): + super().__init__(*args, **kwargs) + try: + encoding = handler.stream.encoding + if encoding is None: + raise AttributeError + except AttributeError: + encoding = locale.getpreferredencoding() + try: + encoding = codecs.lookup(encoding).name + except LookupError: + self.is_utf8 = False + else: + self.is_utf8 = (encoding == codecs.lookup('utf-8').name) + def format(self, record): + fields = {'prefix': "", 'suffix': ""} + if getattr(record, 'depth', 0) > 0: + if record.msg.startswith("#"): + prefix = (record.depth - 1)*"│" + "└" + elif record.depth == 1: + prefix = "┬" + else: + prefix = (record.depth - 2)*"│" + "├┬" + if not self.is_utf8: + prefix = prefix.translate(ASCII_MAP) + "-" + fields['prefix'] = prefix + " " + if hasattr(record, 'size'): + # Show object size in human-redable form. + power = int(math.log(record.size, 2)) // 10 + size = record.size >> power*10 + fields['suffix'] = " [%d %sB]" % (size, "KMGTP"[power] + "i" if power else "") + vars(record).update(fields) + return super().format(record) + +logger = logging.getLogger('dill') +adapter = TraceAdapter(logger) +stderr_handler = logging.StreamHandler() +adapter.addHandler(stderr_handler) + +def trace(arg: Union[bool, TextIO, str, os.PathLike] = None, *, mode: str = 'a') -> NoReturn: + """print a trace through the stack when pickling; useful for debugging + + With a single boolean argument, enable or disable the tracing. + + Example usage: + + >>> import dill + >>> dill.detect.trace(True) + >>> dill.dump_session() + + Alternatively, ``trace()`` can be used as a context manager. With no + arguments, it just takes care of restoring the tracing state on exit. + Either a file handle, or a file name and (optionally) a file mode may be + specitfied to redirect the tracing output in the ``with`` block context. A + log function is yielded by the manager so the user can write extra + information to the file. + + Example usage: + + >>> from dill import detect + >>> D = {'a': 42, 'b': {'x': None}} + >>> with detect.trace(): + >>> dumps(D) + ┬ D2: + ├┬ D2: + │└ # D2 [8 B] + └ # D2 [22 B] + >>> squared = lambda x: x**2 + >>> with detect.trace('output.txt', mode='w') as log: + >>> log("> D = %r", D) + >>> dumps(D) + >>> log("> squared = %r", squared) + >>> dumps(squared) + + Arguments: + arg: a boolean value, or an optional file-like or path-like object for the context manager + mode: mode string for ``open()`` if a file name is passed as the first argument + """ + if not isinstance(arg, bool): + return TraceManager(file=arg, mode=mode) + logger.setLevel(logging.INFO if arg else logging.WARNING) + +class TraceManager(contextlib.AbstractContextManager): + """context manager version of trace(); can redirect the trace to a file""" + def __init__(self, file, mode): + self.file = file + self.mode = mode + self.redirect = file is not None + self.file_is_stream = hasattr(file, 'write') + def __enter__(self): + if self.redirect: + stderr_handler.flush() + if self.file_is_stream: + self.handler = logging.StreamHandler(self.file) + else: + self.handler = logging.FileHandler(self.file, self.mode) + adapter.removeHandler(stderr_handler) + adapter.addHandler(self.handler) + self.old_level = adapter.getEffectiveLevel() + adapter.setLevel(logging.INFO) + return adapter.info + def __exit__(self, *exc_info): + adapter.setLevel(self.old_level) + if self.redirect: + adapter.removeHandler(self.handler) + adapter.addHandler(stderr_handler) + if not self.file_is_stream: + self.handler.close() diff --git a/dill/session.py b/dill/session.py new file mode 100644 index 00000000..6a037d61 --- /dev/null +++ b/dill/session.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Author: Leonardo Gama (@leogama) +# Copyright (c) 2008-2015 California Institute of Technology. +# Copyright (c) 2016-2022 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +""" +Pickle and restore the intepreter session. +""" + +__all__ = [ + 'dump_module', 'load_module', 'load_module_asdict', + 'dump_session', 'load_session' # backward compatibility +] + +import re +import sys +import warnings + +from dill import _dill, Pickler, Unpickler +from ._dill import ( + BuiltinMethodType, FunctionType, MethodType, ModuleType, TypeType, + _import_module, _is_builtin_module, _is_imported_module, _main_module, + _reverse_typemap, __builtin__, +) + +# Type hints. +from typing import Optional, Union + +import pathlib +import tempfile + +TEMPDIR = pathlib.PurePath(tempfile.gettempdir()) + +def _module_map(): + """get map of imported modules""" + from collections import defaultdict + from types import SimpleNamespace + modmap = SimpleNamespace( + by_name=defaultdict(list), + by_id=defaultdict(list), + top_level={}, + ) + for modname, module in sys.modules.items(): + if modname in ('__main__', '__mp_main__') or not isinstance(module, ModuleType): + continue + if '.' not in modname: + modmap.top_level[id(module)] = modname + for objname, modobj in module.__dict__.items(): + modmap.by_name[objname].append((modobj, modname)) + modmap.by_id[id(modobj)].append((modobj, objname, modname)) + return modmap + +IMPORTED_AS_TYPES = (ModuleType, TypeType, FunctionType, MethodType, BuiltinMethodType) +if 'PyCapsuleType' in _reverse_typemap: + IMPORTED_AS_TYPES += (_reverse_typemap['PyCapsuleType'],) +IMPORTED_AS_MODULES = ('ctypes', 'typing', 'subprocess', 'threading', + r'concurrent\.futures(\.\w+)?', r'multiprocessing(\.\w+)?') +IMPORTED_AS_MODULES = tuple(re.compile(x) for x in IMPORTED_AS_MODULES) + +def _lookup_module(modmap, name, obj, main_module): + """lookup name or id of obj if module is imported""" + for modobj, modname in modmap.by_name[name]: + if modobj is obj and sys.modules[modname] is not main_module: + return modname, name + __module__ = getattr(obj, '__module__', None) + if isinstance(obj, IMPORTED_AS_TYPES) or (__module__ is not None + and any(regex.fullmatch(__module__) for regex in IMPORTED_AS_MODULES)): + for modobj, objname, modname in modmap.by_id[id(obj)]: + if sys.modules[modname] is not main_module: + return modname, objname + return None, None + +def _stash_modules(main_module): + modmap = _module_map() + newmod = ModuleType(main_module.__name__) + + imported = [] + imported_as = [] + imported_top_level = [] # keep separated for backward compatibility + original = {} + for name, obj in main_module.__dict__.items(): + if obj is main_module: + original[name] = newmod # self-reference + elif obj is main_module.__dict__: + original[name] = newmod.__dict__ + # Avoid incorrectly matching a singleton value in another package (ex.: __doc__). + elif any(obj is singleton for singleton in (None, False, True)) \ + or isinstance(obj, ModuleType) and _is_builtin_module(obj): # always saved by ref + original[name] = obj + else: + source_module, objname = _lookup_module(modmap, name, obj, main_module) + if source_module is not None: + if objname == name: + imported.append((source_module, name)) + else: + imported_as.append((source_module, objname, name)) + else: + try: + imported_top_level.append((modmap.top_level[id(obj)], name)) + except KeyError: + original[name] = obj + + if len(original) < len(main_module.__dict__): + newmod.__dict__.update(original) + newmod.__dill_imported = imported + newmod.__dill_imported_as = imported_as + newmod.__dill_imported_top_level = imported_top_level + if getattr(newmod, '__loader__', None) is None and _is_imported_module(main_module): + # Trick _is_imported_module() to force saving as an imported module. + newmod.__loader__ = True # will be discarded by save_module() + return newmod + else: + return main_module + +def _restore_modules(unpickler, main_module): + try: + for modname, name in main_module.__dict__.pop('__dill_imported'): + main_module.__dict__[name] = unpickler.find_class(modname, name) + for modname, objname, name in main_module.__dict__.pop('__dill_imported_as'): + main_module.__dict__[name] = unpickler.find_class(modname, objname) + for modname, name in main_module.__dict__.pop('__dill_imported_top_level'): + main_module.__dict__[name] = __import__(modname) + except KeyError: + pass + +#NOTE: 06/03/15 renamed main_module to main +def dump_module( + filename = str(TEMPDIR/'session.pkl'), + module: Union[ModuleType, str] = None, + refimported: bool = False, + **kwds +) -> None: + """Pickle the current state of :py:mod:`__main__` or another module to a file. + + Save the contents of :py:mod:`__main__` (e.g. from an interactive + interpreter session), an imported module, or a module-type object (e.g. + built with :py:class:`~types.ModuleType`), to a file. The pickled + module can then be restored with the function :py:func:`load_module`. + + Parameters: + filename: a path-like object or a writable stream. + module: a module object or the name of an importable module. If `None` + (the default), :py:mod:`__main__` is saved. + refimported: if `True`, all objects identified as having been imported + into the module's namespace are saved by reference. *Note:* this is + similar but independent from ``dill.settings[`byref`]``, as + ``refimported`` refers to virtually all imported objects, while + ``byref`` only affects select objects. + **kwds: extra keyword arguments passed to :py:class:`Pickler()`. + + Raises: + :py:exc:`PicklingError`: if pickling fails. + + Examples: + + - Save current interpreter session state: + + >>> import dill + >>> squared = lambda x: x*x + >>> dill.dump_module() # save state of __main__ to /tmp/session.pkl + + - Save the state of an imported/importable module: + + >>> import dill + >>> import pox + >>> pox.plus_one = lambda x: x+1 + >>> dill.dump_module('pox_session.pkl', module=pox) + + - Save the state of a non-importable, module-type object: + + >>> import dill + >>> from types import ModuleType + >>> foo = ModuleType('foo') + >>> foo.values = [1,2,3] + >>> import math + >>> foo.sin = math.sin + >>> dill.dump_module('foo_session.pkl', module=foo, refimported=True) + + - Restore the state of the saved modules: + + >>> import dill + >>> dill.load_module() + >>> squared(2) + 4 + >>> pox = dill.load_module('pox_session.pkl') + >>> pox.plus_one(1) + 2 + >>> foo = dill.load_module('foo_session.pkl') + >>> [foo.sin(x) for x in foo.values] + [0.8414709848078965, 0.9092974268256817, 0.1411200080598672] + + *Changed in version 0.3.6:* Function ``dump_session()`` was renamed to + ``dump_module()``. Parameters ``main`` and ``byref`` were renamed to + ``module`` and ``refimported``, respectively. + + Note: + Currently, ``dill.settings['byref']`` and ``dill.settings['recurse']`` + don't apply to this function.` + """ + for old_par, par in [('main', 'module'), ('byref', 'refimported')]: + if old_par in kwds: + message = "The argument %r has been renamed %r" % (old_par, par) + if old_par == 'byref': + message += " to distinguish it from dill.settings['byref']" + warnings.warn(message + ".", PendingDeprecationWarning) + if locals()[par]: # the defaults are None and False + raise TypeError("both %r and %r arguments were used" % (par, old_par)) + refimported = kwds.pop('byref', refimported) + module = kwds.pop('main', module) + + from .settings import settings + protocol = settings['protocol'] + main = module + if main is None: + main = _main_module + elif isinstance(main, str): + main = _import_module(main) + if not isinstance(main, ModuleType): + raise TypeError("%r is not a module" % main) + if hasattr(filename, 'write'): + file = filename + else: + file = open(filename, 'wb') + try: + pickler = Pickler(file, protocol, **kwds) + pickler._original_main = main + if refimported: + main = _stash_modules(main) + pickler._main = main #FIXME: dill.settings are disabled + pickler._byref = False # disable pickling by name reference + pickler._recurse = False # disable pickling recursion for globals + pickler._session = True # is best indicator of when pickling a session + pickler._first_pass = True + pickler._main_modified = main is not pickler._original_main + pickler.dump(main) + finally: + if file is not filename: # if newly opened file + file.close() + return + +# Backward compatibility. +def dump_session(filename=str(TEMPDIR/'session.pkl'), main=None, byref=False, **kwds): + warnings.warn("dump_session() has been renamed dump_module()", PendingDeprecationWarning) + dump_module(filename, module=main, refimported=byref, **kwds) +dump_session.__doc__ = dump_module.__doc__ + +class _PeekableReader: + """lightweight stream wrapper that implements peek()""" + def __init__(self, stream): + self.stream = stream + def read(self, n): + return self.stream.read(n) + def readline(self): + return self.stream.readline() + def tell(self): + return self.stream.tell() + def close(self): + return self.stream.close() + def peek(self, n): + stream = self.stream + try: + if hasattr(stream, 'flush'): stream.flush() + position = stream.tell() + stream.seek(position) # assert seek() works before reading + chunk = stream.read(n) + stream.seek(position) + return chunk + except (AttributeError, OSError): + raise NotImplementedError("stream is not peekable: %r", stream) from None + +def _make_peekable(stream): + """return stream as an object with a peek() method""" + import io + if hasattr(stream, 'peek'): + return stream + if not (hasattr(stream, 'tell') and hasattr(stream, 'seek')): + try: + return io.BufferedReader(stream) + except Exception: + pass + return _PeekableReader(stream) + +def _identify_module(file, main=None): + """identify the name of the module stored in the given file-type object""" + from pickletools import genops + UNICODE = {'UNICODE', 'BINUNICODE', 'SHORT_BINUNICODE'} + found_import = False + try: + for opcode, arg, pos in genops(file.peek(256)): + if not found_import: + if opcode.name in ('GLOBAL', 'SHORT_BINUNICODE') and \ + arg.endswith('_import_module'): + found_import = True + else: + if opcode.name in UNICODE: + return arg + else: + raise UnpicklingError("reached STOP without finding main module") + except (NotImplementedError, ValueError) as error: + # ValueError occours when the end of the chunk is reached (without a STOP). + if isinstance(error, NotImplementedError) and main is not None: + # file is not peekable, but we have main. + return None + raise UnpicklingError("unable to identify main module") from error + +def load_module( + filename = str(TEMPDIR/'session.pkl'), + module: Union[ModuleType, str] = None, + **kwds +) -> Optional[ModuleType]: + """Update the selected module (default is :py:mod:`__main__`) with + the state saved at ``filename``. + + Restore a module to the state saved with :py:func:`dump_module`. The + saved module can be :py:mod:`__main__` (e.g. an interpreter session), + an imported module, or a module-type object (e.g. created with + :py:class:`~types.ModuleType`). + + When restoring the state of a non-importable module-type object, the + current instance of this module may be passed as the argument ``main``. + Otherwise, a new instance is created with :py:class:`~types.ModuleType` + and returned. + + Parameters: + filename: a path-like object or a readable stream. + module: a module object or the name of an importable module; + the module name and kind (i.e. imported or non-imported) must + match the name and kind of the module stored at ``filename``. + **kwds: extra keyword arguments passed to :py:class:`Unpickler()`. + + Raises: + :py:exc:`UnpicklingError`: if unpickling fails. + :py:exc:`ValueError`: if the argument ``main`` and module saved + at ``filename`` are incompatible. + + Returns: + A module object, if the saved module is not :py:mod:`__main__` or + a module instance wasn't provided with the argument ``main``. + + Examples: + + - Save the state of some modules: + + >>> import dill + >>> squared = lambda x: x*x + >>> dill.dump_module() # save state of __main__ to /tmp/session.pkl + >>> + >>> import pox # an imported module + >>> pox.plus_one = lambda x: x+1 + >>> dill.dump_module('pox_session.pkl', module=pox) + >>> + >>> from types import ModuleType + >>> foo = ModuleType('foo') # a module-type object + >>> foo.values = [1,2,3] + >>> import math + >>> foo.sin = math.sin + >>> dill.dump_module('foo_session.pkl', module=foo, refimported=True) + + - Restore the state of the interpreter: + + >>> import dill + >>> dill.load_module() # updates __main__ from /tmp/session.pkl + >>> squared(2) + 4 + + - Load the saved state of an importable module: + + >>> import dill + >>> pox = dill.load_module('pox_session.pkl') + >>> pox.plus_one(1) + 2 + >>> import sys + >>> pox in sys.modules.values() + True + + - Load the saved state of a non-importable module-type object: + + >>> import dill + >>> foo = dill.load_module('foo_session.pkl') + >>> [foo.sin(x) for x in foo.values] + [0.8414709848078965, 0.9092974268256817, 0.1411200080598672] + >>> import math + >>> foo.sin is math.sin # foo.sin was saved by reference + True + >>> import sys + >>> foo in sys.modules.values() + False + + - Update the state of a non-importable module-type object: + + >>> import dill + >>> from types import ModuleType + >>> foo = ModuleType('foo') + >>> foo.values = ['a','b'] + >>> foo.sin = lambda x: x*x + >>> dill.load_module('foo_session.pkl', module=foo) + >>> [foo.sin(x) for x in foo.values] + [0.8414709848078965, 0.9092974268256817, 0.1411200080598672] + + *Changed in version 0.3.6:* Function ``load_session()`` was renamed to + ``load_module()``. Parameter ``main`` was renamed to ``module``. + + See also: + :py:func:`load_module_asdict` to load the contents of module saved + with :py:func:`dump_module` into a dictionary. + """ + if 'main' in kwds: + warnings.warn( + "The argument 'main' has been renamed 'module'.", + PendingDeprecationWarning + ) + if module is not None: + raise TypeError("both 'module' and 'main' arguments were used") + module = kwds.pop('main') + main = module + if hasattr(filename, 'read'): + file = filename + else: + file = open(filename, 'rb') + try: + file = _make_peekable(file) + #FIXME: dill.settings are disabled + unpickler = Unpickler(file, **kwds) + unpickler._session = True + + # Resolve unpickler._main + pickle_main = _identify_module(file, main) + if main is None and pickle_main is not None: + main = pickle_main + if isinstance(main, str): + if main.startswith('__runtime__.'): + # Create runtime module to load the session into. + main = ModuleType(main.partition('.')[-1]) + else: + main = _import_module(main) + if main is not None: + if not isinstance(main, ModuleType): + raise TypeError("%r is not a module" % main) + unpickler._main = main + else: + main = unpickler._main + + # Check against the pickle's main. + is_main_imported = _is_imported_module(main) + if pickle_main is not None: + is_runtime_mod = pickle_main.startswith('__runtime__.') + if is_runtime_mod: + pickle_main = pickle_main.partition('.')[-1] + error_msg = "can't update{} module{} %r with the saved state of{} module{} %r" + if is_runtime_mod and is_main_imported: + raise ValueError( + error_msg.format(" imported", "", "", "-type object") + % (main.__name__, pickle_main) + ) + if not is_runtime_mod and not is_main_imported: + raise ValueError( + error_msg.format("", "-type object", " imported", "") + % (pickle_main, main.__name__) + ) + if main.__name__ != pickle_main: + raise ValueError(error_msg.format("", "", "", "") % (main.__name__, pickle_main)) + + # This is for find_class() to be able to locate it. + if not is_main_imported: + runtime_main = '__runtime__.%s' % main.__name__ + sys.modules[runtime_main] = main + + loaded = unpickler.load() + finally: + if not hasattr(filename, 'read'): # if newly opened file + file.close() + try: + del sys.modules[runtime_main] + except (KeyError, NameError): + pass + assert loaded is main + _restore_modules(unpickler, main) + if main is _main_module or main is module: + return None + else: + return main + +# Backward compatibility. +def load_session(filename=str(TEMPDIR/'session.pkl'), main=None, **kwds): + warnings.warn("load_session() has been renamed load_module().", PendingDeprecationWarning) + load_module(filename, module=main, **kwds) +load_session.__doc__ = load_module.__doc__ + +def load_module_asdict( + filename = str(TEMPDIR/'session.pkl'), + update: bool = False, + **kwds +) -> dict: + """ + Load the contents of a saved module into a dictionary. + + ``load_module_asdict()`` is the near-equivalent of:: + + lambda filename: vars(dill.load_module(filename)).copy() + + however, does not alter the original module. Also, the path of + the loaded module is stored in the ``__session__`` attribute. + + Parameters: + filename: a path-like object or a readable stream + update: if `True`, initialize the dictionary with the current state + of the module prior to loading the state stored at filename. + **kwds: extra keyword arguments passed to :py:class:`Unpickler()` + + Raises: + :py:exc:`UnpicklingError`: if unpickling fails + + Returns: + A copy of the restored module's dictionary. + + Note: + If ``update`` is True, the corresponding module may first be imported + into the current namespace before the saved state is loaded from + filename to the dictionary. Note that any module that is imported into + the current namespace as a side-effect of using ``update`` will not be + modified by loading the saved module in filename to a dictionary. + + Example: + >>> import dill + >>> alist = [1, 2, 3] + >>> anum = 42 + >>> dill.dump_module() + >>> anum = 0 + >>> new_var = 'spam' + >>> main = dill.load_module_asdict() + >>> main['__name__'], main['__session__'] + ('__main__', '/tmp/session.pkl') + >>> main is globals() # loaded objects don't reference globals + False + >>> main['alist'] == alist + True + >>> main['alist'] is alist # was saved by value + False + >>> main['anum'] == anum # changed after the session was saved + False + >>> new_var in main # would be True if the option 'update' was set + False + """ + if 'module' in kwds: + raise TypeError("'module' is an invalid keyword argument for load_module_asdict()") + if hasattr(filename, 'read'): + file = filename + else: + file = open(filename, 'rb') + try: + file = _make_peekable(file) + main_name = _identify_module(file) + old_main = sys.modules.get(main_name) + main = ModuleType(main_name) + if update: + if old_main is None: + old_main = _import_module(main_name) + main.__dict__.update(old_main.__dict__) + else: + main.__builtins__ = __builtin__ + sys.modules[main_name] = main + load_module(file, **kwds) + finally: + if not hasattr(filename, 'read'): # if newly opened file + file.close() + try: + if old_main is None: + del sys.modules[main_name] + else: + sys.modules[main_name] = old_main + except NameError: # failed before setting old_main + pass + main.__session__ = str(filename) + return main.__dict__ + + +# Internal exports for backward compatibility with dill v0.3.5.1 +# Can't be placed in dill._dill because of circular import problems. +for name in ( + '_lookup_module', '_module_map', '_restore_modules', '_stash_modules', + 'dump_session', 'load_session' # backward compatibility functions +): + setattr(_dill, name, globals()[name]) +del name diff --git a/dill/settings.py b/dill/settings.py index 4d0226b0..b105d2e8 100644 --- a/dill/settings.py +++ b/dill/settings.py @@ -9,10 +9,7 @@ global settings for Pickler """ -try: - from pickle import DEFAULT_PROTOCOL -except ImportError: - from pickle import HIGHEST_PROTOCOL as DEFAULT_PROTOCOL +from pickle import DEFAULT_PROTOCOL settings = { #'main' : None, diff --git a/dill/source.py b/dill/source.py index 47064a16..229a3575 100644 --- a/dill/source.py +++ b/dill/source.py @@ -12,7 +12,7 @@ """ Extensions to python's 'inspect' module, which can be used to retrieve information from live python objects. The methods -defined in this module are augmented to facilitate access to +defined in this module are augmented to facilitate access to source code of interactively defined functions and classes, as well as provide access to source code for objects defined in a file. @@ -29,7 +29,7 @@ ismodule, istraceback) from tokenize import TokenError -from ._dill import PY3 +from ._dill import IS_IPYTHON def isfrommain(obj): @@ -43,7 +43,7 @@ def isfrommain(obj): def isdynamic(obj): "check if object was built in the interpreter" try: file = getfile(obj) - except TypeError: file = None + except TypeError: file = None if file == '' and isfrommain(obj): return True return False @@ -58,7 +58,7 @@ def _matchlambda(func, line): lhs,rhs = line.split('lambda ',1)[-1].split(":", 1) #FIXME: if !1 inputs try: #FIXME: unsafe _ = eval("lambda %s : %s" % (lhs,rhs), globals(),locals()) - except: _ = dummy + except Exception: _ = dummy # get code objects, for comparison _, code = getcode(_).co_code, getcode(func).co_code # check if func is in closure @@ -80,7 +80,7 @@ def _matchlambda(func, line): _lhs,_rhs = rhs.split('lambda ',1)[-1].split(":",1) #FIXME: if !1 inputs try: #FIXME: unsafe _f = eval("lambda %s : %s" % (_lhs,_rhs), globals(),locals()) - except: _f = dummy + except Exception: _f = dummy # get code objects, for comparison _, code = getcode(_f).co_code, getcode(func).co_code if len(_) != len(code): return False @@ -114,13 +114,35 @@ def findsource(object): module = getmodule(object) try: file = getfile(module) - except TypeError: file = None + except TypeError: file = None + is_module_main = (module and module.__name__ == '__main__' and not file) + if IS_IPYTHON and is_module_main: + #FIXME: quick fix for functions and classes in IPython interpreter + try: + file = getfile(object) + sourcefile = getsourcefile(object) + except TypeError: + if isclass(object): + for object_method in filter(isfunction, object.__dict__.values()): + # look for a method of the class + file_candidate = getfile(object_method) + if not file_candidate.startswith('': pat1 = r'(.*(?': pat1 = r'(.*(? indent or spaces < 0: spaces = indent for i in range(len(lines) if all else 1): #FIXME: works... but shouldn't outdent 2nd+ lines of multiline doc @@ -503,7 +522,7 @@ def _outdent(lines, spaces=None, all=True): def outdent(code, spaces=None, all=True): '''outdent a block of code (default is to strip all leading whitespace)''' - indent = indentsize(code) + indent = indentsize(code) if spaces is None or spaces > indent or spaces < 0: spaces = indent #XXX: will this delete '\n' in some cases? if not all: return code[spaces:] @@ -511,42 +530,20 @@ def outdent(code, spaces=None, all=True): #XXX: not sure what the point of _wrap is... -#exec_ = lambda s, *a: eval(compile(s, '', 'exec'), *a) __globals__ = globals() __locals__ = locals() -wrap2 = ''' -def _wrap(f): - """ encapsulate a function and it's __import__ """ - def func(*args, **kwds): - try: - # _ = eval(getsource(f, force=True)) #XXX: safer but less robust - exec getimportable(f, alias='_') in %s, %s - except: - raise ImportError('cannot import name ' + f.__name__) - return _(*args, **kwds) - func.__name__ = f.__name__ - func.__doc__ = f.__doc__ - return func -''' % ('__globals__', '__locals__') -wrap3 = ''' def _wrap(f): """ encapsulate a function and it's __import__ """ def func(*args, **kwds): try: # _ = eval(getsource(f, force=True)) #XXX: safer but less robust - exec(getimportable(f, alias='_'), %s, %s) - except: + exec(getimportable(f, alias='_'), __globals__, __locals__) + except Exception: raise ImportError('cannot import name ' + f.__name__) return _(*args, **kwds) func.__name__ = f.__name__ func.__doc__ = f.__doc__ return func -''' % ('__globals__', '__locals__') -if PY3: - exec(wrap3) -else: - exec(wrap2) -del wrap2, wrap3 def _enclose(object, alias=''): #FIXME: needs alias to hold returned object @@ -581,17 +578,14 @@ def dumpsource(object, alias='', new=False, enclose=True): else: stub = alias pre = '%s = ' % stub if alias else alias - + # if a 'new' instance is not needed, then just dump and load if not new or not _isinstance(object): code += pre + 'dill.loads(%s)\n' % pik else: #XXX: other cases where source code is needed??? code += getsource(object.__class__, alias='', lstrip=True, force=True) mod = repr(object.__module__) # should have a module (no builtins here) - if PY3: - code += pre + 'dill.loads(%s.replace(b%s,bytes(__name__,"UTF-8")))\n' % (pik,mod) - else: - code += pre + 'dill.loads(%s.replace(%s,__name__))\n' % (pik,mod) + code += pre + 'dill.loads(%s.replace(b%s,bytes(__name__,"UTF-8")))\n' % (pik,mod) #code += 'del %s' % object.__class__.__name__ #NOTE: kills any existing! if enclose: @@ -654,7 +648,7 @@ def _namespace(obj): if module in ['builtins','__builtin__']: # BuiltinFunctionType if _intypes(name): return ['types'] + [name] return qual + [name] #XXX: can be wrong for some aliased objects - except: pass + except Exception: pass # special case: numpy.inf and numpy.nan (we don't want them as floats) if str(obj) in ['inf','nan','Inf','NaN']: # is more, but are they needed? return ['numpy'] + [str(obj)] @@ -742,7 +736,7 @@ def getimport(obj, alias='', verify=True, builtin=False, enclosing=False): try: # look for '<...>' and be mindful it might be in lists, dicts, etc... name = repr(obj).split('<',1)[1].split('>',1)[1] name = None # we have a 'object'-style repr - except: # it's probably something 'importable' + except Exception: # it's probably something 'importable' if head in ['builtins','__builtin__']: name = repr(obj) #XXX: catch [1,2], (1,2), set([1,2])... others? else: @@ -800,7 +794,7 @@ def _importable(obj, alias='', source=None, enclosing=False, force=True, \ try: return getsource(obj, alias, enclosing=enclosing, \ force=force, lstrip=lstrip, builtin=builtin) - except: pass + except Exception: pass try: if not _isinstance(obj): return getimport(obj, alias, enclosing=enclosing, \ @@ -815,12 +809,12 @@ def _importable(obj, alias='', source=None, enclosing=False, force=True, \ if alias == name: _alias = "" return _import+_alias+"%s\n" % name - except: pass + except Exception: pass if not source: # try getsource, only if it hasn't been tried yet try: return getsource(obj, alias, enclosing=enclosing, \ force=force, lstrip=lstrip, builtin=builtin) - except: pass + except Exception: pass # get the name (of functions, lambdas, and classes) # or hope that obj can be built from the __repr__ #XXX: what to do about class instances and such? @@ -859,7 +853,7 @@ def _closuredimport(func, alias='', builtin=False): re.match(pat, line)] if not candidate: mod = getname(getmodule(fobj)) - #HACK: get file containing 'inner' function; is func there? + #HACK: get file containing 'inner' function; is func there? lines,_ = findsource(fobj) candidate = [line for line in lines \ if getname(fobj) in line and re.match(pat, line)] @@ -960,7 +954,7 @@ def importable(obj, alias='', source=None, builtin=True): if len(src) > 1: raise NotImplementedError('not implemented') return list(src.values())[0] - except: + except Exception: if tried_source: raise tried_import = True # we want the source @@ -999,7 +993,7 @@ def _code_stitcher(block): if not obj: return src if not src: return obj return obj + src - except: + except Exception: if tried_import: raise tried_source = True source = not source diff --git a/dill/temp.py b/dill/temp.py index 251a8e30..215ba63a 100644 --- a/dill/temp.py +++ b/dill/temp.py @@ -17,7 +17,6 @@ 'capture'] import contextlib -from ._dill import PY3 @contextlib.contextmanager @@ -32,10 +31,7 @@ def capture(stream='stdout'): """ import sys - if PY3: - from io import StringIO - else: - from StringIO import StringIO + from io import StringIO orig = getattr(sys, stream) setattr(sys, stream, StringIO()) try: @@ -176,10 +172,7 @@ def loadIO(buffer, **kwds): [1, 2, 3, 4, 5] """ import dill as pickle - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO + from io import BytesIO as StringIO value = getattr(buffer, 'getvalue', buffer) # value or buffer.getvalue if value != buffer: value = value() # buffer.getvalue() return pickle.load(StringIO(value)) @@ -193,10 +186,7 @@ def dumpIO(object, **kwds): [1, 2, 3, 4, 5] """ import dill as pickle - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO + from io import BytesIO as StringIO file = StringIO() pickle.dump(object, file) file.flush() @@ -217,7 +207,7 @@ def loadIO_source(buffer, **kwds): alias = kwds.pop('alias', None) source = getattr(buffer, 'getvalue', buffer) # source or buffer.getvalue if source != buffer: source = source() # buffer.getvalue() - if PY3: source = source.decode() # buffer to string + source = source.decode() # buffer to string if not alias: tag = source.strip().splitlines()[-1].split() if tag[0] != '#NAME:': @@ -243,10 +233,7 @@ def dumpIO_source(object, **kwds): If 'alias' is specified, the object will be renamed to the given string. """ from .source import importable, getname - if PY3: - from io import BytesIO as StringIO - else: - from StringIO import StringIO + from io import BytesIO as StringIO alias = kwds.pop('alias', '') #XXX: include an alias so a name is known name = str(alias) or getname(object) name = "\n#NAME: %s\n" % name diff --git a/tests/__init__.py b/dill/tests/__init__.py similarity index 100% rename from tests/__init__.py rename to dill/tests/__init__.py diff --git a/tests/__main__.py b/dill/tests/__main__.py similarity index 66% rename from tests/__main__.py rename to dill/tests/__main__.py index e82993c6..b68e8677 100644 --- a/tests/__main__.py +++ b/dill/tests/__main__.py @@ -5,17 +5,17 @@ # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/dill/blob/master/LICENSE -from __future__ import print_function import glob import os +import sys +import subprocess as sp +python = sys.executable try: import pox - python = pox.which_python(version=True, fullpath=False) or 'python' + python = pox.which_python(version=True) or python except ImportError: - python = 'python' -import subprocess as sp -from sys import platform -shell = platform[:3] == 'win' + pass +shell = sys.platform[:3] == 'win' suite = os.path.dirname(__file__) or os.path.curdir tests = glob.glob(suite + os.path.sep + 'test_*.py') @@ -23,8 +23,13 @@ if __name__ == '__main__': + failed = 0 for test in tests: p = sp.Popen([python, test], shell=shell).wait() - if not p: - print('.', end='') + if p: + print('F', end='', flush=True) + failed = 1 + else: + print('.', end='', flush=True) print('') + exit(failed) diff --git a/tests/test_abc.py b/dill/tests/test_abc.py similarity index 93% rename from tests/test_abc.py rename to dill/tests/test_abc.py index e0cab11d..a90c73e4 100644 --- a/tests/test_abc.py +++ b/dill/tests/test_abc.py @@ -4,20 +4,13 @@ """ import dill import abc +from abc import ABC from types import FunctionType dill.settings['recurse'] = True -if dill._dill.PY3: - ABC = abc.ABC -else: - ABC = object - class OneTwoThree(ABC): - if not dill._dill.PY3: - __metaclass__ = abc.ABCMeta - @abc.abstractmethod def foo(self): """A method""" @@ -91,9 +84,6 @@ def test_abc_local(): Test using locally scoped ABC class """ class LocalABC(ABC): - if not dill._dill.PY3: - __metaclass__ = abc.ABCMeta - @abc.abstractmethod def foo(self): pass @@ -129,8 +119,7 @@ def baz(self): labc2, pik = dill.copy((labc, Real())) assert 'Real' == type(pik).__name__ - if dill._dill.PY3: - assert '.Real' in type(pik).__qualname__ + assert '.Real' in type(pik).__qualname__ assert type(pik) is not Real assert labc2 is not LocalABC assert labc2 is not labc diff --git a/tests/test_check.py b/dill/tests/test_check.py similarity index 71% rename from tests/test_check.py rename to dill/tests/test_check.py index 134b8b0c..0a22b276 100644 --- a/tests/test_check.py +++ b/dill/tests/test_check.py @@ -10,7 +10,6 @@ import sys from dill.temp import capture -from dill._dill import PY3 #FIXME: this doesn't catch output... it's from the internal call @@ -30,24 +29,24 @@ def raise_check(func, **kwds): f = lambda x:x**2 -def test_simple(): - raise_check(f) +def test_simple(verbose=None): + raise_check(f, verbose=verbose) -def test_recurse(): - raise_check(f, recurse=True) +def test_recurse(verbose=None): + raise_check(f, recurse=True, verbose=verbose) -def test_byref(): - raise_check(f, byref=True) +def test_byref(verbose=None): + raise_check(f, byref=True, verbose=verbose) -def test_protocol(): - raise_check(f, protocol=True) +def test_protocol(verbose=None): + raise_check(f, protocol=True, verbose=verbose) -def test_python(): - raise_check(f, python=None) +def test_python(verbose=None): + raise_check(f, python=None, verbose=verbose) #TODO: test incompatible versions diff --git a/tests/test_classdef.py b/dill/tests/test_classdef.py similarity index 71% rename from tests/test_classdef.py rename to dill/tests/test_classdef.py index c6522ebc..e29ee694 100644 --- a/tests/test_classdef.py +++ b/dill/tests/test_classdef.py @@ -54,11 +54,11 @@ def ok(self): nc = _newclass2() m = _mclass() -if sys.hexversion >= 0x03070000: +if sys.hexversion < 0x03090000: import typing class customIntList(typing.List[int]): pass -elif sys.hexversion >= 0x03090000: +else: class customIntList(list[int]): pass @@ -98,19 +98,15 @@ def test_specialtypes(): assert dill.pickles(type(NotImplemented)) assert dill.pickles(type(Ellipsis)) -if hex(sys.hexversion) >= '0x20600f0': - from collections import namedtuple - Z = namedtuple("Z", ['a','b']) - Zi = Z(0,1) - X = namedtuple("Y", ['a','b']) - X.__name__ = "X" - if hex(sys.hexversion) >= '0x30300f0': - X.__qualname__ = "X" #XXX: name must 'match' or fails to pickle - Xi = X(0,1) - Bad = namedtuple("FakeName", ['a','b']) - Badi = Bad(0,1) -else: - Z = Zi = X = Xi = Bad = Badi = None +from collections import namedtuple +Z = namedtuple("Z", ['a','b']) +Zi = Z(0,1) +X = namedtuple("Y", ['a','b']) +X.__name__ = "X" +X.__qualname__ = "X" #XXX: name must 'match' or fails to pickle +Xi = X(0,1) +Bad = namedtuple("FakeName", ['a','b']) +Badi = Bad(0,1) # test namedtuple def test_namedtuple(): @@ -131,8 +127,7 @@ class B(namedtuple("C", ["one", "two"])): assert dill.copy(a) assert dill.copy(A.B).__name__ == 'B' - if dill._dill.PY3: - assert dill.copy(A.B).__qualname__.endswith('..A.B') + assert dill.copy(A.B).__qualname__.endswith('..A.B') assert dill.copy(A.B).__doc__ == 'docstring' assert dill.copy(A.B).__module__ == 'testing' @@ -176,12 +171,12 @@ def __getnewargs__(self): return np.asarray(self), self.color a1 = TestArray(np.zeros(100), color='green') - if dill._dill.PY3 and not dill._dill.IS_PYPY: + if not dill._dill.IS_PYPY: assert dill.pickles(a1) assert a1.__dict__ == dill.copy(a1).__dict__ a2 = a1[0:9] - if dill._dill.PY3 and not dill._dill.IS_PYPY: + if not dill._dill.IS_PYPY: assert dill.pickles(a2) assert a2.__dict__ == dill.copy(a2).__dict__ @@ -190,7 +185,7 @@ class TestArray2(np.ndarray): a3 = TestArray2([1,2,3,4,5]) a3.color = 'green' - if dill._dill.PY3 and not dill._dill.IS_PYPY: + if not dill._dill.IS_PYPY: assert dill.pickles(a3) assert a3.__dict__ == dill.copy(a3).__dict__ @@ -211,7 +206,7 @@ def test(cls): # test slots class Y(object): - __slots__ = ['y'] + __slots__ = ('y', '__weakref__') def __init__(self, y): self.y = y @@ -225,8 +220,7 @@ def test_slots(): assert dill.copy(y).y == value def test_origbases(): - if sys.hexversion >= 0x03070000: - assert dill.copy(customIntList).__orig_bases__ == customIntList.__orig_bases__ + assert dill.copy(customIntList).__orig_bases__ == customIntList.__orig_bases__ def test_attr(): import attr @@ -238,37 +232,21 @@ class A: assert dill.copy(v) == v def test_metaclass(): - if dill._dill.PY3: - class metaclass_with_new(type): - def __new__(mcls, name, bases, ns, **kwds): - cls = super().__new__(mcls, name, bases, ns, **kwds) - assert mcls is not None - assert cls.method(mcls) - return cls - def method(cls, mcls): - return isinstance(cls, mcls) - - l = locals() - exec("""class subclass_with_new(metaclass=metaclass_with_new): - def __new__(cls): - self = super().__new__(cls) - return self""", None, l) - subclass_with_new = l['subclass_with_new'] - else: - class metaclass_with_new(type): - def __new__(mcls, name, bases, ns, **kwds): - cls = super(mcls, metaclass_with_new).__new__(mcls, name, bases, ns, **kwds) - assert mcls is not None - assert cls.method(mcls) - return cls - def method(cls, mcls): - return isinstance(cls, mcls) - - class subclass_with_new: - __metaclass__ = metaclass_with_new - def __new__(cls): - self = super(subclass_with_new, cls).__new__(cls) - return self + class metaclass_with_new(type): + def __new__(mcls, name, bases, ns, **kwds): + cls = super().__new__(mcls, name, bases, ns, **kwds) + assert mcls is not None + assert cls.method(mcls) + return cls + def method(cls, mcls): + return isinstance(cls, mcls) + + l = locals() + exec("""class subclass_with_new(metaclass=metaclass_with_new): + def __new__(cls): + self = super().__new__(cls) + return self""", None, l) + subclass_with_new = l['subclass_with_new'] assert dill.copy(subclass_with_new()) diff --git a/dill/tests/test_dataclasses.py b/dill/tests/test_dataclasses.py new file mode 100644 index 00000000..15351885 --- /dev/null +++ b/dill/tests/test_dataclasses.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Author: Anirudh Vegesana (avegesan@cs.stanford.edu) +# Copyright (c) 2022 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +""" +test pickling a dataclass +""" + +import dill +import dataclasses + +def test_dataclasses(): + # Issue #500 + @dataclasses.dataclass + class A: + x: int + y: str + + @dataclasses.dataclass + class B: + a: A + + a = A(1, "test") + before = B(a) + save = dill.dumps(before) + after = dill.loads(save) + assert before != after # classes don't match + assert before == B(A(**dataclasses.asdict(after.a))) + assert dataclasses.asdict(before) == dataclasses.asdict(after) + +if __name__ == '__main__': + test_dataclasses() diff --git a/tests/test_detect.py b/dill/tests/test_detect.py similarity index 96% rename from tests/test_detect.py rename to dill/tests/test_detect.py index 06339acd..4dac7aaf 100644 --- a/tests/test_detect.py +++ b/dill/tests/test_detect.py @@ -8,7 +8,7 @@ from dill.detect import baditems, badobjects, badtypes, errors, parent, at, globalvars from dill import settings -from dill._dill import IS_PYPY, IS_PYPY2 +from dill._dill import IS_PYPY from pickle import PicklingError import inspect @@ -21,7 +21,7 @@ def test_bad_things(): #assert baditems(globals()) == [f] #XXX assert badobjects(f) is f assert badtypes(f) == type(f) - assert type(errors(f)) is PicklingError if IS_PYPY2 else TypeError + assert type(errors(f)) is TypeError d = badtypes(f, 1) assert isinstance(d, dict) assert list(badobjects(f, 1).keys()) == list(d.keys()) @@ -30,7 +30,7 @@ def test_bad_things(): a = dict(s) if not os.environ.get('COVERAGE'): #XXX: travis-ci assert len(s) is len(a) # TypeError (and possibly PicklingError) - n = 1 if IS_PYPY2 else 2 + n = 2 assert len(a) is n if 'PicklingError' in a.keys() else n-1 def test_parent(): diff --git a/tests/test_dictviews.py b/dill/tests/test_dictviews.py similarity index 79% rename from tests/test_dictviews.py rename to dill/tests/test_dictviews.py index 3bbc5d62..87d0a9eb 100644 --- a/tests/test_dictviews.py +++ b/dill/tests/test_dictviews.py @@ -1,13 +1,16 @@ #!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) -# Copyright (c) 2008-2016 California Institute of Technology. -# Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. +# Author: Anirudh Vegesana (avegesan@cs.stanford.edu) +# Copyright (c) 2021-2022 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/dill/blob/master/LICENSE import dill -from dill._dill import OLD310, MAPPING_PROXY_TRICK +from dill._dill import OLD310, MAPPING_PROXY_TRICK, DictProxyType + +def test_dictproxy(): + assert dill.copy(DictProxyType({'a': 2})) def test_dictviews(): x = {'a': 1} @@ -31,5 +34,6 @@ def test_dictproxy_trick(): assert dict(seperate_views[1]) == new_x if __name__ == '__main__': + test_dictproxy() test_dictviews() test_dictproxy_trick() diff --git a/tests/test_diff.py b/dill/tests/test_diff.py similarity index 83% rename from tests/test_diff.py rename to dill/tests/test_diff.py index 3277682b..652d819b 100644 --- a/tests/test_diff.py +++ b/dill/tests/test_diff.py @@ -55,18 +55,15 @@ def test_diff(): assert changed[1] if not IS_PYPY: - try: - import abc - # make sure the "_abc_invaldation_counter" doesn't make test fail - diff.memorise(abc.ABCMeta, force=True) - assert not diff.has_changed(abc) - abc.ABCMeta.zzz = 1 - assert diff.has_changed(abc) - changed = diff.whats_changed(abc) - assert list(changed[0].keys()) == ["ABCMeta"] - assert not changed[1] - except ImportError: - pass + import abc + # make sure the "_abc_invaldation_counter" doesn't make test fail + diff.memorise(abc.ABCMeta, force=True) + assert not diff.has_changed(abc) + abc.ABCMeta.zzz = 1 + assert diff.has_changed(abc) + changed = diff.whats_changed(abc) + assert list(changed[0].keys()) == ["ABCMeta"] + assert not changed[1] ''' import Queue diff --git a/tests/test_enum.py b/dill/tests/test_enum.py similarity index 86% rename from tests/test_enum.py rename to dill/tests/test_enum.py index 71fb3850..adbe83d6 100644 --- a/tests/test_enum.py +++ b/dill/tests/test_enum.py @@ -1,14 +1,6 @@ -try: - import enum - from enum import Enum, IntEnum, EnumMeta, Flag, IntFlag, unique, auto -except: - try: - import enum34 - from enum34 import Enum, IntEnum, EnumMeta, Flag, IntFlag, unique, auto - except: - Enum = None - import abc +import enum +from enum import Enum, IntEnum, EnumMeta, Flag, IntFlag, unique, auto import dill import sys @@ -82,19 +74,12 @@ def __new__(mcls, *args, **kw): pass return abstract_enum_cls - if dill._dill.PY3: - l = locals() - exec("""class StrEnum(str, abc.ABC, Enum, metaclass=ABCEnumMeta): - 'accepts only string values' - def invisible(self): - return "did you see me?" """, None, l) - StrEnum = l['StrEnum'] - else: - class StrEnum(str, abc.ABC, Enum): - __metaclass__ = ABCEnumMeta - 'accepts only string values' - def invisible(self): - return "did you see me?" + l = locals() + exec("""class StrEnum(str, abc.ABC, Enum, metaclass=ABCEnumMeta): + 'accepts only string values' + def invisible(self): + return "did you see me?" """, None, l) + StrEnum = l['StrEnum'] class Name(StrEnum): BDFL = 'Guido van Rossum' @@ -173,5 +158,4 @@ class Color(AutoNumber): # assert list(map(int, Color_)) == [1, 2, 3] if __name__ == '__main__': - if Enum: - test_enums() + test_enums() diff --git a/tests/test_extendpickle.py b/dill/tests/test_extendpickle.py similarity index 91% rename from tests/test_extendpickle.py rename to dill/tests/test_extendpickle.py index 49b6f152..10dc0804 100644 --- a/tests/test_extendpickle.py +++ b/dill/tests/test_extendpickle.py @@ -7,10 +7,7 @@ # - https://github.com/uqfoundation/dill/blob/master/LICENSE import dill as pickle -try: - from StringIO import StringIO -except ImportError: - from io import BytesIO as StringIO +from io import BytesIO as StringIO def my_fn(x): @@ -47,7 +44,7 @@ def test_isdill(): pickler = mp.reduction.ForkingPickler(obj_io) assert pickle._dill.is_dill(pickler, child=True) is True assert pickle._dill.is_dill(pickler, child=False) is False - except: + except Exception: pass diff --git a/tests/test_fglobals.py b/dill/tests/test_fglobals.py similarity index 100% rename from tests/test_fglobals.py rename to dill/tests/test_fglobals.py diff --git a/tests/test_file.py b/dill/tests/test_file.py similarity index 99% rename from tests/test_file.py rename to dill/tests/test_file.py index 8118e3a8..b766552e 100644 --- a/tests/test_file.py +++ b/dill/tests/test_file.py @@ -19,8 +19,6 @@ fname = "_test_file.txt" rand_chars = list(string.ascii_letters) + ["\n"] * 40 # bias newline -if sys.hexversion < 0x03030000: - FileNotFoundError = IOError buffer_error = ValueError("invalid buffer size") dne_error = FileNotFoundError("[Errno 2] No such file or directory: '%s'" % fname) diff --git a/tests/test_functions.py b/dill/tests/test_functions.py similarity index 53% rename from tests/test_functions.py rename to dill/tests/test_functions.py index ec9670e2..d8c73396 100644 --- a/tests/test_functions.py +++ b/dill/tests/test_functions.py @@ -11,10 +11,6 @@ dill.settings['recurse'] = True -def is_py3(): - return hex(sys.hexversion) >= '0x30000f0' - - def function_a(a): return a @@ -34,18 +30,17 @@ def function_d(d, d1, d2=1): function_d.__module__ = 'a module' -if is_py3(): - exec(''' +exec(''' def function_e(e, *e1, e2=1, e3=2): return e + sum(e1) + e2 + e3''') - globalvar = 0 +globalvar = 0 - @functools.lru_cache(None) - def function_with_cache(x): - global globalvar - globalvar += x - return globalvar +@functools.lru_cache(None) +def function_with_cache(x): + global globalvar + globalvar += x + return globalvar def function_with_unassigned_variable(): @@ -54,6 +49,21 @@ def function_with_unassigned_variable(): return (lambda: value) +def test_issue_510(): + # A very bizzare use of functions and methods that pickle doesn't get + # correctly for odd reasons. + class Foo: + def __init__(self): + def f2(self): + return self + self.f2 = f2.__get__(self) + + import dill, pickletools + f = Foo() + f1 = dill.copy(f) + assert f1.f2() is f1 + + def test_functions(): dumped_func_a = dill.dumps(function_a) assert dill.loads(dumped_func_a)(0) == 0 @@ -72,28 +82,26 @@ def test_functions(): assert dill.loads(dumped_func_d)(1, 2, 3) == 6 assert dill.loads(dumped_func_d)(1, 2, d2=3) == 6 - if is_py3(): - function_with_cache(1) - globalvar = 0 - dumped_func_cache = dill.dumps(function_with_cache) - assert function_with_cache(2) == 3 - assert function_with_cache(1) == 1 - assert function_with_cache(3) == 6 - assert function_with_cache(2) == 3 + function_with_cache(1) + globalvar = 0 + dumped_func_cache = dill.dumps(function_with_cache) + assert function_with_cache(2) == 3 + assert function_with_cache(1) == 1 + assert function_with_cache(3) == 6 + assert function_with_cache(2) == 3 empty_cell = function_with_unassigned_variable() cell_copy = dill.loads(dill.dumps(empty_cell)) assert 'empty' in str(cell_copy.__closure__[0]) try: cell_copy() - except: + except Exception: # this is good pass else: raise AssertionError('cell_copy() did not read an empty cell') - if is_py3(): - exec(''' + exec(''' dumped_func_e = dill.dumps(function_e) assert dill.loads(dumped_func_e)(1, 2) == 6 assert dill.loads(dumped_func_e)(1, 2, 3) == 9 @@ -102,5 +110,29 @@ def test_functions(): assert dill.loads(dumped_func_e)(1, 2, 3, e2=4) == 12 assert dill.loads(dumped_func_e)(1, 2, 3, e2=4, e3=5) == 15''') +def test_code_object(): + from dill._dill import ALL_CODE_PARAMS, CODE_PARAMS, CODE_VERSION, _create_code + code = function_c.__code__ + LNOTAB = getattr(code, 'co_lnotab', b'') + fields = {f: getattr(code, 'co_'+f) for f in CODE_PARAMS} + fields.setdefault('posonlyargcount', 0) # python >= 3.8 + fields.setdefault('lnotab', LNOTAB) # python <= 3.9 + fields.setdefault('linetable', b'') # python >= 3.10 + fields.setdefault('qualname', fields['name']) # python >= 3.11 + fields.setdefault('exceptiontable', b'') # python >= 3.11 + fields.setdefault('endlinetable', None) # python == 3.11a + fields.setdefault('columntable', None) # python == 3.11a + + for version, _, params in ALL_CODE_PARAMS: + args = tuple(fields[p] for p in params.split()) + try: + _create_code(*args) + if version >= (3,10): + _create_code(fields['lnotab'], *args) + except Exception as error: + raise Exception("failed to construct code object with format version {}".format(version)) from error + if __name__ == '__main__': test_functions() + test_issue_510() + test_code_object() diff --git a/tests/test_functors.py b/dill/tests/test_functors.py similarity index 100% rename from tests/test_functors.py rename to dill/tests/test_functors.py diff --git a/dill/tests/test_logger.py b/dill/tests/test_logger.py new file mode 100644 index 00000000..b4e4881a --- /dev/null +++ b/dill/tests/test_logger.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +# Author: Leonardo Gama (@leogama) +# Copyright (c) 2022 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE + +import logging +import re +import tempfile + +import dill +from dill import detect +from dill.logger import stderr_handler, adapter as logger + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + +test_obj = {'a': (1, 2), 'b': object(), 'f': lambda x: x**2, 'big': list(range(10))} + +def test_logging(should_trace): + buffer = StringIO() + handler = logging.StreamHandler(buffer) + logger.addHandler(handler) + try: + dill.dumps(test_obj) + if should_trace: + regex = re.compile(r'(\S*┬ \w.*[^)]' # begin pickling object + r'|│*└ # \w.* \[\d+ (\wi)?B])' # object written (with size) + ) + for line in buffer.getvalue().splitlines(): + assert regex.fullmatch(line) + return buffer.getvalue() + else: + assert buffer.getvalue() == "" + finally: + logger.removeHandler(handler) + buffer.close() + +def test_trace_to_file(stream_trace): + file = tempfile.NamedTemporaryFile(mode='r') + with detect.trace(file.name, mode='w'): + dill.dumps(test_obj) + file_trace = file.read() + file.close() + # Apparently, objects can change location in memory... + reghex = re.compile(r'0x[0-9A-Za-z]+') + file_trace, stream_trace = reghex.sub('0x', file_trace), reghex.sub('0x', stream_trace) + # PyPy prints dictionary contents with repr(dict)... + regdict = re.compile(r'(dict\.__repr__ of ).*') + file_trace, stream_trace = regdict.sub(r'\1{}>', file_trace), regdict.sub(r'\1{}>', stream_trace) + assert file_trace == stream_trace + +if __name__ == '__main__': + logger.removeHandler(stderr_handler) + test_logging(should_trace=False) + detect.trace(True) + test_logging(should_trace=True) + detect.trace(False) + test_logging(should_trace=False) + + loglevel = logging.ERROR + logger.setLevel(loglevel) + with detect.trace(): + stream_trace = test_logging(should_trace=True) + test_logging(should_trace=False) + assert logger.getEffectiveLevel() == loglevel + test_trace_to_file(stream_trace) diff --git a/tests/test_mixins.py b/dill/tests/test_mixins.py similarity index 100% rename from tests/test_mixins.py rename to dill/tests/test_mixins.py diff --git a/tests/test_module.py b/dill/tests/test_module.py similarity index 97% rename from tests/test_module.py rename to dill/tests/test_module.py index 5c5e000c..2d009f26 100644 --- a/tests/test_module.py +++ b/dill/tests/test_module.py @@ -9,8 +9,7 @@ import sys import dill import test_mixins as module -try: from importlib import reload -except ImportError: pass +from importlib import reload dill.settings['recurse'] = True cached = (module.__cached__ if hasattr(module, "__cached__") diff --git a/tests/test_moduledict.py b/dill/tests/test_moduledict.py similarity index 100% rename from tests/test_moduledict.py rename to dill/tests/test_moduledict.py diff --git a/tests/test_nested.py b/dill/tests/test_nested.py similarity index 99% rename from tests/test_nested.py rename to dill/tests/test_nested.py index 144f54b0..950641ca 100644 --- a/tests/test_nested.py +++ b/dill/tests/test_nested.py @@ -110,7 +110,7 @@ def test_pickled_inner(): def test_moduledict_where_not_main(): try: from . import test_moduledict - except: + except ImportError: import test_moduledict name = 'test_moduledict.py' if os.path.exists(name) and os.path.exists(name+'c'): diff --git a/tests/test_objects.py b/dill/tests/test_objects.py similarity index 96% rename from tests/test_objects.py rename to dill/tests/test_objects.py index 985041be..e69a2293 100644 --- a/tests/test_objects.py +++ b/dill/tests/test_objects.py @@ -57,6 +57,7 @@ def test_objects(): #pickles(member, exact=True) pickles(member, exact=False) - if __name__ == '__main__': + import warnings + warnings.simplefilter('ignore') test_objects() diff --git a/tests/test_properties.py b/dill/tests/test_properties.py similarity index 100% rename from tests/test_properties.py rename to dill/tests/test_properties.py diff --git a/dill/tests/test_pycapsule.py b/dill/tests/test_pycapsule.py new file mode 100644 index 00000000..6e115ffd --- /dev/null +++ b/dill/tests/test_pycapsule.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Author: Anirudh Vegesana (avegesan@cs.stanford.edu) +# Copyright (c) 2022 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE +""" +test pickling a PyCapsule object +""" + +import dill +import warnings + +test_pycapsule = None + +if dill._dill._testcapsule is not None: + import ctypes + def test_pycapsule(): + name = ctypes.create_string_buffer(b'dill._testcapsule') + capsule = dill._dill._PyCapsule_New( + ctypes.cast(dill._dill._PyCapsule_New, ctypes.c_void_p), + name, + None + ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + dill.copy(capsule) + dill._testcapsule = capsule + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + dill.copy(capsule) + dill._testcapsule = None + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", dill.PicklingWarning) + dill.copy(capsule) + except dill.UnpicklingError: + pass + else: + raise AssertionError("Expected a different error") + +if __name__ == '__main__': + if test_pycapsule is not None: + test_pycapsule() diff --git a/tests/test_recursive.py b/dill/tests/test_recursive.py similarity index 94% rename from tests/test_recursive.py rename to dill/tests/test_recursive.py index ee71a688..0f3a53a1 100644 --- a/tests/test_recursive.py +++ b/dill/tests/test_recursive.py @@ -6,7 +6,6 @@ # - https://github.com/uqfoundation/dill/blob/master/LICENSE import dill -from dill._dill import PY3 from functools import partial import warnings @@ -15,7 +14,7 @@ def copy(obj, byref=False, recurse=False): if byref: try: return dill.copy(obj, byref=byref, recurse=recurse) - except: + except Exception: pass else: raise AssertionError('Copy of %s with byref=True should have given a warning!' % (obj,)) @@ -113,9 +112,8 @@ def __init__(self): def test_circular_reference(): assert copy(obj4()) obj4_copy = dill.loads(dill.dumps(obj4())) - if PY3: - assert type(obj4_copy) is type(obj4_copy).__init__.__closure__[0].cell_contents - assert type(obj4_copy.b) is type(obj4_copy.b).__init__.__closure__[0].cell_contents + assert type(obj4_copy) is type(obj4_copy).__init__.__closure__[0].cell_contents + assert type(obj4_copy.b) is type(obj4_copy.b).__init__.__closure__[0].cell_contents def f(): @@ -146,7 +144,7 @@ def test_recursive_function(): for _fib in (fib3, fib4): try: _fib(5) - except: + except Exception: # This is expected to fail because fib no longer exists pass else: diff --git a/dill/tests/test_registered.py b/dill/tests/test_registered.py new file mode 100644 index 00000000..c24824f2 --- /dev/null +++ b/dill/tests/test_registered.py @@ -0,0 +1,54 @@ +import dill +from dill._objects import failures, registered, succeeds +import warnings +warnings.filterwarnings('ignore') + +def check(d, ok=True): + res = [] + for k,v in d.items(): + try: + z = dill.copy(v) + if ok: res.append(k) + except: + if not ok: res.append(k) + return res + +fails = check(failures) +try: + assert not bool(fails) +except AssertionError as e: + print("FAILS: %s" % fails) + raise e from None + +register = check(registered, ok=False) +try: + assert not bool(register) +except AssertionError as e: + print("REGISTER: %s" % register) + raise e from None + +success = check(succeeds, ok=False) +try: + assert not bool(success) +except AssertionError as e: + print("SUCCESS: %s" % success) + raise e from None + +import builtins +import types +q = dill._dill._reverse_typemap +p = {k:v for k,v in q.items() if k not in vars(builtins) and k not in vars(types)} + +diff = set(p.keys()).difference(registered.keys()) +try: + assert not bool(diff) +except AssertionError as e: + print("DIFF: %s" % diff) + raise e from None + +miss = set(registered.keys()).difference(p.keys()) +try: + assert not bool(miss) +except AssertionError as e: + print("MISS: %s" % miss) + raise e from None diff --git a/tests/test_restricted.py b/dill/tests/test_restricted.py similarity index 100% rename from tests/test_restricted.py rename to dill/tests/test_restricted.py diff --git a/tests/test_selected.py b/dill/tests/test_selected.py similarity index 76% rename from tests/test_selected.py rename to dill/tests/test_selected.py index bd790835..2f0eda73 100644 --- a/tests/test_selected.py +++ b/dill/tests/test_selected.py @@ -19,7 +19,7 @@ def test_dict_contents(): for i,j in c.items(): #try: ok = dill.pickles(j) - #except: + #except Exception: # print ("FAIL: %s with %s" % (i, dill.detect.errors(j))) if verbose: print ("%s: %s, %s" % (ok, type(j), j)) assert ok @@ -29,7 +29,7 @@ def _g(x): yield x; def _f(): try: raise - except: + except Exception: from sys import exc_info e, er, tb = exc_info() return er, tb @@ -42,6 +42,13 @@ def _method(self): from dill import load_types load_types(pickleable=True,unpickleable=False) _newclass = objects['ClassObjectType'] +# some clean-up #FIXME: should happen internal to dill +objects['TemporaryFileType'].close() +objects['TextWrapperType'].close() +objects['BufferedRandomType'].close() +objects['BufferedReaderType'].close() +objects['BufferedWriterType'].close() +objects['FileType'].close() del objects # getset_descriptor for new-style classes (fails on '_method', if not __main__) @@ -76,7 +83,7 @@ def test_frame_related(): g = _g(1) f = g.gi_frame e,t = _f() - _is = lambda ok: not ok if dill._dill.IS_PYPY2 else ok + _is = lambda ok: ok ok = dill.pickles(f) if verbose: print ("%s: %s, %s" % (ok, type(f), f)) assert not ok @@ -91,9 +98,28 @@ def test_frame_related(): assert ok if verbose: print ("") +def test_typing(): + import typing + x = typing.Any + assert x == dill.copy(x) + x = typing.Dict[int, str] + assert x == dill.copy(x) + x = typing.List[int] + assert x == dill.copy(x) + x = typing.Tuple[int, str] + assert x == dill.copy(x) + x = typing.Tuple[int] + assert x == dill.copy(x) + x = typing.Tuple[()] + assert x == dill.copy(x) + x = typing.Tuple[()].copy_with(()) + assert x == dill.copy(x) + return + if __name__ == '__main__': test_frame_related() test_dict_contents() test_class() test_class_descriptors() + test_typing() diff --git a/dill/tests/test_session.py b/dill/tests/test_session.py new file mode 100644 index 00000000..51128916 --- /dev/null +++ b/dill/tests/test_session.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python + +# Author: Leonardo Gama (@leogama) +# Copyright (c) 2022 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE + +import atexit +import os +import sys +import __main__ +from contextlib import suppress +from io import BytesIO + +import dill + +session_file = os.path.join(os.path.dirname(__file__), 'session-refimported-%s.pkl') + +################### +# Child process # +################### + +def _error_line(error, obj, refimported): + import traceback + line = traceback.format_exc().splitlines()[-2].replace('[obj]', '['+repr(obj)+']') + return "while testing (with refimported=%s): %s" % (refimported, line.lstrip()) + +if __name__ == '__main__' and len(sys.argv) >= 3 and sys.argv[1] == '--child': + # Test session loading in a fresh interpreter session. + refimported = (sys.argv[2] == 'True') + dill.load_module(session_file % refimported, module='__main__') + + def test_modules(refimported): + # FIXME: In this test setting with CPython 3.7, 'calendar' is not included + # in sys.modules, independent of the value of refimported. Tried to + # run garbage collection just before loading the session with no luck. It + # fails even when preceding them with 'import calendar'. Needed to run + # these kinds of tests in a supbrocess. Failing test sample: + # assert globals()['day_name'] is sys.modules['calendar'].__dict__['day_name'] + try: + for obj in ('json', 'url', 'local_mod', 'sax', 'dom'): + assert globals()[obj].__name__ in sys.modules + assert 'calendar' in sys.modules and 'cmath' in sys.modules + import calendar, cmath + + for obj in ('Calendar', 'isleap'): + assert globals()[obj] is sys.modules['calendar'].__dict__[obj] + assert __main__.day_name.__module__ == 'calendar' + if refimported: + assert __main__.day_name is calendar.day_name + + assert __main__.complex_log is cmath.log + + except AssertionError as error: + error.args = (_error_line(error, obj, refimported),) + raise + + test_modules(refimported) + sys.exit() + +#################### +# Parent process # +#################### + +# Create various kinds of objects to test different internal logics. + +## Modules. +import json # top-level module +import urllib as url # top-level module under alias +from xml import sax # submodule +import xml.dom.minidom as dom # submodule under alias +import test_dictviews as local_mod # non-builtin top-level module + +## Imported objects. +from calendar import Calendar, isleap, day_name # class, function, other object +from cmath import log as complex_log # imported with alias + +## Local objects. +x = 17 +empty = None +names = ['Alice', 'Bob', 'Carol'] +def squared(x): return x**2 +cubed = lambda x: x**3 +class Person: + def __init__(self, name, age): + self.name = name + self.age = age +person = Person(names[0], x) +class CalendarSubclass(Calendar): + def weekdays(self): + return [day_name[i] for i in self.iterweekdays()] +cal = CalendarSubclass() +selfref = __main__ + +# Setup global namespace for session saving tests. +class TestNamespace: + test_globals = globals().copy() + def __init__(self, **extra): + self.extra = extra + def __enter__(self): + self.backup = globals().copy() + globals().clear() + globals().update(self.test_globals) + globals().update(self.extra) + return self + def __exit__(self, *exc_info): + globals().clear() + globals().update(self.backup) + +def _clean_up_cache(module): + cached = module.__file__.split('.', 1)[0] + '.pyc' + cached = module.__cached__ if hasattr(module, '__cached__') else cached + pycache = os.path.join(os.path.dirname(module.__file__), '__pycache__') + for remove, file in [(os.remove, cached), (os.removedirs, pycache)]: + with suppress(OSError): + remove(file) + +atexit.register(_clean_up_cache, local_mod) + +def _test_objects(main, globals_copy, refimported): + try: + main_dict = __main__.__dict__ + global Person, person, Calendar, CalendarSubclass, cal, selfref + + for obj in ('json', 'url', 'local_mod', 'sax', 'dom'): + assert globals()[obj].__name__ == globals_copy[obj].__name__ + + for obj in ('x', 'empty', 'names'): + assert main_dict[obj] == globals_copy[obj] + + for obj in ['squared', 'cubed']: + assert main_dict[obj].__globals__ is main_dict + assert main_dict[obj](3) == globals_copy[obj](3) + + assert Person.__module__ == __main__.__name__ + assert isinstance(person, Person) + assert person.age == globals_copy['person'].age + + assert issubclass(CalendarSubclass, Calendar) + assert isinstance(cal, CalendarSubclass) + assert cal.weekdays() == globals_copy['cal'].weekdays() + + assert selfref is __main__ + + except AssertionError as error: + error.args = (_error_line(error, obj, refimported),) + raise + +def test_session_main(refimported): + """test dump/load_module() for __main__, both in this process and in a subprocess""" + extra_objects = {} + if refimported: + # Test unpickleable imported object in main. + from sys import flags + extra_objects['flags'] = flags + + with TestNamespace(**extra_objects) as ns: + try: + # Test session loading in a new session. + dill.dump_module(session_file % refimported, refimported=refimported) + from dill.tests.__main__ import python, shell, sp + error = sp.call([python, __file__, '--child', str(refimported)], shell=shell) + if error: sys.exit(error) + finally: + with suppress(OSError): + os.remove(session_file % refimported) + + # Test session loading in the same session. + session_buffer = BytesIO() + dill.dump_module(session_buffer, refimported=refimported) + session_buffer.seek(0) + dill.load_module(session_buffer, module='__main__') + ns.backup['_test_objects'](__main__, ns.backup, refimported) + +def test_session_other(): + """test dump/load_module() for a module other than __main__""" + import test_classdef as module + atexit.register(_clean_up_cache, module) + module.selfref = module + dict_objects = [obj for obj in module.__dict__.keys() if not obj.startswith('__')] + + session_buffer = BytesIO() + dill.dump_module(session_buffer, module) + + for obj in dict_objects: + del module.__dict__[obj] + + session_buffer.seek(0) + dill.load_module(session_buffer, module) + + assert all(obj in module.__dict__ for obj in dict_objects) + assert module.selfref is module + +def test_runtime_module(): + from types import ModuleType + modname = '__runtime__' + runtime = ModuleType(modname) + runtime.x = 42 + + mod = dill.session._stash_modules(runtime) + if mod is not runtime: + print("There are objects to save by referenece that shouldn't be:", + mod.__dill_imported, mod.__dill_imported_as, mod.__dill_imported_top_level, + file=sys.stderr) + + # This is also for code coverage, tests the use case of dump_module(refimported=True) + # without imported objects in the namespace. It's a contrived example because + # even dill can't be in it. This should work after fixing #462. + session_buffer = BytesIO() + dill.dump_module(session_buffer, module=runtime, refimported=True) + session_dump = session_buffer.getvalue() + + # Pass a new runtime created module with the same name. + runtime = ModuleType(modname) # empty + return_val = dill.load_module(BytesIO(session_dump), module=runtime) + assert return_val is None + assert runtime.__name__ == modname + assert runtime.x == 42 + assert runtime not in sys.modules.values() + + # Pass nothing as main. load_module() must create it. + session_buffer.seek(0) + runtime = dill.load_module(BytesIO(session_dump)) + assert runtime.__name__ == modname + assert runtime.x == 42 + assert runtime not in sys.modules.values() + +def test_refimported_imported_as(): + import collections + import concurrent.futures + import types + import typing + mod = sys.modules['__test__'] = types.ModuleType('__test__') + dill.executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) + mod.Dict = collections.UserDict # select by type + mod.AsyncCM = typing.AsyncContextManager # select by __module__ + mod.thread_exec = dill.executor # select by __module__ with regex + + session_buffer = BytesIO() + dill.dump_module(session_buffer, mod, refimported=True) + session_buffer.seek(0) + mod = dill.load(session_buffer) + del sys.modules['__test__'] + + assert set(mod.__dill_imported_as) == { + ('collections', 'UserDict', 'Dict'), + ('typing', 'AsyncContextManager', 'AsyncCM'), + ('dill', 'executor', 'thread_exec'), + } + +def test_load_module_asdict(): + with TestNamespace(): + session_buffer = BytesIO() + dill.dump_module(session_buffer) + + global empty, names, x, y + x = y = 0 # change x and create y + del empty + globals_state = globals().copy() + + session_buffer.seek(0) + main_vars = dill.load_module_asdict(session_buffer) + + assert main_vars is not globals() + assert globals() == globals_state + + assert main_vars['__name__'] == '__main__' + assert main_vars['names'] == names + assert main_vars['names'] is not names + assert main_vars['x'] != x + assert 'y' not in main_vars + assert 'empty' in main_vars + +if __name__ == '__main__': + test_session_main(refimported=False) + test_session_main(refimported=True) + test_session_other() + test_runtime_module() + test_refimported_imported_as() + test_load_module_asdict() diff --git a/tests/test_source.py b/dill/tests/test_source.py similarity index 89% rename from tests/test_source.py rename to dill/tests/test_source.py index 57d25d12..01fc1eb3 100644 --- a/tests/test_source.py +++ b/dill/tests/test_source.py @@ -11,9 +11,7 @@ from dill._dill import IS_PYPY import sys -PY3 = sys.version_info[0] >= 3 -IS_PYPY3 = IS_PYPY and PY3 -PY310b = '0x30a00b1' +PY310b = 0x30a00b1 f = lambda x: x**2 def g(x): return f(x) - x @@ -61,14 +59,12 @@ def test_itself(): # builtin functions and objects def test_builtin(): - if PY3: builtin = 'builtins' - else: builtin = '__builtin__' assert likely_import(pow) == 'pow\n' assert likely_import(100) == '100\n' assert likely_import(True) == 'True\n' - assert likely_import(pow, explicit=True) == 'from %s import pow\n' % builtin + assert likely_import(pow, explicit=True) == 'from builtins import pow\n' assert likely_import(100, explicit=True) == '100\n' - assert likely_import(True, explicit=True) == 'True\n' if PY3 else 'from %s import True\n' % builtin + assert likely_import(True, explicit=True) == 'True\n' # this is kinda BS... you can't import a None assert likely_import(None) == 'None\n' assert likely_import(None, explicit=True) == 'None\n' @@ -87,14 +83,9 @@ def test_dynamic(): # classes and class instances def test_classes(): - try: #XXX: should this be a 'special case'? - from StringIO import StringIO - y = "from StringIO import StringIO\n" - x = y - except ImportError: - from io import BytesIO as StringIO - y = "from _io import BytesIO\n" - x = y if (IS_PYPY3 or hex(sys.hexversion) >= PY310b) else "from io import BytesIO\n" + from io import BytesIO as StringIO + y = "from _io import BytesIO\n" + x = y if (IS_PYPY or sys.hexversion >= PY310b) else "from io import BytesIO\n" s = StringIO() assert likely_import(StringIO) == x diff --git a/tests/test_temp.py b/dill/tests/test_temp.py similarity index 100% rename from tests/test_temp.py rename to dill/tests/test_temp.py diff --git a/tests/test_weakref.py b/dill/tests/test_weakref.py similarity index 98% rename from tests/test_weakref.py rename to dill/tests/test_weakref.py index ada7d140..0e99f3ea 100644 --- a/tests/test_weakref.py +++ b/dill/tests/test_weakref.py @@ -76,7 +76,7 @@ def test_dictproxy(): from dill._dill import DictProxyType try: m = DictProxyType({"foo": "bar"}) - except: + except Exception: m = type.__dict__ mp = dill.copy(m) assert mp.items() == m.items() diff --git a/docs/source/conf.py b/docs/source/conf.py index 19171caf..ead9ed06 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -58,19 +58,20 @@ master_doc = 'index' # General information about the project. -project = u'dill' +project = 'dill' year = datetime.now().year -copyright = u'%d, The Uncertainty Quantification Foundation' % year -author = u'Mike McKerns' +copyright = '%d, The Uncertainty Quantification Foundation' % year +author = 'Mike McKerns' # extension config github_project_url = "https://github.com/uqfoundation/dill" -autoclass_content= 'both' +autoclass_content = 'both' +autodoc_typehints = 'description' napoleon_include_init_with_doc = True napoleon_include_private_with_doc = False napoleon_include_special_with_doc = True -napoleon_use_param = False napoleon_use_ivar = True +napoleon_use_param = True # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -191,8 +192,8 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'dill.tex', u'dill Documentation', - u'Mike McKerns', 'manual'), + (master_doc, 'dill.tex', 'dill Documentation', + 'Mike McKerns', 'manual'), ] @@ -201,7 +202,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'dill', u'dill Documentation', + (master_doc, 'dill', 'dill Documentation', [author], 1) ] @@ -212,7 +213,7 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'dill', u'dill Documentation', + (master_doc, 'dill', 'dill Documentation', author, 'dill', 'Serialize all of python.', 'Miscellaneous'), ] diff --git a/docs/source/dill.rst b/docs/source/dill.rst index 9061863c..2770af2a 100644 --- a/docs/source/dill.rst +++ b/docs/source/dill.rst @@ -25,6 +25,18 @@ detect module :imported-members: .. :exclude-members: ismethod, isfunction, istraceback, isframe, iscode, parent, reference, at, parents, children +logger module +------------- + +.. automodule:: dill.logger + :members: + :undoc-members: + :private-members: + :special-members: + :show-inheritance: + :imported-members: +.. :exclude-members: + objtypes module --------------- @@ -49,6 +61,18 @@ pointers module :imported-members: .. :exclude-members: +session module +--------------- + +.. automodule:: dill.session + :members: + :undoc-members: + :private-members: + :special-members: + :show-inheritance: + :imported-members: + :exclude-members: dump_session, load_session + settings module --------------- diff --git a/setup.py b/setup.py index 0d3e76cb..bd182e23 100644 --- a/setup.py +++ b/setup.py @@ -15,41 +15,16 @@ # get distribution meta info here = os.path.abspath(os.path.dirname(__file__)) -meta_fh = open(os.path.join(here, 'dill/__init__.py')) -try: - meta = {} - for line in meta_fh: - if line.startswith('__version__'): - VERSION = line.split()[-1].strip("'").strip('"') - break - meta['VERSION'] = VERSION - for line in meta_fh: - if line.startswith('__author__'): - AUTHOR = line.split(' = ')[-1].strip().strip("'").strip('"') - break - meta['AUTHOR'] = AUTHOR - LONG_DOC = "" - DOC_STOP = "FAKE_STOP_12345" - for line in meta_fh: - if LONG_DOC: - if line.startswith(DOC_STOP): - LONG_DOC = LONG_DOC.strip().strip("'").strip('"').lstrip() - break - else: - LONG_DOC += line - elif line.startswith('__doc__'): - DOC_STOP = line.split(' = ')[-1] - LONG_DOC = "\n" - meta['LONG_DOC'] = LONG_DOC -finally: - meta_fh.close() - -# get version numbers, long_description, etc -AUTHOR = meta['AUTHOR'] -VERSION = meta['VERSION'] -LONG_DOC = meta['LONG_DOC'] #FIXME: near-duplicate of README.md -#LICENSE = meta['LICENSE'] #FIXME: duplicate of LICENSE -AUTHOR_EMAIL = 'mmckerns@uqfoundation.org' +sys.path.append(here) +from version import (__version__, __author__, __contact__ as AUTHOR_EMAIL, + get_license_text, get_readme_as_rst, write_info_file) +LICENSE = get_license_text(os.path.join(here, 'LICENSE')) +README = get_readme_as_rst(os.path.join(here, 'README.md')) + +# write meta info file +write_info_file(here, 'dill', doc=README, license=LICENSE, + version=__version__, author=__author__) +del here, get_license_text, get_readme_as_rst, write_info_file # check if setuptools is available try: @@ -64,12 +39,12 @@ # build the 'setup' call setup_kwds = dict( name='dill', - version=VERSION, + version=__version__, description='serialize all of python', - long_description = LONG_DOC, - author = AUTHOR, + long_description = README.strip(), + author = __author__, author_email = AUTHOR_EMAIL, - maintainer = AUTHOR, + maintainer = __author__, maintainer_email = AUTHOR_EMAIL, license = '3-clause BSD', platforms = ['Linux', 'Windows', 'Mac'], @@ -97,7 +72,7 @@ 'Topic :: Software Development', ], packages = ['dill','dill.tests'], - package_dir = {'dill':'dill', 'dill.tests':'tests'}, + package_dir = {'dill':'dill', 'dill.tests':'dill/tests'}, scripts=['scripts/undill','scripts/get_objgraph'], ) diff --git a/tests/test_session.py b/tests/test_session.py deleted file mode 100644 index fd71ea05..00000000 --- a/tests/test_session.py +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env python - -# Author: Leonardo Gama (@leogama) -# Copyright (c) 2022 The Uncertainty Quantification Foundation. -# License: 3-clause BSD. The full license text is available at: -# - https://github.com/uqfoundation/dill/blob/master/LICENSE - -from __future__ import print_function -import atexit, dill, os, sys, __main__ - -session_file = os.path.join(os.path.dirname(__file__), 'session-byref-%s.pkl') - -def test_modules(main, byref): - main_dict = main.__dict__ - - try: - for obj in ('json', 'url', 'local_mod', 'sax', 'dom'): - assert main_dict[obj].__name__ in sys.modules - - for obj in ('Calendar', 'isleap'): - assert main_dict[obj] is sys.modules['calendar'].__dict__[obj] - assert main.day_name.__module__ == 'calendar' - if byref: - assert main.day_name is sys.modules['calendar'].__dict__['day_name'] - - assert main.complex_log is sys.modules['cmath'].__dict__['log'] - - except AssertionError: - import traceback - error_line = traceback.format_exc().splitlines()[-2].replace('[obj]', '['+repr(obj)+']') - print("Error while testing (byref=%s):" % byref, error_line, sep="\n", file=sys.stderr) - raise - - -# Test session loading in a fresh interpreter session. -if __name__ == '__main__' and len(sys.argv) >= 3 and sys.argv[1] == '--child': - byref = sys.argv[2] == 'True' - dill.load_session(session_file % byref) - test_modules(__main__, byref) - sys.exit() - -del test_modules - - -def _clean_up_cache(module): - cached = module.__file__.split('.', 1)[0] + '.pyc' - cached = module.__cached__ if hasattr(module, '__cached__') else cached - pycache = os.path.join(os.path.dirname(module.__file__), '__pycache__') - for remove, file in [(os.remove, cached), (os.removedirs, pycache)]: - try: - remove(file) - except OSError: - pass - - -# To clean up namespace before loading the session. -original_modules = set(sys.modules.keys()) - \ - set(['json', 'urllib', 'xml.sax', 'xml.dom.minidom', 'calendar', 'cmath']) -original_objects = set(__main__.__dict__.keys()) -original_objects.add('original_objects') - - -# Create various kinds of objects to test different internal logics. - -## Modules. -import json # top-level module -import urllib as url # top-level module under alias -from xml import sax # submodule -import xml.dom.minidom as dom # submodule under alias -import test_dictviews as local_mod # non-builtin top-level module -atexit.register(_clean_up_cache, local_mod) - -## Imported objects. -from calendar import Calendar, isleap, day_name # class, function, other object -from cmath import log as complex_log # imported with alias - -## Local objects. -x = 17 -empty = None -names = ['Alice', 'Bob', 'Carol'] -def squared(x): return x**2 -cubed = lambda x: x**3 -class Person: - def __init__(self, name, age): - self.name = name - self.age = age -person = Person(names[0], x) -class CalendarSubclass(Calendar): - def weekdays(self): - return [day_name[i] for i in self.iterweekdays()] -cal = CalendarSubclass() -selfref = __main__ - - -def test_objects(main, copy_dict, byref): - main_dict = main.__dict__ - - try: - for obj in ('json', 'url', 'local_mod', 'sax', 'dom'): - assert main_dict[obj].__name__ == copy_dict[obj].__name__ - - #FIXME: In the second test call, 'calendar' is not included in - # sys.modules, independent of the value of byref. Tried to run garbage - # collection before with no luck. This block fails even with - # "import calendar" before it. Needed to restore the original modules - # with the 'copy_modules' object. (Moved to "test_session_{1,2}.py".) - - #for obj in ('Calendar', 'isleap'): - # assert main_dict[obj] is sys.modules['calendar'].__dict__[obj] - #assert main_dict['day_name'].__module__ == 'calendar' - #if byref: - # assert main_dict['day_name'] is sys.modules['calendar'].__dict__['day_name'] - - for obj in ('x', 'empty', 'names'): - assert main_dict[obj] == copy_dict[obj] - - globs = '__globals__' if dill._dill.PY3 else 'func_globals' - for obj in ['squared', 'cubed']: - assert getattr(main_dict[obj], globs) is main_dict - assert main_dict[obj](3) == copy_dict[obj](3) - - assert main.Person.__module__ == main.__name__ - assert isinstance(main.person, main.Person) - assert main.person.age == copy_dict['person'].age - - assert issubclass(main.CalendarSubclass, main.Calendar) - assert isinstance(main.cal, main.CalendarSubclass) - assert main.cal.weekdays() == copy_dict['cal'].weekdays() - - assert main.selfref is main - - except AssertionError: - import traceback - error_line = traceback.format_exc().splitlines()[-2].replace('[obj]', '['+repr(obj)+']') - print("Error while testing (byref=%s):" % byref, error_line, sep="\n", file=sys.stderr) - raise - - -if __name__ == '__main__': - - # Test dump_session() and load_session(). - for byref in (False, True): - if byref: - # Test unpickleable imported object in main. - from sys import flags - - #print(sorted(set(sys.modules.keys()) - original_modules)) - dill._test_file = dill._dill.StringIO() - try: - # For the subprocess. - dill.dump_session(session_file % byref, byref=byref) - - dill.dump_session(dill._test_file, byref=byref) - dump = dill._test_file.getvalue() - dill._test_file.close() - - import __main__ - copy_dict = __main__.__dict__.copy() - copy_modules = sys.modules.copy() - del copy_dict['dump'] - del copy_dict['__main__'] - for name in copy_dict.keys(): - if name not in original_objects: - del __main__.__dict__[name] - for module in list(sys.modules.keys()): - if module not in original_modules: - del sys.modules[module] - - dill._test_file = dill._dill.StringIO(dump) - dill.load_session(dill._test_file) - #print(sorted(set(sys.modules.keys()) - original_modules)) - - # Test session loading in a new session. - from dill.tests.__main__ import python, shell, sp - error = sp.call([python, __file__, '--child', str(byref)], shell=shell) - if error: sys.exit(error) - del python, shell, sp - - finally: - dill._test_file.close() - try: - os.remove(session_file % byref) - except OSError: - pass - - test_objects(__main__, copy_dict, byref) - __main__.__dict__.update(copy_dict) - sys.modules.update(copy_modules) - del __main__, copy_dict, copy_modules, dump - - - # This is for code coverage, tests the use case of dump_session(byref=True) - # without imported objects in the namespace. It's a contrived example because - # even dill can't be in it. - from types import ModuleType - modname = '__test_main__' - main = ModuleType(modname) - main.x = 42 - - _main = dill._dill._stash_modules(main) - if _main is not main: - print("There are objects to save by referenece that shouldn't be:", - _main.__dill_imported, _main.__dill_imported_as, _main.__dill_imported_top_level, - file=sys.stderr) - - test_file = dill._dill.StringIO() - try: - dill.dump_session(test_file, main=main, byref=True) - dump = test_file.getvalue() - test_file.close() - - sys.modules[modname] = ModuleType(modname) # empty - # This should work after fixing https://github.com/uqfoundation/dill/issues/462 - test_file = dill._dill.StringIO(dump) - dill.load_session(test_file) - finally: - test_file.close() - - assert x == 42 - - - # Dump session for module that is not __main__: - import test_classdef as module - atexit.register(_clean_up_cache, module) - module.selfref = module - dict_objects = [obj for obj in module.__dict__.keys() if not obj.startswith('__')] - - test_file = dill._dill.StringIO() - try: - dill.dump_session(test_file, main=module) - dump = test_file.getvalue() - test_file.close() - - for obj in dict_objects: - del module.__dict__[obj] - - test_file = dill._dill.StringIO(dump) - dill.load_session(test_file, main=module) - finally: - test_file.close() - - assert all(obj in module.__dict__ for obj in dict_objects) - assert module.selfref is module diff --git a/tox.ini b/tox.ini index 55b8f077..12c896c0 100644 --- a/tox.ini +++ b/tox.ini @@ -15,8 +15,7 @@ envlist = deps = # numpy whitelist_externals = - bash +# bash commands = {envpython} -m pip install . - bash -c "failed=0; for test in tests/__main__.py; do echo $test; \ - {envpython} $test || failed=1; done; exit $failed" + {envpython} dill/tests/__main__.py diff --git a/version.py b/version.py new file mode 100644 index 00000000..465f3ce4 --- /dev/null +++ b/version.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2022 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/dill/blob/master/LICENSE + +__version__ = '0.3.6.dev0' +__author__ = 'Mike McKerns' +__contact__ = 'mmckerns@uqfoundation.org' + + +def get_license_text(filepath): + "open the LICENSE file and read the contents" + try: + LICENSE = open(filepath).read() + except: + LICENSE = '' + return LICENSE + + +def get_readme_as_rst(filepath): + "open the README file and read the markdown as rst" + try: + fh = open(filepath) + name, null = fh.readline().rstrip(), fh.readline() + tag, null = fh.readline(), fh.readline() + tag = "%s: %s" % (name, tag) + split = '-'*(len(tag)-1)+'\n' + README = ''.join((null,split,tag,split,'\n')) + skip = False + for line in fh: + if line.startswith('['): + continue + elif skip and line.startswith(' http'): + README += '\n' + line + elif line.startswith('* with'): #XXX: don't indent + README += line + elif line.startswith('* '): + README += line.replace('* ',' - ',1) + elif line.startswith('-'): + README += line.replace('-','=') + '\n' + else: + README += line + skip = line.endswith(':\n') + fh.close() + except: + README = '' + return README + + +def write_info_file(dirpath, modulename, **info): + """write the given info to 'modulename/__info__.py' + + info expects: + doc: the module's long_description + version: the module's version string + author: the module's author string + license: the module's license contents + """ + import os + infofile = os.path.join(dirpath, '%s/__info__.py' % modulename) + header = '''#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2022 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/%s/blob/master/LICENSE +''' % modulename #XXX: author and email are hardwired in the header + doc = info.get('doc', None) + version = info.get('version', None) + author = info.get('author', None) + license = info.get('license', None) + with open(infofile, 'w') as fh: + fh.write(header) + if doc is not None: fh.write("'''%s'''\n\n" % doc) + if version is not None: fh.write("__version__ = %r\n" % version) + if author is not None: fh.write("__author__ = %r\n\n" % author) + if license is not None: fh.write("__license__ = '''\n%s'''\n" % license) + return