diff --git a/dill/_dill.py b/dill/_dill.py index df3f69fa..415e2b06 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -42,6 +42,7 @@ def _trace(boolean): OLDER = (PY3 and sys.hexversion < 0x3040000) or (sys.hexversion < 0x2070ab1) OLD33 = (sys.hexversion < 0x3030000) OLD37 = (sys.hexversion < 0x3070000) +OLD38 = (sys.hexversion < 0x3080000) OLD39 = (sys.hexversion < 0x3090000) OLD310 = (sys.hexversion < 0x30a0000) PY34 = (0x3040000 <= sys.hexversion < 0x3050000) @@ -87,6 +88,7 @@ def _trace(boolean): import gc # import zlib from weakref import ReferenceType, ProxyType, CallableProxyType +from collections import OrderedDict from functools import partial from operator import itemgetter, attrgetter # new in python3.3 @@ -274,8 +276,6 @@ def get_file_type(*args, **kwargs): except NameError: ExitType = None singletontypes = [] -from collections import OrderedDict - import inspect ### Shims for different versions of Python and dill @@ -572,7 +572,6 @@ def __init__(self, *args, **kwds): self._strictio = False #_strictio self._fmode = settings['fmode'] if _fmode is None else _fmode self._recurse = settings['recurse'] if _recurse is None else _recurse - from collections import OrderedDict self._postproc = OrderedDict() def dump(self, obj): #NOTE: if settings change, need to update attributes @@ -723,6 +722,15 @@ def _create_typemap(): 'SuperType': SuperType, 'ItemGetterType': ItemGetterType, 'AttrGetterType': AttrGetterType, +}) + +# "Incidental" implementation specific types. Unpickling these types in another +# implementation of Python (PyPy -> CPython) is not gauranteed to work + +# This dictionary should contain all types that appear in Python implementations +# but are not defined in https://docs.python.org/3/library/types.html#standard-interpreter-types +x=OrderedDict() +_incedental_reverse_typemap = { 'FileType': FileType, 'BufferedRandomType': BufferedRandomType, 'BufferedReaderType': BufferedReaderType, @@ -732,18 +740,62 @@ def _create_typemap(): 'PyBufferedReaderType': PyBufferedReaderType, 'PyBufferedWriterType': PyBufferedWriterType, 'PyTextWrapperType': PyTextWrapperType, -}) +} + +if PY3: + _incedental_reverse_typemap.update({ + "DictKeysType": type({}.keys()), + "DictValuesType": type({}.values()), + "DictItemsType": type({}.items()), + + "OdictKeysType": type(x.keys()), + "OdictValuesType": type(x.values()), + "OdictItemsType": type(x.items()), + }) +else: + _incedental_reverse_typemap.update({ + "DictKeysType": type({}.viewkeys()), + "DictValuesType": type({}.viewvalues()), + "DictItemsType": type({}.viewitems()), + }) + if ExitType: - _reverse_typemap['ExitType'] = ExitType + _incedental_reverse_typemap['ExitType'] = ExitType if InputType: - _reverse_typemap['InputType'] = InputType - _reverse_typemap['OutputType'] = OutputType + _incedental_reverse_typemap['InputType'] = InputType + _incedental_reverse_typemap['OutputType'] = OutputType if not IS_PYPY: - _reverse_typemap['WrapperDescriptorType'] = WrapperDescriptorType - _reverse_typemap['MethodDescriptorType'] = MethodDescriptorType - _reverse_typemap['ClassMethodDescriptorType'] = ClassMethodDescriptorType + _incedental_reverse_typemap['WrapperDescriptorType'] = WrapperDescriptorType + _incedental_reverse_typemap['MethodDescriptorType'] = MethodDescriptorType + _incedental_reverse_typemap['ClassMethodDescriptorType'] = ClassMethodDescriptorType else: - _reverse_typemap['MemberDescriptorType'] = MemberDescriptorType + _incedental_reverse_typemap['MemberDescriptorType'] = MemberDescriptorType + +try: + import symtable + _incedental_reverse_typemap["SymtableStentryType"] = type(symtable.symtable("", "string", "exec")._table) +except: + pass + +if sys.hexversion >= 0x30a00a0: + _incedental_reverse_typemap['LineIteratorType'] = type(compile('3', '', 'eval').co_lines()) + +if sys.hexversion >= 0x30b00b0: + from types import GenericAlias + _incedental_reverse_typemap["GenericAliasIteratorType"] = type(iter(GenericAlias(list, (int,)))) + _incedental_reverse_typemap['PositionsIteratorType'] = type(compile('3', '', 'eval').co_positions()) + +try: + import winreg + _incedental_reverse_typemap["HKEYType"] = winreg.HKEYType +except: + pass + +_reverse_typemap.update(_incedental_reverse_typemap) +_incedental_types = set(_incedental_reverse_typemap.values()) + +del x + if PY3: _typemap = dict((v, k) for k, v in _reverse_typemap.items()) else: @@ -1988,6 +2040,8 @@ def save_module(pickler, obj): def save_type(pickler, obj, postproc_list=None): if obj in _typemap: log.info("T1: %s" % obj) + # if obj in _incedental_types: + # warnings.warn('Type %r may only exist on this implementation of Python and cannot be unpickled in other implementations.' % (obj,), PicklingWarning) pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj) log.info("# T1") elif obj.__bases__ == (tuple,) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]): @@ -2255,6 +2309,9 @@ def save_capsule(pickler, obj): destructor = _PyCapsule_GetDestructor(obj) pickler.save_reduce(_create_capsule, (pointer, name, context, destructor), obj=obj) log.info("# Cap") + _incedental_reverse_typemap['PyCapsuleType'] = PyCapsuleType + _reverse_typemap['PyCapsuleType'] = PyCapsuleType + _incedental_types.add(PyCapsuleType) else: _testcapsule = None diff --git a/dill/_objects.py b/dill/_objects.py index 31823636..f8680ad3 100644 --- a/dill/_objects.py +++ b/dill/_objects.py @@ -417,9 +417,49 @@ class _Struct(ctypes.Structure): a['FileType'] = open(os.devnull, 'rb', buffering=0) # same 'wb','wb+','rb+' # FIXME: FileType fails >= 3.1 # built-in functions (CH 2) +# Iterators: a['ListIteratorType'] = iter(_list) # empty vs non-empty FIXME: fail < 3.2 +x['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty a['TupleIteratorType']= iter(_tuple) # empty vs non-empty FIXME: fail < 3.2 a['XRangeIteratorType'] = iter(_xrange) # empty vs non-empty FIXME: fail < 3.2 +a["BytesIteratorType"] = iter(b'') +a["BytearrayIteratorType"] = iter(bytearray(b'')) +a["CallableIteratorType"] = iter(iter, None) +a["MemoryIteratorType"] = iter(memoryview(b'')) +a["ListReverseiteratorType"] = reversed([]) +X = a['OrderedDictType'] +a["OdictKeysType"] = X.keys() +a["OdictValuesType"] = X.values() +a["OdictItemsType"] = X.items() +a["OdictIteratorType"] = iter(X.keys()) +del X +if PY3: + x['DictionaryItemIteratorType'] = iter(type.__dict__.items()) + x['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) + x['DictionaryValueIteratorType'] = iter(type.__dict__.values()) +else: + x['DictionaryItemIteratorType'] = type.__dict__.iteritems() + x['DictionaryKeyIteratorType'] = type.__dict__.iterkeys() + x['DictionaryValueIteratorType'] = type.__dict__.itervalues() +if sys.hexversion >= 0x30800a0: + a["DictReversekeyiteratorType"] = reversed({}.keys()) + a["DictReversevalueiteratorType"] = reversed({}.values()) + a["DictReverseitemiteratorType"] = reversed({}.items()) + +try: + import symtable + a["SymtableEntryType"] = symtable.symtable("", "string", "exec")._table +except: + pass + +if sys.hexversion >= 0x30a00a0: + a['LineIteratorType'] = compile('3', '', 'eval').co_lines() + +if sys.hexversion >= 0x30b00b0: + from types import GenericAlias + a["GenericAliasIteratorType"] = iter(GenericAlias(list, (int,))) + a['PositionsIteratorType'] = compile('3', '', 'eval').co_positions() + # data types (CH 8) a['PrettyPrinterType'] = pprint.PrettyPrinter() #FIXME: fail >= 3.2 and == 2.5 # numeric and mathematical types (CH 9) @@ -452,16 +492,7 @@ class _Struct(ctypes.Structure): # other (concrete) object types # (also: Capsule / CObject ?) # built-in functions (CH 2) -x['SetIteratorType'] = iter(_set) #XXX: empty vs non-empty # built-in types (CH 5) -if PY3: - x['DictionaryItemIteratorType'] = iter(type.__dict__.items()) - x['DictionaryKeyIteratorType'] = iter(type.__dict__.keys()) - x['DictionaryValueIteratorType'] = iter(type.__dict__.values()) -else: - x['DictionaryItemIteratorType'] = type.__dict__.iteritems() - x['DictionaryKeyIteratorType'] = type.__dict__.iterkeys() - x['DictionaryValueIteratorType'] = type.__dict__.itervalues() # string services (CH 7) x['StructType'] = struct.Struct('c') x['CallableIteratorType'] = _srepattern.finditer('')