From e15a4d73c40af1d5345522f62d54ac1ad4554fd2 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 10 Feb 2020 16:10:01 +0100 Subject: [PATCH 01/11] remove Python 2 entries from CI config --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 4b8ac6ddd..5eae6fe7d 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -29,7 +29,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python_version: [2.7, 3.5, 3.6, 3.7, 3.8, "pypy3"] + python_version: [3.5, 3.6, 3.7, 3.8, "pypy3"] exclude: # Do not test all minor versions on all platforms, especially if they # are not the oldest/newest supported versions From c33841295597f93623534c440b232b167d89d9e4 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 10 Feb 2020 16:14:48 +0100 Subject: [PATCH 02/11] remove Python 2-only tests/pytest-directives --- tests/cloudpickle_test.py | 40 ++------------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index db5a38909..41147845c 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -204,9 +204,6 @@ def test_memoryview(self): self.assertEqual(pickle_depickle(buffer_obj, protocol=self.protocol), buffer_obj.tobytes()) - @pytest.mark.skipif(sys.version_info < (3, 4), - reason="non-contiguous memoryview not implemented in " - "old Python versions") def test_sliced_and_non_contiguous_memoryview(self): buffer_obj = memoryview(b"Hello!" * 3)[2:15:2] self.assertEqual(pickle_depickle(buffer_obj, protocol=self.protocol), @@ -473,8 +470,6 @@ def f(self, x): g = pickle_depickle(F.f, protocol=self.protocol) self.assertEqual(g.__name__, F.f.__name__) - if sys.version_info[0] < 3: - self.assertEqual(g.im_class.__name__, F.f.im_class.__name__) # self.assertEqual(g(F(), 1), 2) # still fails def test_module(self): @@ -843,9 +838,8 @@ def test_builtin_slotmethod(self): assert depickled_clsdict_meth is clsdict_slotmethod @pytest.mark.skipif( - platform.python_implementation() == "PyPy" or - sys.version_info[:1] < (3,), - reason="No known staticmethod example in the python 2 / pypy stdlib") + platform.python_implementation() == "PyPy", + reason="No known staticmethod example in the pypy stdlib") def test_builtin_staticmethod(self): obj = "foo" # str object @@ -1740,33 +1734,6 @@ def process_data(): """.format(protocol=self.protocol) assert_run_python_script(code) - @pytest.mark.skipif(sys.version_info >= (3, 0), - reason="hardcoded pickle bytes for 2.7") - def test_function_pickle_compat_0_4_0(self): - # The result of `cloudpickle.dumps(lambda x: x)` in cloudpickle 0.4.0, - # Python 2.7 - pickled = (b'\x80\x02ccloudpickle.cloudpickle\n_fill_function\nq\x00(c' - b'cloudpickle.cloudpickle\n_make_skel_func\nq\x01ccloudpickle.clou' - b'dpickle\n_builtin_type\nq\x02U\x08CodeTypeq\x03\x85q\x04Rq\x05(K' - b'\x01K\x01K\x01KCU\x04|\x00\x00Sq\x06N\x85q\x07)U\x01xq\x08\x85q' - b'\tU\x07q\nU\x08q\x0bK\x01U\x00q\x0c))tq\rRq\x0eJ' - b'\xff\xff\xff\xff}q\x0f\x87q\x10Rq\x11}q\x12N}q\x13NtR.') - self.assertEqual(42, cloudpickle.loads(pickled)(42)) - - @pytest.mark.skipif(sys.version_info >= (3, 0), - reason="hardcoded pickle bytes for 2.7") - def test_function_pickle_compat_0_4_1(self): - # The result of `cloudpickle.dumps(lambda x: x)` in cloudpickle 0.4.1, - # Python 2.7 - pickled = (b'\x80\x02ccloudpickle.cloudpickle\n_fill_function\nq\x00(c' - b'cloudpickle.cloudpickle\n_make_skel_func\nq\x01ccloudpickle.clou' - b'dpickle\n_builtin_type\nq\x02U\x08CodeTypeq\x03\x85q\x04Rq\x05(K' - b'\x01K\x01K\x01KCU\x04|\x00\x00Sq\x06N\x85q\x07)U\x01xq\x08\x85q' - b'\tU\x07q\nU\x08q\x0bK\x01U\x00q\x0c))tq\rRq\x0eJ' - b'\xff\xff\xff\xff}q\x0f\x87q\x10Rq\x11}q\x12N}q\x13U\x08__main__q' - b'\x14NtR.') - self.assertEqual(42, cloudpickle.loads(pickled)(42)) - def test_pickle_reraise(self): for exc_type in [Exception, ValueError, TypeError, RuntimeError]: obj = RaiserOnPickle(exc_type("foo")) @@ -2006,9 +1973,6 @@ def g(): cloned_func = pickle_depickle(func, protocol=self.protocol) assert cloned_func() == "hello from a {}!".format(source) - @pytest.mark.skipif(sys.version_info[0] < 3, - reason="keyword only arguments were introduced in " - "python 3") def test_interactively_defined_func_with_keyword_only_argument(self): # fixes https://github.com/cloudpipe/cloudpickle/issues/263 # The source code of this test is bundled in a string and is ran from From 1e2a1d702437e4bd87da8bd70071dc968b63aebc Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 10 Feb 2020 16:43:04 +0100 Subject: [PATCH 03/11] cleanup Python 2 only code in cloudpickle --- cloudpickle/cloudpickle.py | 182 ++++++++------------------------ cloudpickle/cloudpickle_fast.py | 4 +- 2 files changed, 44 insertions(+), 142 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 861be4b26..e8be85785 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -85,24 +85,10 @@ # builtin-code objects only exist in pypy builtin_code_type = type(float.__new__.__code__) -if sys.version_info[0] < 3: # pragma: no branch - from pickle import Pickler - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO - import __builtin__ as builtins - string_types = (basestring,) # noqa - PY3 = False - PY2 = True -else: - from pickle import _Pickler as Pickler - from io import BytesIO as StringIO - string_types = (str,) - PY3 = True - PY2 = False - from importlib._bootstrap import _find_spec - import builtins +from pickle import _Pickler as Pickler +from io import BytesIO +from importlib._bootstrap import _find_spec +import builtins _extract_code_globals_cache = weakref.WeakKeyDictionary() @@ -355,41 +341,23 @@ def _cell_set_factory(value): co = _cell_set_factory.__code__ - if PY2: # pragma: no branch - _cell_set_template_code = types.CodeType( - co.co_argcount, - co.co_nlocals, - co.co_stacksize, - co.co_flags, - co.co_code, - co.co_consts, - co.co_names, - co.co_varnames, - co.co_filename, - co.co_name, - co.co_firstlineno, - co.co_lnotab, - co.co_cellvars, # co_freevars is initialized with co_cellvars - (), # co_cellvars is made empty - ) - else: - _cell_set_template_code = types.CodeType( - co.co_argcount, - co.co_kwonlyargcount, # Python 3 only argument - co.co_nlocals, - co.co_stacksize, - co.co_flags, - co.co_code, - co.co_consts, - co.co_names, - co.co_varnames, - co.co_filename, - co.co_name, - co.co_firstlineno, - co.co_lnotab, - co.co_cellvars, # co_freevars is initialized with co_cellvars - (), # co_cellvars is made empty - ) + _cell_set_template_code = types.CodeType( + co.co_argcount, + co.co_kwonlyargcount, # Python 3 only argument + co.co_nlocals, + co.co_stacksize, + co.co_flags, + co.co_code, + co.co_consts, + co.co_names, + co.co_varnames, + co.co_filename, + co.co_name, + co.co_firstlineno, + co.co_lnotab, + co.co_cellvars, # co_freevars is initialized with co_cellvars + (), # co_cellvars is made empty + ) return _cell_set_template_code @@ -422,8 +390,6 @@ def _walk_global_ops(code): global-referencing instructions in *code*. """ code = getattr(code, 'co_code', b'') - if PY2: # pragma: no branch - code = map(ord, code) n = len(code) i = 0 @@ -501,12 +467,6 @@ def save_memoryview(self, obj): dispatch[memoryview] = save_memoryview - if PY2: # pragma: no branch - def save_buffer(self, obj): - self.save(str(obj)) - - dispatch[buffer] = save_buffer # noqa: F821 'buffer' was removed in Python 3 - def save_module(self, obj): """ Save a module as an import @@ -524,29 +484,22 @@ def save_codeobject(self, obj): """ Save a code object """ - if PY3: # pragma: no branch - if hasattr(obj, "co_posonlyargcount"): # pragma: no branch - args = ( - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, - obj.co_cellvars - ) - else: - args = ( - obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, - obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, - obj.co_names, obj.co_varnames, obj.co_filename, - obj.co_name, obj.co_firstlineno, obj.co_lnotab, - obj.co_freevars, obj.co_cellvars - ) + if hasattr(obj, "co_posonlyargcount"): # pragma: no branch + args = ( + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, + obj.co_cellvars + ) else: args = ( - obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, - obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars + obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, + obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, + obj.co_names, obj.co_varnames, obj.co_filename, + obj.co_name, obj.co_firstlineno, obj.co_lnotab, + obj.co_freevars, obj.co_cellvars ) self.save_reduce(types.CodeType, args, obj=obj) @@ -658,7 +611,7 @@ def save_dynamic_class(self, obj): # pickle string length optimization: member descriptors of obj are # created automatically from obj's __slots__ attribute, no need to # save them in obj's state - if isinstance(obj.__slots__, string_types): + if isinstance(obj.__slots__, str): clsdict.pop(obj.__slots__) else: for k in obj.__slots__: @@ -833,45 +786,6 @@ def extract_func_data(self, func): return (code, f_globals, defaults, closure, dct, base_globals) - if not PY3: # pragma: no branch - # Python3 comes with native reducers that allow builtin functions and - # methods pickling as module/class attributes. The following method - # extends this for python2. - # Please note that currently, neither pickle nor cloudpickle support - # dynamically created builtin functions/method pickling. - def save_builtin_function_or_method(self, obj): - is_bound = getattr(obj, '__self__', None) is not None - if is_bound: - # obj is a bound builtin method. - rv = (getattr, (obj.__self__, obj.__name__)) - return self.save_reduce(obj=obj, *rv) - - is_unbound = hasattr(obj, '__objclass__') - if is_unbound: - # obj is an unbound builtin method (accessed from its class) - rv = (getattr, (obj.__objclass__, obj.__name__)) - return self.save_reduce(obj=obj, *rv) - - # Otherwise, obj is not a method, but a function. Fallback to - # default pickling by attribute. - return Pickler.save_global(self, obj) - - dispatch[types.BuiltinFunctionType] = save_builtin_function_or_method - - # A comprehensive summary of the various kinds of builtin methods can - # be found in PEP 579: https://www.python.org/dev/peps/pep-0579/ - classmethod_descriptor_type = type(float.__dict__['fromhex']) - wrapper_descriptor_type = type(float.__repr__) - method_wrapper_type = type(1.5.__repr__) - - dispatch[classmethod_descriptor_type] = save_builtin_function_or_method - dispatch[wrapper_descriptor_type] = save_builtin_function_or_method - dispatch[method_wrapper_type] = save_builtin_function_or_method - - if sys.version_info[:2] < (3, 4): - method_descriptor = type(str.upper) - dispatch[method_descriptor] = save_builtin_function_or_method - def save_getset_descriptor(self, obj): return self.save_reduce(getattr, (obj.__objclass__, obj.__name__)) @@ -901,20 +815,13 @@ def save_global(self, obj, name=None, pack=struct.pack): Pickler.save_global(self, obj, name=name) dispatch[type] = save_global - if PY2: - dispatch[types.ClassType] = save_global def save_instancemethod(self, obj): # Memoization rarely is ever useful due to python bounding if obj.__self__ is None: self.save_reduce(getattr, (obj.im_class, obj.__name__)) else: - if PY3: # pragma: no branch - self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj) - else: - self.save_reduce( - types.MethodType, - (obj.__func__, obj.__self__, type(obj.__self__)), obj=obj) + self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj) dispatch[types.MethodType] = save_instancemethod @@ -963,12 +870,10 @@ def save_inst(self, obj): save(stuff) write(pickle.BUILD) - if PY2: # pragma: no branch - dispatch[types.InstanceType] = save_inst - def save_property(self, obj): # properties not correctly saved in python - self.save_reduce(property, (obj.fget, obj.fset, obj.fdel, obj.__doc__), obj=obj) + self.save_reduce(property, (obj.fget, obj.fset, obj.fdel, obj.__doc__), + obj=obj) dispatch[property] = save_property @@ -1016,10 +921,6 @@ def __getattribute__(self, item): def save_file(self, obj): """Save a file""" - try: - import StringIO as pystringIO # we can't use cStringIO as it lacks the name attribute - except ImportError: - import io as pystringIO if not hasattr(obj, 'name') or not hasattr(obj, 'mode'): raise pickle.PicklingError("Cannot pickle files that do not map to an actual file") @@ -1038,7 +939,8 @@ def save_file(self, obj): name = obj.name - retval = pystringIO.StringIO() + # TODO: also support binary mode files with io.BytesIO + retval = io.StringIO() try: # Read the whole file @@ -1142,7 +1044,7 @@ def dumps(obj, protocol=None): Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure compatibility with older versions of Python. """ - file = StringIO() + file = BytesIO() try: cp = CloudPickler(file, protocol=protocol) cp.dump(obj) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 738467166..1ea235ca3 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -27,7 +27,7 @@ _is_dynamic, _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, _find_imported_submodules, _get_cell_contents, _is_global, _builtin_type, Enum, _ensure_tracking, _make_skeleton_class, _make_skeleton_enum, - _extract_class_dict, string_types, dynamic_subimport, subimport + _extract_class_dict, dynamic_subimport, subimport ) load, loads = _pickle.load, _pickle.loads @@ -149,7 +149,7 @@ def _class_getstate(obj): # pickle string length optimization: member descriptors of obj are # created automatically from obj's __slots__ attribute, no need to # save them in obj's state - if isinstance(obj.__slots__, string_types): + if isinstance(obj.__slots__, str): clsdict.pop(obj.__slots__) else: for k in obj.__slots__: From ca033b736d03c2e86cf96f46981e73054c47bf3c Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 10 Feb 2020 16:44:59 +0100 Subject: [PATCH 04/11] make flake8 happy --- cloudpickle/cloudpickle.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index e8be85785..734eb80d2 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -43,6 +43,7 @@ from __future__ import print_function import abc +import builtins import dis from functools import partial import io @@ -60,6 +61,10 @@ import uuid import threading +from pickle import _Pickler as Pickler +from io import BytesIO +from importlib._bootstrap import _find_spec + try: from enum import Enum @@ -85,11 +90,6 @@ # builtin-code objects only exist in pypy builtin_code_type = type(float.__new__.__code__) -from pickle import _Pickler as Pickler -from io import BytesIO -from importlib._bootstrap import _find_spec -import builtins - _extract_code_globals_cache = weakref.WeakKeyDictionary() From 077da7378ac8c279a77b77732a5825780cf0da47 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 10 Feb 2020 16:52:33 +0100 Subject: [PATCH 05/11] remove Python 3.4 only tests/pytest-directives --- tests/cloudpickle_test.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 41147845c..c34fdf0c7 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -333,9 +333,6 @@ def it_works(self): self.assertEqual(depickled_C.instance_of_C.it_works(), "woohoo!") self.assertEqual(depickled_instance.it_works(), "woohoo!") - @pytest.mark.skipif(sys.version_info >= (3, 4) - and sys.version_info < (3, 4, 3), - reason="subprocess has a bug in 3.4.0 to 3.4.2") def test_locally_defined_function_and_class(self): LOCAL_CONSTANT = 42 @@ -1861,8 +1858,6 @@ def test_dataclass(self): pickle_depickle(DataClass, protocol=self.protocol) assert data.x == pickle_depickle(data, protocol=self.protocol).x == 42 - @unittest.skipIf(sys.version_info[:2] < (3, 5), - "Only support enum pickling on Python 3.5+") def test_locally_defined_enum(self): enum = pytest.importorskip("enum") @@ -1894,8 +1889,6 @@ def is_green(self): green3 = pickle_depickle(Color.GREEN, protocol=self.protocol) assert green3 is Color.GREEN - @unittest.skipIf(sys.version_info[:2] < (3, 5), - "Only support enum pickling on Python 3.5+") def test_locally_defined_intenum(self): enum = pytest.importorskip("enum") # Try again with a IntEnum defined with the functional API @@ -1910,8 +1903,6 @@ def test_locally_defined_intenum(self): assert green1 is not ClonedDynamicColor.BLUE assert ClonedDynamicColor is DynamicColor - @unittest.skipIf(sys.version_info[:2] < (3, 5), - "Only support enum pickling on Python 3.5+") def test_interactively_defined_enum(self): pytest.importorskip("enum") code = """if __name__ == "__main__": From a3ea59402bf318ebfbb9a5e9e5b305cc58030d85 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 10 Feb 2020 16:54:09 +0100 Subject: [PATCH 06/11] update tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 086cc91ff..a1f0fbd02 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py34, py35, py36, py37, pypy, pypy3 +envlist = py35, py36, py37, py38, pypy3 [testenv] deps = -rdev-requirements.txt From dc86b46d83d897e9b4ffa4050acd54f18c2d8015 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 10 Feb 2020 17:03:23 +0100 Subject: [PATCH 07/11] remove python 2-only code in testutils --- tests/testutils.py | 41 ++++++++--------------------------------- 1 file changed, 8 insertions(+), 33 deletions(-) diff --git a/tests/testutils.py b/tests/testutils.py index e26849758..303d0a996 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -10,18 +10,9 @@ import psutil from cloudpickle import dumps +from subprocess import TimeoutExpired TIMEOUT = 60 -try: - from subprocess import TimeoutExpired - timeout_supported = True -except ImportError: - # no support for timeout in Python 2 - class TimeoutExpired(Exception): - pass - timeout_supported = False - - TEST_GLOBALS = "a test value" @@ -46,20 +37,6 @@ def _make_cwd_env(): return cloudpickle_repo_folder, env -def _pack(input_data, protocol=None): - pickled_input_data = dumps(input_data, protocol=protocol) - # Under Windows + Python 2.7, subprocess / communicate truncate the data - # on some specific bytes. To avoid this issue, let's use the pure ASCII - # Base32 encoding to encapsulate the pickle message sent to the child - # process. - return base64.b32encode(pickled_input_data) - - -def _unpack(packed_data): - decoded_data = base64.b32decode(packed_data) - return loads(decoded_data) - - def subprocess_pickle_echo(input_data, protocol=None, timeout=TIMEOUT): """Echo function with a child Python process @@ -80,17 +57,16 @@ def subprocess_pickle_echo(input_data, protocol=None, timeout=TIMEOUT): cwd, env = _make_cwd_env() proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=cwd, env=env, bufsize=4096) - pickled_b32 = _pack(input_data, protocol=protocol) + pickle_string = dumps(input_data, protocol=protocol) try: comm_kwargs = {} - if timeout_supported: - comm_kwargs['timeout'] = timeout - out, err = proc.communicate(pickled_b32, **comm_kwargs) + comm_kwargs['timeout'] = timeout + out, err = proc.communicate(pickle_string, **comm_kwargs) if proc.returncode != 0 or len(err): message = "Subprocess returned %d: " % proc.returncode message += err.decode('utf-8') raise RuntimeError(message) - return _unpack(out) + return loads(out) except TimeoutExpired: proc.kill() out, err = proc.communicate() @@ -121,11 +97,11 @@ def pickle_echo(stream_in=None, stream_out=None, protocol=None): if hasattr(stream_out, 'buffer'): stream_out = stream_out.buffer - input_bytes = base64.b32decode(_read_all_bytes(stream_in)) + input_bytes = _read_all_bytes(stream_in) stream_in.close() obj = loads(input_bytes) repickled_bytes = dumps(obj, protocol=protocol) - stream_out.write(base64.b32encode(repickled_bytes)) + stream_out.write(repickled_bytes) stream_out.close() @@ -201,8 +177,7 @@ def assert_run_python_script(source_code, timeout=TIMEOUT): coverage_rc = os.environ.get("COVERAGE_PROCESS_START") if coverage_rc: kwargs['env']['COVERAGE_PROCESS_START'] = coverage_rc - if timeout_supported: - kwargs['timeout'] = timeout + kwargs['timeout'] = timeout try: try: out = check_output(cmd, **kwargs) From 58bebbc3fb9532ffeb36d8fbe6fc97556bb8acb5 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 10 Feb 2020 17:12:33 +0100 Subject: [PATCH 08/11] more Python 2 cleanups --- cloudpickle/cloudpickle.py | 58 ++++++---------------------------- tests/cloudpickle_file_test.py | 11 ------- 2 files changed, 10 insertions(+), 59 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 734eb80d2..317bc6600 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -62,6 +62,7 @@ import threading from pickle import _Pickler as Pickler +from pickle import _getattribute from io import BytesIO from importlib._bootstrap import _find_spec @@ -111,21 +112,6 @@ def _lookup_class_or_track(class_tracker_id, class_def): _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id return class_def -if sys.version_info[:2] >= (3, 5): - from pickle import _getattribute -elif sys.version_info[:2] >= (3, 4): - from pickle import _getattribute as _py34_getattribute - # pickle._getattribute does not return the parent under Python 3.4 - def _getattribute(obj, name): - return _py34_getattribute(obj, name), None -else: - # pickle._getattribute is a python3 addition and enchancement of getattr, - # that can handle dotted attribute names. In cloudpickle for python2, - # handling dotted names is not needed, so we simply define _getattribute as - # a wrapper around getattr. - def _getattribute(obj, name): - return getattr(obj, name, None), None - def _whichmodule(obj, name): """Find the module an object belongs to. @@ -383,39 +369,15 @@ def _builtin_type(name): return getattr(types, name) -if sys.version_info < (3, 4): # pragma: no branch - def _walk_global_ops(code): - """ - Yield (opcode, argument number) tuples for all - global-referencing instructions in *code*. - """ - code = getattr(code, 'co_code', b'') - - n = len(code) - i = 0 - extended_arg = 0 - while i < n: - op = code[i] - i += 1 - if op >= HAVE_ARGUMENT: - oparg = code[i] + code[i + 1] * 256 + extended_arg - extended_arg = 0 - i += 2 - if op == EXTENDED_ARG: - extended_arg = oparg * 65536 - if op in GLOBAL_OPS: - yield op, oparg - -else: - def _walk_global_ops(code): - """ - Yield (opcode, argument number) tuples for all - global-referencing instructions in *code*. - """ - for instr in dis.get_instructions(code): - op = instr.opcode - if op in GLOBAL_OPS: - yield op, instr.arg +def _walk_global_ops(code): + """ + Yield (opcode, argument number) tuples for all + global-referencing instructions in *code*. + """ + for instr in dis.get_instructions(code): + op = instr.opcode + if op in GLOBAL_OPS: + yield op, instr.arg def _extract_class_dict(cls): diff --git a/tests/cloudpickle_file_test.py b/tests/cloudpickle_file_test.py index 16abc9cc6..4f05186e3 100644 --- a/tests/cloudpickle_file_test.py +++ b/tests/cloudpickle_file_test.py @@ -81,17 +81,6 @@ def test_seek(self): self.assertEqual(self.teststring, unpickled.read()) os.remove(self.tmpfilepath) - @pytest.mark.skipif(sys.version_info >= (3,), - reason="only works on Python 2.x") - def test_temp_file(self): - with tempfile.NamedTemporaryFile(mode='ab+') as fp: - fp.write(self.teststring.encode('UTF-8')) - fp.seek(0) - f = fp.file - # FIXME this doesn't work yet: cloudpickle.dumps(fp) - newfile = pickle.loads(cloudpickle.dumps(f)) - self.assertEqual(self.teststring, newfile.read()) - def test_pickling_special_file_handles(self): # Warning: if you want to run your tests with nose, add -s option for out in sys.stdout, sys.stderr: # Regression test for SPARK-3415 From f91161a3bf2b969cc1770c99ff4dcd2af5a2b522 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 10 Feb 2020 17:29:57 +0100 Subject: [PATCH 09/11] more Python 2-related cleanups --- cloudpickle/cloudpickle.py | 18 +++++++----------- dev-requirements.txt | 1 - tests/cloudpickle_test.py | 13 ++----------- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 317bc6600..9925424b5 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -515,13 +515,12 @@ def _save_dynamic_enum(self, obj, clsdict): """ members = dict((e.name, e.value) for e in obj) - # Python 2.7 with enum34 can have no qualname: - qualname = getattr(obj, "__qualname__", None) - - self.save_reduce(_make_skeleton_enum, - (obj.__bases__, obj.__name__, qualname, members, - obj.__module__, _ensure_tracking(obj), None), - obj=obj) + self.save_reduce( + _make_skeleton_enum, + (obj.__bases__, obj.__name__, obj.__qualname__, + members, obj.__module__, _ensure_tracking(obj), None), + obj=obj + ) # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class: # Those attributes are already handled by the metaclass. @@ -1228,10 +1227,7 @@ class id will also reuse this enum definition. classdict[member_name] = member_value enum_class = metacls.__new__(metacls, name, bases, classdict) enum_class.__module__ = module - - # Python 2.7 compat - if qualname is not None: - enum_class.__qualname__ = qualname + enum_class.__qualname__ = qualname return _lookup_class_or_track(class_tracker_id, enum_class) diff --git a/dev-requirements.txt b/dev-requirements.txt index 1849f20d5..89f163825 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,7 +3,6 @@ flake8 pytest pytest-cov psutil -futures; python_version < '3.4' # Code coverage uploader for Travis: codecov coverage diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index c34fdf0c7..b525d8f93 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -1966,23 +1966,14 @@ def g(): def test_interactively_defined_func_with_keyword_only_argument(self): # fixes https://github.com/cloudpipe/cloudpickle/issues/263 - # The source code of this test is bundled in a string and is ran from - # the __main__ module of a subprocess in order to avoid a SyntaxError - # in python2 when pytest imports this file, as the keyword-only syntax - # is python3-only. - code = """ - from cloudpickle import loads, dumps - def f(a, *, b=1): return a + b - depickled_f = loads(dumps(f, protocol={protocol})) + depickled_f = pickle_depickle(f, protocol=self.protocol) for func in (f, depickled_f): assert func(2) == 3 - assert func.__kwdefaults__ == {{'b': 1}} - """.format(protocol=self.protocol) - assert_run_python_script(textwrap.dedent(code)) + assert func.__kwdefaults__ == {'b': 1} @pytest.mark.skipif(not hasattr(types.CodeType, "co_posonlyargcount"), reason="Requires positional-only argument syntax") From 1761d1ba1a0717860d55f77ec7d3b58d86eab33c Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 10 Feb 2020 17:34:10 +0100 Subject: [PATCH 10/11] more Python 2-related cleanups --- tests/cloudpickle_test.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index b525d8f93..4f6e71132 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -181,11 +181,7 @@ def __reduce__(self): def foo(): sys.exit(0) - func_code = getattr(foo, '__code__', None) - if func_code is None: # PY2 backwards compatibility - func_code = foo.func_code - - self.assertTrue("exit" in func_code.co_names) + self.assertTrue("exit" in foo.__code__.co_names) cloudpickle.dumps(foo) def test_buffer(self): From b8a93febfbf22cbd604f699c5199794de3216385 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Mon, 10 Feb 2020 21:49:30 +0100 Subject: [PATCH 11/11] Trigger CI