From 851b5493691480777e4687ace6a571b4c6ea1ee3 Mon Sep 17 00:00:00 2001 From: Anselm Kruis Date: Sat, 31 Dec 2016 20:41:26 +0100 Subject: [PATCH] Issue #114: Test suite: test all pickle protocols Enhance the test suite to test all pickle protocols using both - the C and the Python - pickle implementations. https://bitbucket.org/stackless-dev/stackless/issues/114 (grafted from 5da071b503d30801d5eaf3f44b5d3854f9b2c176) --- Stackless/unittests/support.py | 132 +++++++++++++++++++++------ Stackless/unittests/test_miscell.py | 9 +- Stackless/unittests/test_pickle.py | 82 ++--------------- Stackless/unittests/test_watchdog.py | 13 +-- 4 files changed, 125 insertions(+), 111 deletions(-) diff --git a/Stackless/unittests/support.py b/Stackless/unittests/support.py index 0d3597e1b4889e..ee334deb530854 100644 --- a/Stackless/unittests/support.py +++ b/Stackless/unittests/support.py @@ -29,14 +29,14 @@ import weakref import pickle import inspect -from io import StringIO +import io import contextlib import gc # emit warnings about uncollectable objects gc.set_debug(gc.DEBUG_UNCOLLECTABLE) try: - long + long # @UndefinedVariable except NameError: long = int # @ReservedAssignment @@ -50,7 +50,7 @@ @contextlib.contextmanager def captured_stderr(): old = sys.stderr - new = StringIO() + new = io.StringIO() sys.stderr = new try: yield new @@ -172,6 +172,77 @@ def __init__(cls, *args, **kw): class StacklessTestCase(unittest.TestCase, StacklessTestCaseMixin, metaclass=StacklessTestCaseMeta): + @classmethod + def prepare_test_method(cls, func, name): + """Called after class creation + + This method creates the _H methods, which run without + soft switching + """ + if hasattr(func, "enable_softswitch") and not getattr(func, "_H_created", False): + return ((func, name), ) + + def wrapper_hardswitch(self, method=func): + self.assertTrue(self.__setup_called, "Broken test case: it didn't call super(..., self).setUp()") + self.assertFalse(stackless.enable_softswitch(None), "softswitch is enabled") + return method(self) + wrapper_hardswitch.enable_softswitch = False + wrapper_hardswitch.__name__ = name + "_H" + if func.__doc__: + doc = func.__doc__ + if doc.startswith("(soft) "): + doc = doc[7:] + wrapper_hardswitch.__doc__ = "(hard) " + doc + setattr(cls, wrapper_hardswitch.__name__, wrapper_hardswitch) + + if not hasattr(func, "_H_created"): + func._H_created = True + func.enable_softswitch = True + if func.__doc__: + func.__doc__ = "(soft) " + func.__doc__ + return ((func, name), (wrapper_hardswitch, wrapper_hardswitch.__name__)) + + @classmethod + def prepare_pickle_test_method(cls, func, name=None): + """Called after class creation + + This method creates the Py0...n C0...n methods, which run with + the Python or C implementation of the enumerated pickle protocol. + + This method also acts as a method decorator. + """ + if name is None: + # used as a decorator + func.prepare = cls.prepare_pickle_test_method + return func + + if hasattr(func, "_pickle_created"): + return StacklessTestCase.prepare_test_method.__func__(cls, func, name) + setattr(cls, name, None) + r = [] + for i in range(0, pickle.HIGHEST_PROTOCOL + 1): + for p_letter in ("C", "P"): + def test(self, method=func, proto=i, pickle_module=p_letter, unpickle_module=p_letter): + self.assertTrue(self._StacklessTestCase__setup_called, "Broken test case: it didn't call super(..., self).setUp()") + self._pickle_protocol = proto + self._pickle_module = pickle_module + self._unpickle_module = unpickle_module + return method(self) + if i == 0 and p_letter == "C": + test.__name__ = name + else: + test.__name__ = "{:s}_{:s}{:d}".format(name, p_letter, i) + test._pickle_created = True + if func.__doc__: + doc = func.__doc__ + match = re.match(r"\([PC][0-{:d}]\)".format(pickle.HIGHEST_PROTOCOL), doc) + if match: + doc = match.string[match.end():] + test.__doc__ = "({:s}{:d}) {:s}".format(p_letter, i, doc) + setattr(cls, test.__name__, test) + r.extend(StacklessTestCase.prepare_test_method.__func__(cls, test, test.__name__)) + return r + @classmethod def prepare_test_methods(cls): """Called after class creation @@ -183,29 +254,10 @@ def prepare_test_methods(cls): for n in names: m = getattr(cls, n) if inspect.ismethod(m): - m = m.im_func - if hasattr(m, "enable_softswitch") and not getattr(m, "_H_created", False): - continue - - def wrapper_hardswitch(self, method=m): - self.assertTrue(self.__setup_called, "Broken test case: it didn't call super(..., self).setUp()") - self.assertFalse(stackless.enable_softswitch(None), "softswitch is enabled") - return method(self) - wrapper_hardswitch.enable_softswitch = False - wrapper_hardswitch.__name__ = n + "_H" - if m.__doc__: - doc = m.__doc__ - if doc.startswith("(soft) "): - doc = doc[7:] - wrapper_hardswitch.__doc__ = "(hard) " + doc - setattr(cls, wrapper_hardswitch.__name__, wrapper_hardswitch) - - if hasattr(m, "_H_created"): - continue - m._H_created = True - m.enable_softswitch = True - if m.__doc__: - m.__doc__ = "(soft) " + m.__doc__ + m = m.__func__ + prepare = getattr(m, "prepare", cls.prepare_test_method) + for x in prepare.__func__(cls, m, n): + pass __setup_called = False __preexisting_threads = None @@ -233,7 +285,7 @@ def setUpStacklessTestCase(self): This method must be called from :meth:`setUp`. """ - self.__setup_called = True + self._StacklessTestCase__setup_called = True self.addCleanup(stackless.enable_softswitch, stackless.enable_softswitch(self.__enable_softswitch)) self.__active_test_cases[id(self)] = self @@ -278,6 +330,24 @@ def tearDown(self): self.assertEqual(active_count, expected_thread_count, "Leakage from other threads, with %d threads running (%d expected)" % (active_count, expected_thread_count)) gc.collect() # emits warnings about uncollectable objects after each test + def dumps(self, obj, protocol=None, *, fix_imports=True): + if self._pickle_module == "P": + f = io.BytesIO() + pickle._Pickler(f, protocol, fix_imports=fix_imports).dump(obj) + return f.getvalue() + elif self._pickle_module == "C": + return pickle.dumps(obj, protocol=protocol, fix_imports=fix_imports) + raise ValueError("Invalid pickle module") + + def loads(self, s, *, fix_imports=True, encoding="ASCII", errors="strict"): + if self._pickle_module == "P": + file = io.BytesIO(s) + return pickle._Unpickler(file, fix_imports=fix_imports, + encoding=encoding, errors=errors).load() + elif self._pickle_module == "C": + return pickle.loads(s, fix_imports=fix_imports, encoding=encoding, errors=errors) + raise ValueError("Invalid pickle module") + # limited pickling support for test cases # Between setUp() and tearDown() the test-case has a # working __reduce__ method. Later the test case gets pickled by @@ -331,6 +401,14 @@ def _addSkip(self, result, reason): del _tc +class StacklessPickleTestCase(StacklessTestCase): + """A test case class for pickle tests""" + + @classmethod + def prepare_test_method(cls, func, name): + return cls.prepare_pickle_test_method(func, name) + + def restore_testcase_from_id(id_): return StacklessTestCase.restore_testcase_from_id(id_) diff --git a/Stackless/unittests/test_miscell.py b/Stackless/unittests/test_miscell.py index 96bed0afcf8a21..2ab76581a05b1a 100644 --- a/Stackless/unittests/test_miscell.py +++ b/Stackless/unittests/test_miscell.py @@ -1,6 +1,5 @@ # import common -import pickle import unittest import stackless import sys @@ -82,14 +81,16 @@ def test_aliveness1(self): t = stackless.tasklet(runtask)() self.lifecycle(t) + @StacklessTestCase.prepare_pickle_test_method def test_aliveness2(self): """ Same as 1, but with a pickled unrun tasklet. """ t = stackless.tasklet(runtask)() - t_new = pickle.loads(pickle.dumps((t))) + t_new = self.loads(self.dumps((t))) t.remove() t_new.insert() self.lifecycle(t_new) + @StacklessTestCase.prepare_pickle_test_method def test_aliveness3(self): """ Same as 1, but with a pickled run(slightly) tasklet. """ @@ -114,8 +115,8 @@ def test_aliveness3(self): self.assertEqual(t.recursion_depth, softSwitching and 1 or 2) # Now save & load - dumped = pickle.dumps(t) - t_new = pickle.loads(dumped) + dumped = self.dumps(t) + t_new = self.loads(dumped) # Remove and insert & swap names around a bit t.remove() diff --git a/Stackless/unittests/test_pickle.py b/Stackless/unittests/test_pickle.py index 65179e5f99aed5..0658b29d6920f1 100644 --- a/Stackless/unittests/test_pickle.py +++ b/Stackless/unittests/test_pickle.py @@ -1,14 +1,12 @@ import sys import types import unittest -import pickle import gc -import io import inspect from stackless import schedule, tasklet, stackless -from support import StacklessTestCase +from support import StacklessTestCase, StacklessPickleTestCase # because test runner instances in the testsuite contain copies of the old stdin/stdout thingies, @@ -245,32 +243,10 @@ def is_soft(): return softswitch -class PyPickleMixin(object): - - def dumps(self, obj, protocol=None, *, fix_imports=True): - f = io.BytesIO() - pickle._Pickler(f, protocol, fix_imports=fix_imports).dump(obj) - return f.getvalue() - - def loads(self, s, *, fix_imports=True, encoding="ASCII", errors="strict"): - file = io.BytesIO(s) - return pickle._Unpickler(file, fix_imports=fix_imports, - encoding=encoding, errors=errors).load() - - -class CPickleMixin(object): - - def dumps(self, obj, protocol=None, *, fix_imports=True): - return pickle.dumps(obj, protocol=protocol, fix_imports=fix_imports) - - def loads(self, s, *, fix_imports=True, encoding="ASCII", errors="strict"): - return pickle.loads(s, fix_imports=fix_imports, encoding=encoding, errors=errors) - - -class AbstractTestPickledTasklets(StacklessTestCase): +class TestPickledTasklets(StacklessPickleTestCase): def setUp(self): - super(AbstractTestPickledTasklets, self).setUp() + super(TestPickledTasklets, self).setUp() self.verbose = VERBOSE def tearDown(self): @@ -283,7 +259,7 @@ def tearDown(self): current.kill() current = next - super(AbstractTestPickledTasklets, self).tearDown() + super(TestPickledTasklets, self).tearDown() del self.verbose @@ -330,24 +306,6 @@ def run_pickled(self, func, *args): self.assertNotEqual(new_ident, old_ident) self.assertEqual(is_empty(), True) - # compatibility to 2.2.3 - global have_enumerate - try: - enumerate - have_enumerate = True - except NameError: - have_enumerate = False - - global have_fromkeys - try: - {}.fromkeys - have_fromkeys = True - except AttributeError: - have_fromkeys = False - - -class PickledTaskletTestCases(object): - def testClassPersistence(self): t1 = CustomTasklet(nothing)() s = self.dumps(t1) @@ -387,9 +345,8 @@ def testDictIteritems(self): def testSet(self): self.run_pickled(settest, 20, 13) - if have_enumerate: - def testEnumerate(self): - self.run_pickled(enumeratetest, 20, 13) + def testEnumerate(self): + self.run_pickled(enumeratetest, 20, 13) def testTuple(self): self.run_pickled(tupletest, 20, 13) @@ -535,14 +492,6 @@ def testFunctionModulePreservation(self): self.assertEqual(f1.__module__, f2.__module__) -class TestPickledTaskletsPy(AbstractTestPickledTasklets, PickledTaskletTestCases, PyPickleMixin): - pass - - -class TestPickledTaskletsC(AbstractTestPickledTasklets, PickledTaskletTestCases, CPickleMixin): - pass - - class TestFramePickling(StacklessTestCase): def testLocalplus(self): @@ -583,7 +532,7 @@ def func(current): self.assertIs(cell.cell_contents, result) -class DictViewPicklingTestCases(object): +class TestDictViewPickling(StacklessPickleTestCase): def testDictKeyViewPickling(self): # stackless python prior to 2.7.3 used to register its own __reduce__ @@ -605,15 +554,7 @@ def testDictValueViewPickling(self): self.assertEqual(list(view1), list(view2)) -class TestDictViewPicklingPy(AbstractTestPickledTasklets, DictViewPicklingTestCases, PyPickleMixin): - pass - - -class TestDictViewPicklingC(AbstractTestPickledTasklets, DictViewPicklingTestCases, CPickleMixin): - pass - - -class Traceback_TestCases(object): +class TestTraceback(StacklessPickleTestCase): def testTracebackFrameLinkage(self): def a(): # raise an exception @@ -649,13 +590,6 @@ def c(): self.assertListEqual([i[1:] for i in all_outerframes_orig[:l - 1]], [i[1:] for i in all_outerframes[:l - 1]]) -class TestTracebackPy(StacklessTestCase, Traceback_TestCases, PyPickleMixin): - pass - - -class TestTracebackC(StacklessTestCase, Traceback_TestCases, CPickleMixin): - pass - if __name__ == '__main__': if not sys.argv[1:]: sys.argv.append('-v') diff --git a/Stackless/unittests/test_watchdog.py b/Stackless/unittests/test_watchdog.py index a1d632bfafbefc..68533579da857b 100644 --- a/Stackless/unittests/test_watchdog.py +++ b/Stackless/unittests/test_watchdog.py @@ -1,4 +1,3 @@ -import pickle import sys import random import unittest @@ -269,11 +268,12 @@ def get_pickled_tasklet(self): orig.set_ignore_nesting(1) not_finished = stackless.run(100) self.assertEqual(not_finished, orig) - return pickle.dumps(not_finished) + return self.dumps(not_finished) + @StacklessTestCase.prepare_pickle_test_method def test_pickle(self): # Run global - t = pickle.loads(self.get_pickled_tasklet()) + t = self.loads(self.get_pickled_tasklet()) t.insert() if is_soft(): stackless.run() @@ -281,7 +281,7 @@ def test_pickle(self): self.assertRaises(RuntimeError, stackless.run) # Run on tasklet - t = pickle.loads(self.get_pickled_tasklet()) + t = self.loads(self.get_pickled_tasklet()) t.insert() if is_soft(): t.run() @@ -290,17 +290,18 @@ def test_pickle(self): return # enough crap # Run on watchdog - t = pickle.loads(self.get_pickled_tasklet()) + t = self.loads(self.get_pickled_tasklet()) t.insert() while stackless.runcount > 1: returned = stackless.run(100) + @StacklessTestCase.prepare_pickle_test_method def test_run_return(self): # if the main tasklet had previously gone into C stack recusion-based switch, stackless.run() would give # strange results # this would happen after, e.g. tasklet pickling and unpickling # note, the bug was hard to repro, most of the time, it didn't occur. - t = pickle.loads(self.get_pickled_tasklet()) + t = self.loads(self.get_pickled_tasklet()) def func(): pass