Issue python#114: Test suite: test all pickle protocols

Enhance the test suite to test all pickle protocols using both - the C and the Python - pickle implementations.

https://bitbucket.org/stackless-dev/stackless/issues/114
(grafted from 5da071b503d30801d5eaf3f44b5d3854f9b2c176)

Stackless/unittests/support.py

@@ -29,14 +29,14 @@
 import weakref
 import pickle
 import inspect
-from io import StringIO
+import io
 import contextlib
 import gc
 
 # emit warnings about uncollectable objects
 gc.set_debug(gc.DEBUG_UNCOLLECTABLE)
 try:
-    long
+    long  # @UndefinedVariable
 except NameError:
     long = int  # @ReservedAssignment
 
@@ -50,7 +50,7 @@
 @contextlib.contextmanager
 def captured_stderr():
     old = sys.stderr
-    new = StringIO()
+    new = io.StringIO()
     sys.stderr = new
     try:
         yield new
@@ -172,6 +172,77 @@ def __init__(cls, *args, **kw):
 
 class StacklessTestCase(unittest.TestCase, StacklessTestCaseMixin, metaclass=StacklessTestCaseMeta):
 
+    @classmethod
+    def prepare_test_method(cls, func, name):
+        """Called after class creation
+
+        This method creates the _H methods, which run without
+        soft switching
+        """
+        if hasattr(func, "enable_softswitch") and not getattr(func, "_H_created", False):
+            return ((func, name), )
+
+        def wrapper_hardswitch(self, method=func):
+            self.assertTrue(self.__setup_called, "Broken test case: it didn't call super(..., self).setUp()")
+            self.assertFalse(stackless.enable_softswitch(None), "softswitch is enabled")
+            return method(self)
+        wrapper_hardswitch.enable_softswitch = False
+        wrapper_hardswitch.__name__ = name + "_H"
+        if func.__doc__:
+            doc = func.__doc__
+            if doc.startswith("(soft) "):
+                doc = doc[7:]
+            wrapper_hardswitch.__doc__ = "(hard) " + doc
+        setattr(cls, wrapper_hardswitch.__name__, wrapper_hardswitch)
+
+        if not hasattr(func, "_H_created"):
+            func._H_created = True
+            func.enable_softswitch = True
+            if func.__doc__:
+                func.__doc__ = "(soft) " + func.__doc__
+        return ((func, name), (wrapper_hardswitch, wrapper_hardswitch.__name__))
+
+    @classmethod
+    def prepare_pickle_test_method(cls, func, name=None):
+        """Called after class creation
+
+        This method creates the Py0...n C0...n methods, which run with
+        the Python or C implementation of the enumerated pickle protocol.
+
+        This method also acts as a method decorator.
+        """
+        if name is None:
+            # used as a decorator
+            func.prepare = cls.prepare_pickle_test_method
+            return func
+
+        if hasattr(func, "_pickle_created"):
+            return StacklessTestCase.prepare_test_method.__func__(cls, func, name)
+        setattr(cls, name, None)
+        r = []
+        for i in range(0, pickle.HIGHEST_PROTOCOL + 1):
+            for p_letter in ("C", "P"):
+                def test(self, method=func, proto=i, pickle_module=p_letter, unpickle_module=p_letter):
+                    self.assertTrue(self._StacklessTestCase__setup_called, "Broken test case: it didn't call super(..., self).setUp()")
+                    self._pickle_protocol = proto
+                    self._pickle_module = pickle_module
+                    self._unpickle_module = unpickle_module
+                    return method(self)
+                if i == 0 and p_letter == "C":
+                    test.__name__ = name
+                else:
+                    test.__name__ = "{:s}_{:s}{:d}".format(name, p_letter, i)
+                test._pickle_created = True
+                if func.__doc__:
+                    doc = func.__doc__
+                    match = re.match(r"\([PC][0-{:d}]\)".format(pickle.HIGHEST_PROTOCOL), doc)
+                    if match:
+                        doc = match.string[match.end():]
+                    test.__doc__ = "({:s}{:d}) {:s}".format(p_letter, i, doc)
+                setattr(cls, test.__name__, test)
+                r.extend(StacklessTestCase.prepare_test_method.__func__(cls, test, test.__name__))
+        return r
+
     @classmethod
     def prepare_test_methods(cls):
         """Called after class creation
@@ -183,29 +254,10 @@ def prepare_test_methods(cls):
         for n in names:
             m = getattr(cls, n)
             if inspect.ismethod(m):
-                m = m.im_func
-            if hasattr(m, "enable_softswitch") and not getattr(m, "_H_created", False):
-                continue
-
-            def wrapper_hardswitch(self, method=m):
-                self.assertTrue(self.__setup_called, "Broken test case: it didn't call super(..., self).setUp()")
-                self.assertFalse(stackless.enable_softswitch(None), "softswitch is enabled")
-                return method(self)
-            wrapper_hardswitch.enable_softswitch = False
-            wrapper_hardswitch.__name__ = n + "_H"
-            if m.__doc__:
-                doc = m.__doc__
-                if doc.startswith("(soft) "):
-                    doc = doc[7:]
-                wrapper_hardswitch.__doc__ = "(hard) " + doc
-            setattr(cls, wrapper_hardswitch.__name__, wrapper_hardswitch)
-
-            if hasattr(m, "_H_created"):
-                continue
-            m._H_created = True
-            m.enable_softswitch = True
-            if m.__doc__:
-                m.__doc__ = "(soft) " + m.__doc__
+                m = m.__func__
+            prepare = getattr(m, "prepare", cls.prepare_test_method)
+            for x in prepare.__func__(cls, m, n):
+                pass
 
     __setup_called = False
     __preexisting_threads = None
@@ -233,7 +285,7 @@ def setUpStacklessTestCase(self):
 
         This method must be called from :meth:`setUp`.
         """
-        self.__setup_called = True
+        self._StacklessTestCase__setup_called = True
         self.addCleanup(stackless.enable_softswitch, stackless.enable_softswitch(self.__enable_softswitch))
 
         self.__active_test_cases[id(self)] = self
@@ -278,6 +330,24 @@ def tearDown(self):
             self.assertEqual(active_count, expected_thread_count, "Leakage from other threads, with %d threads running (%d expected)" % (active_count, expected_thread_count))
         gc.collect()  # emits warnings about uncollectable objects after each test
 
+    def dumps(self, obj, protocol=None, *, fix_imports=True):
+        if self._pickle_module == "P":
+            f = io.BytesIO()
+            pickle._Pickler(f, protocol, fix_imports=fix_imports).dump(obj)
+            return f.getvalue()
+        elif self._pickle_module == "C":
+            return pickle.dumps(obj, protocol=protocol, fix_imports=fix_imports)
+        raise ValueError("Invalid pickle module")
+
+    def loads(self, s, *, fix_imports=True, encoding="ASCII", errors="strict"):
+        if self._pickle_module == "P":
+            file = io.BytesIO(s)
+            return pickle._Unpickler(file, fix_imports=fix_imports,
+                                     encoding=encoding, errors=errors).load()
+        elif self._pickle_module == "C":
+            return pickle.loads(s, fix_imports=fix_imports, encoding=encoding, errors=errors)
+        raise ValueError("Invalid pickle module")
+
     # limited pickling support for test cases
     # Between setUp() and tearDown() the test-case has a
     # working __reduce__ method. Later the test case gets pickled by
@@ -331,6 +401,14 @@ def _addSkip(self, result, reason):
     del _tc
 
 
+class StacklessPickleTestCase(StacklessTestCase):
+    """A test case class for pickle tests"""
+
+    @classmethod
+    def prepare_test_method(cls, func, name):
+        return cls.prepare_pickle_test_method(func, name)
+
+
 def restore_testcase_from_id(id_):
     return StacklessTestCase.restore_testcase_from_id(id_)
 

Stackless/unittests/test_miscell.py

@@ -1,6 +1,5 @@
 # import common
 
-import pickle
 import unittest
 import stackless
 import sys
@@ -82,14 +81,16 @@ def test_aliveness1(self):
         t = stackless.tasklet(runtask)()
         self.lifecycle(t)
 
+    @StacklessTestCase.prepare_pickle_test_method
     def test_aliveness2(self):
         """ Same as 1, but with a pickled unrun tasklet. """
         t = stackless.tasklet(runtask)()
-        t_new = pickle.loads(pickle.dumps((t)))
+        t_new = self.loads(self.dumps((t)))
         t.remove()
         t_new.insert()
         self.lifecycle(t_new)
 
+    @StacklessTestCase.prepare_pickle_test_method
     def test_aliveness3(self):
         """ Same as 1, but with a pickled run(slightly) tasklet. """
 
@@ -114,8 +115,8 @@ def test_aliveness3(self):
         self.assertEqual(t.recursion_depth, softSwitching and 1 or 2)
 
         # Now save & load
-        dumped = pickle.dumps(t)
-        t_new = pickle.loads(dumped)
+        dumped = self.dumps(t)
+        t_new = self.loads(dumped)
 
         # Remove and insert & swap names around a bit
         t.remove()

Stackless/unittests/test_pickle.py

@@ -1,14 +1,12 @@
 import sys
 import types
 import unittest
-import pickle
 import gc
-import io
 import inspect
 
 from stackless import schedule, tasklet, stackless
 
-from support import StacklessTestCase
+from support import StacklessTestCase, StacklessPickleTestCase
 
 
 # because test runner instances in the testsuite contain copies of the old stdin/stdout thingies,
@@ -245,32 +243,10 @@ def is_soft():
     return softswitch
 
 
-class PyPickleMixin(object):
-
-    def dumps(self, obj, protocol=None, *, fix_imports=True):
-        f = io.BytesIO()
-        pickle._Pickler(f, protocol, fix_imports=fix_imports).dump(obj)
-        return f.getvalue()
-
-    def loads(self, s, *, fix_imports=True, encoding="ASCII", errors="strict"):
-        file = io.BytesIO(s)
-        return pickle._Unpickler(file, fix_imports=fix_imports,
-                                 encoding=encoding, errors=errors).load()
-
-
-class CPickleMixin(object):
-
-    def dumps(self, obj, protocol=None, *, fix_imports=True):
-        return pickle.dumps(obj, protocol=protocol, fix_imports=fix_imports)
-
-    def loads(self, s, *, fix_imports=True, encoding="ASCII", errors="strict"):
-        return pickle.loads(s, fix_imports=fix_imports, encoding=encoding, errors=errors)
-
-
-class AbstractTestPickledTasklets(StacklessTestCase):
+class TestPickledTasklets(StacklessPickleTestCase):
 
     def setUp(self):
-        super(AbstractTestPickledTasklets, self).setUp()
+        super(TestPickledTasklets, self).setUp()
         self.verbose = VERBOSE
 
     def tearDown(self):
@@ -283,7 +259,7 @@ def tearDown(self):
             current.kill()
             current = next
 
-        super(AbstractTestPickledTasklets, self).tearDown()
+        super(TestPickledTasklets, self).tearDown()
 
         del self.verbose
 
@@ -330,24 +306,6 @@ def run_pickled(self, func, *args):
         self.assertNotEqual(new_ident, old_ident)
         self.assertEqual(is_empty(), True)
 
-    # compatibility to 2.2.3
-    global have_enumerate
-    try:
-        enumerate
-        have_enumerate = True
-    except NameError:
-        have_enumerate = False
-
-    global have_fromkeys
-    try:
-        {}.fromkeys
-        have_fromkeys = True
-    except AttributeError:
-        have_fromkeys = False
-
-
-class PickledTaskletTestCases(object):
-
     def testClassPersistence(self):
         t1 = CustomTasklet(nothing)()
         s = self.dumps(t1)
@@ -387,9 +345,8 @@ def testDictIteritems(self):
     def testSet(self):
         self.run_pickled(settest, 20, 13)
 
-    if have_enumerate:
-        def testEnumerate(self):
-            self.run_pickled(enumeratetest, 20, 13)
+    def testEnumerate(self):
+        self.run_pickled(enumeratetest, 20, 13)
 
     def testTuple(self):
         self.run_pickled(tupletest, 20, 13)
@@ -535,14 +492,6 @@ def testFunctionModulePreservation(self):
         self.assertEqual(f1.__module__, f2.__module__)
 
 
-class TestPickledTaskletsPy(AbstractTestPickledTasklets, PickledTaskletTestCases, PyPickleMixin):
-    pass
-
-
-class TestPickledTaskletsC(AbstractTestPickledTasklets, PickledTaskletTestCases, CPickleMixin):
-    pass
-
-
 class TestFramePickling(StacklessTestCase):
 
     def testLocalplus(self):
@@ -583,7 +532,7 @@ def func(current):
             self.assertIs(cell.cell_contents, result)
 
 
-class DictViewPicklingTestCases(object):
+class TestDictViewPickling(StacklessPickleTestCase):
 
     def testDictKeyViewPickling(self):
         # stackless python prior to 2.7.3 used to register its own __reduce__
@@ -605,15 +554,7 @@ def testDictValueViewPickling(self):
         self.assertEqual(list(view1), list(view2))
 
 
-class TestDictViewPicklingPy(AbstractTestPickledTasklets, DictViewPicklingTestCases, PyPickleMixin):
-    pass
-
-
-class TestDictViewPicklingC(AbstractTestPickledTasklets, DictViewPicklingTestCases, CPickleMixin):
-    pass
-
-
-class Traceback_TestCases(object):
+class TestTraceback(StacklessPickleTestCase):
     def testTracebackFrameLinkage(self):
         def a():
             # raise an exception
@@ -649,13 +590,6 @@ def c():
         self.assertListEqual([i[1:] for i in all_outerframes_orig[:l - 1]], [i[1:] for i in all_outerframes[:l - 1]])
 
 
-class TestTracebackPy(StacklessTestCase, Traceback_TestCases, PyPickleMixin):
-    pass
-
-
-class TestTracebackC(StacklessTestCase, Traceback_TestCases, CPickleMixin):
-    pass
-
 if __name__ == '__main__':
     if not sys.argv[1:]:
         sys.argv.append('-v')