From fd9d70a94de5b0756b52b9ae21e236e25545db4f Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 16 Aug 2023 12:20:42 +0300 Subject: [PATCH] gh-106300: Improve errors testing in test_unittest.test_runner (GH-106737) Use a custom exception to prevent unintentional silence of actual errors. --- Lib/test/test_unittest/test_runner.py | 117 ++++++++++++++------------ 1 file changed, 65 insertions(+), 52 deletions(-) diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index f3b2c0cffd4513..1b9cef43e3f9c5 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -24,6 +24,13 @@ def getRunner(): stream=io.StringIO()) +class CustomError(Exception): + pass + +# For test output compat: +CustomErrorRepr = f"{__name__ + '.' if __name__ != '__main__' else ''}CustomError" + + def runTests(*cases): suite = unittest.TestSuite() for case in cases: @@ -46,7 +53,7 @@ def cleanup(ordering, blowUp=False): ordering.append('cleanup_good') else: ordering.append('cleanup_exc') - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class TestCM: @@ -108,8 +115,8 @@ def testNothing(self): result = unittest.TestResult() outcome = test._outcome = _Outcome(result=result) - CleanUpExc = Exception('foo') - exc2 = Exception('bar') + CleanUpExc = CustomError('foo') + exc2 = CustomError('bar') def cleanup1(): raise CleanUpExc @@ -125,10 +132,10 @@ def cleanup2(): (_, msg2), (_, msg1) = result.errors self.assertIn('in cleanup1', msg1) self.assertIn('raise CleanUpExc', msg1) - self.assertIn('Exception: foo', msg1) + self.assertIn(f'{CustomErrorRepr}: foo', msg1) self.assertIn('in cleanup2', msg2) self.assertIn('raise exc2', msg2) - self.assertIn('Exception: bar', msg2) + self.assertIn(f'{CustomErrorRepr}: bar', msg2) def testCleanupInRun(self): blowUp = False @@ -139,7 +146,7 @@ def setUp(self): ordering.append('setUp') test.addCleanup(cleanup2) if blowUp: - raise Exception('foo') + raise CustomError('foo') def testNothing(self): ordering.append('test') @@ -280,7 +287,7 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() def testNothing(self): ordering.append('test') @classmethod @@ -306,7 +313,7 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() def testNothing(self): ordering.append('test') @classmethod @@ -346,7 +353,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'CleanUpExc') self.assertEqual(ordering, @@ -366,10 +373,10 @@ def testNothing(self): @classmethod def tearDownClass(cls): ordering.append('tearDownClass') - raise Exception('TearDownClassExc') + raise CustomError('TearDownClassExc') suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) @@ -379,7 +386,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) @@ -392,16 +399,22 @@ def testNothing(self): pass def cleanup1(): - raise Exception('cleanup1') + raise CustomError('cleanup1') def cleanup2(): - raise Exception('cleanup2') + raise CustomError('cleanup2') TestableTest.addClassCleanup(cleanup1) TestableTest.addClassCleanup(cleanup2) - with self.assertRaises(Exception) as e: - TestableTest.doClassCleanups() - self.assertEqual(e, 'cleanup1') + TestableTest.doClassCleanups() + + self.assertEqual(len(TestableTest.tearDown_exceptions), 2) + + e1, e2 = TestableTest.tearDown_exceptions + self.assertIsInstance(e1[1], CustomError) + self.assertEqual(str(e1[1]), 'cleanup2') + self.assertIsInstance(e2[1], CustomError) + self.assertEqual(str(e2[1]), 'cleanup1') def test_with_errors_addCleanUp(self): ordering = [] @@ -421,7 +434,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'cleanup_exc', 'tearDownClass', 'cleanup_good']) @@ -444,7 +457,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'cleanup_good', 'tearDownClass', 'cleanup_exc']) @@ -460,11 +473,11 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering, blowUp=True) if class_blow_up: - raise Exception('ClassExc') + raise CustomError('ClassExc') def setUp(self): ordering.append('setUp') if method_blow_up: - raise Exception('MethodExc') + raise CustomError('MethodExc') def testNothing(self): ordering.append('test') @classmethod @@ -473,7 +486,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'tearDownClass', 'cleanup_exc']) @@ -483,9 +496,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ClassExc') + f'{CustomErrorRepr}: ClassExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'cleanup_exc']) @@ -494,9 +507,9 @@ def tearDownClass(cls): method_blow_up = True result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: MethodExc') + f'{CustomErrorRepr}: MethodExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'tearDownClass', 'cleanup_exc']) @@ -513,11 +526,11 @@ def testNothing(self): @classmethod def tearDownClass(cls): ordering.append('tearDownClass') - raise Exception('TearDownExc') + raise CustomError('TearDownExc') result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: TearDownExc') + f'{CustomErrorRepr}: TearDownExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass', 'cleanup_good']) @@ -620,7 +633,7 @@ def module_cleanup_good(*args, **kwargs): module_cleanups.append((3, args, kwargs)) def module_cleanup_bad(*args, **kwargs): - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class Module(object): unittest.addModuleCleanup(module_cleanup_good, 1, 2, 3, @@ -630,7 +643,7 @@ class Module(object): [(module_cleanup_good, (1, 2, 3), dict(four='hello', five='goodbye')), (module_cleanup_bad, (), {})]) - with self.assertRaises(Exception) as e: + with self.assertRaises(CustomError) as e: unittest.case.doModuleCleanups() self.assertEqual(str(e.exception), 'CleanUpExc') self.assertEqual(unittest.case._module_cleanups, []) @@ -659,7 +672,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) if blowUp: - raise Exception('setUpModule Exc') + raise CustomError('setUpModule Exc') @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -679,7 +692,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(ordering, ['setUpModule', 'cleanup_good']) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: setUpModule Exc') + f'{CustomErrorRepr}: setUpModule Exc') ordering = [] blowUp = False @@ -699,7 +712,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -710,7 +723,7 @@ def setUpModule(): ordering.append('setUpModule2') unittest.addModuleCleanup(cleanup, ordering) if blowUp2: - raise Exception() + raise CustomError() @staticmethod def tearDownModule(): ordering.append('tearDownModule2') @@ -799,7 +812,7 @@ def setUpModule(): @staticmethod def tearDownModule(): ordering.append('tearDownModule') - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class TestableTest(unittest.TestCase): @classmethod @@ -815,7 +828,7 @@ def tearDownClass(cls): sys.modules['Module'] = Module result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'tearDownModule', 'cleanup_good']) @@ -855,7 +868,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -873,7 +886,7 @@ def setUpModule(): @staticmethod def tearDownModule(): ordering.append('tearDownModule') - raise Exception('TearDownModuleExc') + raise CustomError('TearDownModuleExc') class TestableTest(unittest.TestCase): @classmethod @@ -888,7 +901,7 @@ def tearDownClass(cls): TestableTest.__module__ = 'Module' sys.modules['Module'] = Module suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -899,7 +912,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -978,7 +991,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) @@ -1008,7 +1021,7 @@ def tearDown(self): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUp', 'test', 'tearDown', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) @@ -1024,7 +1037,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering, blowUp=True) if module_blow_up: - raise Exception('ModuleExc') + raise CustomError('ModuleExc') @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -1034,11 +1047,11 @@ class TestableTest(unittest.TestCase): def setUpClass(cls): ordering.append('setUpClass') if class_blow_up: - raise Exception('ClassExc') + raise CustomError('ClassExc') def setUp(self): ordering.append('setUp') if method_blow_up: - raise Exception('MethodExc') + raise CustomError('MethodExc') def testNothing(self): ordering.append('test') @classmethod @@ -1050,7 +1063,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp', 'test', 'tearDownClass', 'tearDownModule', @@ -1062,9 +1075,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ModuleExc') + f'{CustomErrorRepr}: ModuleExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'cleanup_exc']) ordering = [] @@ -1073,9 +1086,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ClassExc') + f'{CustomErrorRepr}: ClassExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'tearDownModule', 'cleanup_exc']) @@ -1085,9 +1098,9 @@ def tearDownClass(cls): method_blow_up = True result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: MethodExc') + f'{CustomErrorRepr}: MethodExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp', 'tearDownClass', 'tearDownModule', 'cleanup_exc'])