Skip to content

Commit

Permalink
gh-98903: Test suite fails with exit code 4 if no tests ran (#98904)
Browse files Browse the repository at this point in the history 
The Python test suite now fails wit exit code 4 if no tests ran. It
should help detecting typos in test names and test methods.

* Add "EXITCODE_" constants to Lib/test/libregrtest/main.py.
* Fix a typo: "NO TEST RUN" becomes "NO TESTS RAN"
  • Loading branch information
@vstinner
vstinner authored Nov 2, 2022
1 parent 0689b99 commit c76db37
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 26 deletions.
22 changes: 16 additions & 6 deletions Lib/test/libregrtest/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@
# Must be smaller than buildbot "1200 seconds without output" limit.
EXIT_TIMEOUT = 120.0

EXITCODE_BAD_TEST = 2
EXITCODE_INTERRUPTED = 130
EXITCODE_ENV_CHANGED = 3
EXITCODE_NO_TESTS_RAN = 4


class Regrtest:
"""Execute a test suite.
Expand Down Expand Up @@ -493,15 +498,18 @@ def display_header(self):
print("== encodings: locale=%s, FS=%s"
% (locale.getencoding(), sys.getfilesystemencoding()))

def no_tests_run(self):
return not any((self.good, self.bad, self.skipped, self.interrupted,
self.environment_changed))

def get_tests_result(self):
result = []
if self.bad:
result.append("FAILURE")
elif self.ns.fail_env_changed and self.environment_changed:
result.append("ENV CHANGED")
elif not any((self.good, self.bad, self.skipped, self.interrupted,
self.environment_changed)):
result.append("NO TEST RUN")
elif self.no_tests_run():
result.append("NO TESTS RAN")

if self.interrupted:
result.append("INTERRUPTED")
Expand Down Expand Up @@ -750,11 +758,13 @@ def _main(self, tests, kwargs):
self.save_xml_result()

if self.bad:
sys.exit(2)
sys.exit(EXITCODE_BAD_TEST)
if self.interrupted:
sys.exit(130)
sys.exit(EXITCODE_INTERRUPTED)
if self.ns.fail_env_changed and self.environment_changed:
sys.exit(3)
sys.exit(EXITCODE_ENV_CHANGED)
if self.no_tests_run():
sys.exit(EXITCODE_NO_TESTS_RAN)
sys.exit(0)


Expand Down
54 changes: 34 additions & 20 deletions Lib/test/test_regrtest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@
ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?'

EXITCODE_BAD_TEST = 2
EXITCODE_ENV_CHANGED = 3
EXITCODE_NO_TESTS_RAN = 4
EXITCODE_INTERRUPTED = 130

TEST_INTERRUPTED = textwrap.dedent("""
from signal import SIGINT, raise_signal
try:
Expand Down Expand Up @@ -497,7 +502,7 @@ def list_regex(line_format, tests):
result.append('INTERRUPTED')
if not any((good, result, failed, interrupted, skipped,
env_changed, fail_env_changed)):
result.append("NO TEST RUN")
result.append("NO TESTS RAN")
elif not result:
result.append('SUCCESS')
result = ', '.join(result)
Expand Down Expand Up @@ -707,7 +712,7 @@ def test_failing(self):
test_failing = self.create_test('failing', code=code)
tests = [test_ok, test_failing]

output = self.run_tests(*tests, exitcode=2)
output = self.run_tests(*tests, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, tests, failed=test_failing)

def test_resources(self):
Expand Down Expand Up @@ -748,13 +753,14 @@ def test_random(self):
test = self.create_test('random', code)

# first run to get the output with the random seed
output = self.run_tests('-r', test)
output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN)
randseed = self.parse_random_seed(output)
match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
test_random = int(match.group(1))

# try to reproduce with the random seed
output = self.run_tests('-r', '--randseed=%s' % randseed, test)
output = self.run_tests('-r', '--randseed=%s' % randseed, test,
exitcode=EXITCODE_NO_TESTS_RAN)
randseed2 = self.parse_random_seed(output)
self.assertEqual(randseed2, randseed)

Expand Down Expand Up @@ -813,7 +819,7 @@ def test_fromfile(self):
def test_interrupted(self):
code = TEST_INTERRUPTED
test = self.create_test('sigint', code=code)
output = self.run_tests(test, exitcode=130)
output = self.run_tests(test, exitcode=EXITCODE_INTERRUPTED)
self.check_executed_tests(output, test, omitted=test,
interrupted=True)

Expand All @@ -838,7 +844,7 @@ def test_slowest_interrupted(self):
args = ("--slowest", "-j2", test)
else:
args = ("--slowest", test)
output = self.run_tests(*args, exitcode=130)
output = self.run_tests(*args, exitcode=EXITCODE_INTERRUPTED)
self.check_executed_tests(output, test,
omitted=test, interrupted=True)

Expand Down Expand Up @@ -878,7 +884,7 @@ def test_run(self):
builtins.__dict__['RUN'] = 1
""")
test = self.create_test('forever', code=code)
output = self.run_tests('--forever', test, exitcode=2)
output = self.run_tests('--forever', test, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, [test]*3, failed=test)

def check_leak(self, code, what):
Expand All @@ -887,7 +893,7 @@ def check_leak(self, code, what):
filename = 'reflog.txt'
self.addCleanup(os_helper.unlink, filename)
output = self.run_tests('--huntrleaks', '3:3:', test,
exitcode=2,
exitcode=EXITCODE_BAD_TEST,
stderr=subprocess.STDOUT)
self.check_executed_tests(output, [test], failed=test)

Expand Down Expand Up @@ -969,7 +975,7 @@ def test_crashed(self):
crash_test = self.create_test(name="crash", code=code)

tests = [crash_test]
output = self.run_tests("-j2", *tests, exitcode=2)
output = self.run_tests("-j2", *tests, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, tests, failed=crash_test,
randomize=True)

Expand Down Expand Up @@ -1069,7 +1075,8 @@ def test_env_changed(self):
self.check_executed_tests(output, [testname], env_changed=testname)

# fail with --fail-env-changed
output = self.run_tests("--fail-env-changed", testname, exitcode=3)
output = self.run_tests("--fail-env-changed", testname,
exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname], env_changed=testname,
fail_env_changed=True)

Expand All @@ -1088,7 +1095,7 @@ def test_fail_always(self):
""")
testname = self.create_test(code=code)

output = self.run_tests("-w", testname, exitcode=2)
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, [testname],
failed=testname, rerun={testname: "test_fail_always"})

Expand Down Expand Up @@ -1123,7 +1130,8 @@ def test_bug(self):
""")
testname = self.create_test(code=code)

output = self.run_tests(testname, "-m", "nosuchtest", exitcode=0)
output = self.run_tests(testname, "-m", "nosuchtest",
exitcode=EXITCODE_NO_TESTS_RAN)
self.check_executed_tests(output, [testname], no_test_ran=testname)

def test_no_tests_ran_skip(self):
Expand All @@ -1136,7 +1144,7 @@ def test_skipped(self):
""")
testname = self.create_test(code=code)

output = self.run_tests(testname, exitcode=0)
output = self.run_tests(testname)
self.check_executed_tests(output, [testname])

def test_no_tests_ran_multiple_tests_nonexistent(self):
Expand All @@ -1150,7 +1158,8 @@ def test_bug(self):
testname = self.create_test(code=code)
testname2 = self.create_test(code=code)

output = self.run_tests(testname, testname2, "-m", "nosuchtest", exitcode=0)
output = self.run_tests(testname, testname2, "-m", "nosuchtest",
exitcode=EXITCODE_NO_TESTS_RAN)
self.check_executed_tests(output, [testname, testname2],
no_test_ran=[testname, testname2])

Expand Down Expand Up @@ -1198,7 +1207,8 @@ def test_garbage(self):
""")
testname = self.create_test(code=code)

output = self.run_tests("--fail-env-changed", testname, exitcode=3)
output = self.run_tests("--fail-env-changed", testname,
exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname],
env_changed=[testname],
fail_env_changed=True)
Expand All @@ -1224,7 +1234,8 @@ def test_sleep(self):
""")
testname = self.create_test(code=code)

output = self.run_tests("-j2", "--timeout=1.0", testname, exitcode=2)
output = self.run_tests("-j2", "--timeout=1.0", testname,
exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, [testname],
failed=testname)
self.assertRegex(output,
Expand Down Expand Up @@ -1256,7 +1267,8 @@ def test_unraisable_exc(self):
""")
testname = self.create_test(code=code)

output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3)
output = self.run_tests("--fail-env-changed", "-v", testname,
exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname],
env_changed=[testname],
fail_env_changed=True)
Expand Down Expand Up @@ -1287,7 +1299,8 @@ def test_threading_excepthook(self):
""")
testname = self.create_test(code=code)

output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3)
output = self.run_tests("--fail-env-changed", "-v", testname,
exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname],
env_changed=[testname],
fail_env_changed=True)
Expand Down Expand Up @@ -1328,7 +1341,7 @@ def test_print_warning(self):
for option in ("-v", "-W"):
with self.subTest(option=option):
cmd = ["--fail-env-changed", option, testname]
output = self.run_tests(*cmd, exitcode=3)
output = self.run_tests(*cmd, exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, [testname],
env_changed=[testname],
fail_env_changed=True)
Expand Down Expand Up @@ -1373,7 +1386,8 @@ def test_leak_tmp_file(self):
""")
testnames = [self.create_test(code=code) for _ in range(3)]

output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames, exitcode=3)
output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames,
exitcode=EXITCODE_ENV_CHANGED)
self.check_executed_tests(output, testnames,
env_changed=testnames,
fail_env_changed=True,
Expand Down
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Tests/2022-10-31-14-47-49.gh-issue-98903.7KinCV.rst
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
The Python test suite now fails wit exit code 4 if no tests ran. It should
help detecting typos in test names and test methods.

0 comments on commit c76db37

Please sign in to comment.