From 7423828adf672ca2b3d9f743f5debd2f62c48bd7 Mon Sep 17 00:00:00 2001 From: Dieter Baron Date: Wed, 10 Apr 2024 11:14:31 +0200 Subject: [PATCH] Include file name of test case in output. This allows jumping directly to the test case in IDEs. --- nihtest/File.py | 15 +++++++-------- nihtest/Output.py | 13 +++++++++++++ nihtest/Test.py | 18 ++++++++++-------- nihtest/Utility.py | 6 +++--- tests/compare-binary-fail.test | 1 + tests/diff-1.test | 1 + tests/diff-2.test | 1 + tests/false-fail.test | 1 + tests/features-fail.test | 1 + tests/file-del-fail.test | 1 + tests/file-fail.test | 1 + tests/file-inline-fail.test | 1 + tests/file-new-fail.test | 1 + tests/precheck-fail.test | 1 + tests/stderr-fail.test | 1 + tests/stdout-fail.test | 1 + tests/true-fail.test | 1 + 17 files changed, 46 insertions(+), 19 deletions(-) create mode 100644 nihtest/Output.py diff --git a/nihtest/File.py b/nihtest/File.py index 68033b1..538da5a 100644 --- a/nihtest/File.py +++ b/nihtest/File.py @@ -23,7 +23,7 @@ def __init__(self, name, input=None, result=None): def file_name(self, directory): return self.name.replace("@SANDBOX@", os.path.abspath(directory)) - def compare(self, configuration, directory): + def compare(self, output, configuration, directory): if not self.result: return True @@ -62,8 +62,8 @@ def compare(self, configuration, directory): print(f"comparing {self.name} failed:") print("\n".join(command.stderr)) else: - print(f"{self.name} differs:") - print("\n".join(command.stdout)) + output.print(f"{self.name} differs:") + output.print("\n".join(command.stdout)) return command.exit_code == 0 output_data = self.result.data @@ -73,21 +73,20 @@ def compare(self, configuration, directory): # TODO: allow binary data command = Command.Command(configuration.find_program(preprocessor[0]), arguments, environment=Environment.Environment(configuration).environment) command.run() - return Utility.compare_lines(self.name, output_data, command.stdout, configuration.verbose != Configuration.When.NEVER) + return Utility.compare_lines(output, self.name, output_data, command.stdout) if not output_is_binary: try: input_data = Utility.read_lines(input_file_name) - return Utility.compare_lines(self.name, output_data, input_data, configuration.verbose != Configuration.When.NEVER) + return Utility.compare_lines(output, self.name, output_data, input_data) except UnicodeDecodeError: output_data = "\n".join(output_data) with open(input_file_name, "rb") as file: input_data = file.read() if input_data != output_data: - if configuration.verbose != Configuration.When.NEVER: - print(f"{self.name} differs:") - print("Binary files differ.") + output.print(f"{self.name} differs:") + output.print("Binary files differ.") return False return True diff --git a/nihtest/Output.py b/nihtest/Output.py new file mode 100644 index 0000000..5dd641e --- /dev/null +++ b/nihtest/Output.py @@ -0,0 +1,13 @@ +class Output: + def __init__(self, header, verbose): + self.header = header + self.header_printed = False + self.verbose = verbose + + def print(self, message): + if not self.verbose: + return + if not self.header_printed: + print(self.header) + self.header_printed = True + print(message) diff --git a/nihtest/Test.py b/nihtest/Test.py index 6afbbce..1ee30c8 100644 --- a/nihtest/Test.py +++ b/nihtest/Test.py @@ -9,6 +9,7 @@ from nihtest import Configuration from nihtest import Environment from nihtest import Features +from nihtest import Output from nihtest import TestCase from nihtest import Sandbox from nihtest import Utility @@ -101,20 +102,22 @@ def run(self): st = os.stat(full_file) os.chmod(full_file, st.st_mode | stat.S_IWRITE) - self.compare("exit code", [str(self.case.exit_code)], [str(command.exit_code)]) - self.compare("output", self.case.stdout, command.stdout) - self.compare("error output", self.case.stderr, self.process_stderr(command.stderr)) + output = Output.Output(self.case.file_name + ":1: test case failed", self.case.configuration.verbose != Configuration.When.NEVER) + + self.compare(output, "exit code", [str(self.case.exit_code)], [str(command.exit_code)]) + self.compare(output,"output", self.case.stdout, command.stdout) + self.compare(output, "error output", self.case.stderr, self.process_stderr(command.stderr)) files_expected = [] for file in self.case.files: if file.result: files_expected.append(file.file_name(self.sandbox.directory)) - self.compare("file list", sorted(files_expected), sorted(files_got)) + self.compare(output, "file list", sorted(files_expected), sorted(files_got)) file_content_ok = True for file in self.case.files: - if file.name in files_got and not file.compare(self.case.configuration, self.sandbox.directory): + if file.name in files_got and not file.compare(output, self.case.configuration, self.sandbox.directory): file_content_ok = False if not file_content_ok: self.failed.append("file contents") @@ -130,9 +133,8 @@ def run(self): else: return TestResult.OK - def compare(self, description, expected, got): - if not Utility.compare_lines(description, expected, got, - self.case.configuration.verbose != Configuration.When.NEVER): + def compare(self, output, description, expected, got): + if not Utility.compare_lines(output, description, expected, got): self.failed.append(description) def list_files(self): diff --git a/nihtest/Utility.py b/nihtest/Utility.py index 11a397d..3453b6d 100644 --- a/nihtest/Utility.py +++ b/nihtest/Utility.py @@ -4,14 +4,14 @@ from nihtest import CompareArrays -def compare_lines(description, expected, got, verbose): - if not verbose: +def compare_lines(output, description, expected, got): + if not output.verbose: return expected == got compare = CompareArrays.CompareArrays(expected, got) diff = compare.get_diff() if diff: - print(f"{description} differs:") + output.print(f"{description} differs:") write_lines(sys.stdout, diff) return False return True diff --git a/tests/compare-binary-fail.test b/tests/compare-binary-fail.test index 4b4deb1..7b46c54 100644 --- a/tests/compare-binary-fail.test +++ b/tests/compare-binary-fail.test @@ -6,6 +6,7 @@ file nihtest.conf nihtest-conf file binary-1 binary-1 file binary-2 binary-2 stdout +test.test:1: test case failed a differs: Binary files differ. test -- FAIL: file contents diff --git a/tests/diff-1.test b/tests/diff-1.test index b183133..effb882 100644 --- a/tests/diff-1.test +++ b/tests/diff-1.test @@ -4,6 +4,7 @@ return 1 file diff-1.test diff-1.input file nihtest.conf nihtest-conf stdout +diff-1.test:1: test case failed output differs: -E A diff --git a/tests/diff-2.test b/tests/diff-2.test index 5fa8b95..50d308d 100644 --- a/tests/diff-2.test +++ b/tests/diff-2.test @@ -4,6 +4,7 @@ return 1 file diff-2.test diff-2.input file nihtest.conf nihtest-conf stdout +diff-2.test:1: test case failed output differs: +E A diff --git a/tests/false-fail.test b/tests/false-fail.test index 02aaaea..2323278 100644 --- a/tests/false-fail.test +++ b/tests/false-fail.test @@ -4,6 +4,7 @@ return 1 file false-fail.test false-fail.input file nihtest.conf nihtest-conf stdout +false-fail.test:1: test case failed exit code differs: -0 +1 diff --git a/tests/features-fail.test b/tests/features-fail.test index 87b5245..b112a54 100644 --- a/tests/features-fail.test +++ b/tests/features-fail.test @@ -5,6 +5,7 @@ file features-fail.test features-fail.input file nihtest.conf nihtest-conf file config.h config.h stdout +features-fail.test:1: test case failed exit code differs: -0 +1 diff --git a/tests/file-del-fail.test b/tests/file-del-fail.test index 0bf7608..5be7c51 100644 --- a/tests/file-del-fail.test +++ b/tests/file-del-fail.test @@ -5,6 +5,7 @@ file file-del-fail.test file-del-fail.input file success.txt success.txt file nihtest.conf nihtest-conf stdout +file-del-fail.test:1: test case failed file list differs: +testfile file-del-fail -- FAIL: file list diff --git a/tests/file-fail.test b/tests/file-fail.test index affda0a..e17bd7a 100644 --- a/tests/file-fail.test +++ b/tests/file-fail.test @@ -4,6 +4,7 @@ return 1 file file-fail.test file-fail.input file nihtest.conf nihtest-conf stdout +file-fail.test:1: test case failed testfile differs: -This is a successful test. +This is not a successful test. diff --git a/tests/file-inline-fail.test b/tests/file-inline-fail.test index d4cda42..763f6a0 100644 --- a/tests/file-inline-fail.test +++ b/tests/file-inline-fail.test @@ -4,6 +4,7 @@ return 1 file file-inline-fail.test file-inline-fail.input file nihtest.conf nihtest-conf stdout +file-inline-fail.test:1: test case failed inline-both differs: -This is the second inline data. +This is the first inline data. diff --git a/tests/file-new-fail.test b/tests/file-new-fail.test index f66bedc..feb18d3 100644 --- a/tests/file-new-fail.test +++ b/tests/file-new-fail.test @@ -4,6 +4,7 @@ return 1 file file-new-fail.test file-new-fail.input file nihtest.conf nihtest-conf stdout +file-new-fail.test:1: test case failed file list differs: -testfile file-new-fail -- FAIL: file list diff --git a/tests/precheck-fail.test b/tests/precheck-fail.test index 3428dbe..4b72288 100644 --- a/tests/precheck-fail.test +++ b/tests/precheck-fail.test @@ -4,6 +4,7 @@ return 1 file precheck-fail.test precheck-fail.input file nihtest.conf nihtest-conf stdout +precheck-fail.test:1: test case failed exit code differs: -0 +1 diff --git a/tests/stderr-fail.test b/tests/stderr-fail.test index 2ee76c3..7a52675 100644 --- a/tests/stderr-fail.test +++ b/tests/stderr-fail.test @@ -4,6 +4,7 @@ return 1 file stderr-fail.test stderr-fail.input file nihtest.conf nihtest-conf stdout +stderr-fail.test:1: test case failed error output differs: -This is a successful test. +This is not a successful test. diff --git a/tests/stdout-fail.test b/tests/stdout-fail.test index 480cd19..69726b4 100644 --- a/tests/stdout-fail.test +++ b/tests/stdout-fail.test @@ -4,6 +4,7 @@ return 1 file stdout-fail.test stdout-fail.input file nihtest.conf nihtest-conf stdout +stdout-fail.test:1: test case failed output differs: -This is a successful test. +This is not a successful test. diff --git a/tests/true-fail.test b/tests/true-fail.test index 116b5e3..0f388bb 100644 --- a/tests/true-fail.test +++ b/tests/true-fail.test @@ -4,6 +4,7 @@ return 1 file true-fail.test true-fail.input file nihtest.conf nihtest-conf stdout +true-fail.test:1: test case failed exit code differs: -1 +0