diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py index 5ccae4be09679..c08c51b7b7a23 100644 --- a/llvm/utils/lit/lit/cl_arguments.py +++ b/llvm/utils/lit/lit/cl_arguments.py @@ -175,6 +175,15 @@ def parse_args(): type=lit.reports.TimeTraceReport, help="Write Chrome tracing compatible JSON to the specified file", ) + execution_group.add_argument( + "--use-unique-output-file-name", + help="When enabled, lit will not overwrite existing test report files. " + "Instead it will write to a new file named the same as the output file " + "name but with an extra part before the file extension. For example " + "if results.xml already exists, results..xml will be written " + "to. The is not ordered in any way. [Default: Off]", + action="store_true", + ) execution_group.add_argument( "--timeout", dest="maxIndividualTestTime", @@ -332,16 +341,21 @@ def parse_args(): else: opts.shard = None - opts.reports = filter( - None, - [ - opts.output, - opts.xunit_xml_output, - opts.resultdb_output, - opts.time_trace_output, - ], + opts.reports = list( + filter( + None, + [ + opts.output, + opts.xunit_xml_output, + opts.resultdb_output, + opts.time_trace_output, + ], + ) ) + for report in opts.reports: + report.use_unique_output_file_name = opts.use_unique_output_file_name + return opts diff --git a/llvm/utils/lit/lit/reports.py b/llvm/utils/lit/lit/reports.py index 8ec83d698ae86..8312dcddc769a 100755 --- a/llvm/utils/lit/lit/reports.py +++ b/llvm/utils/lit/lit/reports.py @@ -1,7 +1,10 @@ +import abc import base64 import datetime import itertools import json +import os +import tempfile from xml.sax.saxutils import quoteattr as quo @@ -14,11 +17,34 @@ def by_suite_and_test_path(test): return (test.suite.name, id(test.suite), test.path_in_suite) -class JsonReport(object): +class Report(object): def __init__(self, output_file): self.output_file = output_file + # Set by the option parser later. + self.use_unique_output_file_name = False def write_results(self, tests, elapsed): + if self.use_unique_output_file_name: + filename, ext = os.path.splitext(os.path.basename(self.output_file)) + fd, _ = tempfile.mkstemp( + suffix=ext, prefix=f"{filename}.", dir=os.path.dirname(self.output_file) + ) + report_file = os.fdopen(fd, "w") + else: + # Overwrite if the results already exist. + report_file = open(self.output_file, "w") + + with report_file: + self._write_results_to_file(tests, elapsed, report_file) + + @abc.abstractmethod + def _write_results_to_file(self, tests, elapsed, file): + """Write test results to the file object "file".""" + pass + + +class JsonReport(Report): + def _write_results_to_file(self, tests, elapsed, file): unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED} tests = [t for t in tests if t.result.code not in unexecuted_codes] # Construct the data we will write. @@ -67,9 +93,8 @@ def write_results(self, tests, elapsed): tests_data.append(test_data) - with open(self.output_file, "w") as file: - json.dump(data, file, indent=2, sort_keys=True) - file.write("\n") + json.dump(data, file, indent=2, sort_keys=True) + file.write("\n") _invalid_xml_chars_dict = { @@ -88,21 +113,18 @@ def remove_invalid_xml_chars(s): return s.translate(_invalid_xml_chars_dict) -class XunitReport(object): - def __init__(self, output_file): - self.output_file = output_file - self.skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED} +class XunitReport(Report): + skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED} - def write_results(self, tests, elapsed): + def _write_results_to_file(self, tests, elapsed, file): tests.sort(key=by_suite_and_test_path) tests_by_suite = itertools.groupby(tests, lambda t: t.suite) - with open(self.output_file, "w") as file: - file.write('\n') - file.write('\n'.format(time=elapsed)) - for suite, test_iter in tests_by_suite: - self._write_testsuite(file, suite, list(test_iter)) - file.write("\n") + file.write('\n') + file.write('\n'.format(time=elapsed)) + for suite, test_iter in tests_by_suite: + self._write_testsuite(file, suite, list(test_iter)) + file.write("\n") def _write_testsuite(self, file, suite, tests): skipped = 0 @@ -206,11 +228,8 @@ def gen_resultdb_test_entry( return test_data -class ResultDBReport(object): - def __init__(self, output_file): - self.output_file = output_file - - def write_results(self, tests, elapsed): +class ResultDBReport(Report): + def _write_results_to_file(self, tests, elapsed, file): unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED} tests = [t for t in tests if t.result.code not in unexecuted_codes] data = {} @@ -249,17 +268,14 @@ def write_results(self, tests, elapsed): ) ) - with open(self.output_file, "w") as file: - json.dump(data, file, indent=2, sort_keys=True) - file.write("\n") + json.dump(data, file, indent=2, sort_keys=True) + file.write("\n") -class TimeTraceReport(object): - def __init__(self, output_file): - self.output_file = output_file - self.skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED} +class TimeTraceReport(Report): + skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED} - def write_results(self, tests, elapsed): + def _write_results_to_file(self, tests, elapsed, file): # Find when first test started so we can make start times relative. first_start_time = min([t.result.start for t in tests]) events = [ @@ -270,8 +286,7 @@ def write_results(self, tests, elapsed): json_data = {"traceEvents": events} - with open(self.output_file, "w") as time_trace_file: - json.dump(json_data, time_trace_file, indent=2, sort_keys=True) + json.dump(json_data, time_trace_file, indent=2, sort_keys=True) def _get_test_event(self, test, first_start_time): test_name = test.getFullName() diff --git a/llvm/utils/lit/tests/unique-output-file.py b/llvm/utils/lit/tests/unique-output-file.py new file mode 100644 index 0000000000000..fea57682d9fda --- /dev/null +++ b/llvm/utils/lit/tests/unique-output-file.py @@ -0,0 +1,22 @@ +## Check that lit will not overwrite existing result files when given +## --use-unique-output-file-name. + +## Files are overwritten without the option. +# RUN: rm -f %t.xunit*.xml +# RUN: echo "test" > %t.xunit.xml +# RUN: not %{lit} --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output +# RUN: FileCheck < %t.xunit.xml %s --check-prefix=NEW +# NEW: +# NEW-NEXT: +## (other tests will check the contents of the whole file) + +# RUN: rm -f %t.xunit*.xml +# RUN: echo "test" > %t.xunit.xml +## Files should not be overwritten with the option. +# RUN: not %{lit} --xunit-xml-output %t.xunit.xml --use-unique-output-file-name %{inputs}/xunit-output +# RUN: FileCheck < %t.xunit.xml %s --check-prefix=EXISTING +# EXISTING: test +## Results in a new file with some discriminator added. +# RUN: ls -l %t.xunit*.xml | wc -l | FileCheck %s --check-prefix=NUMFILES +# NUMFILES: 2 +# RUN: FileCheck < %t.xunit.*.xml %s --check-prefix=NEW