diff --git a/ducktape/mark/__init__.py b/ducktape/mark/__init__.py index f38c58211..d26b5f65b 100644 --- a/ducktape/mark/__init__.py +++ b/ducktape/mark/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ._mark import parametrize, matrix, defaults, ignore, ok_to_fail, parametrized, ignored, oked_to_fail, env, is_env # NOQA +from ._mark import parametrize, matrix, defaults, ignore, ok_to_fail, parametrized, ignored, oked_to_fail, env, is_env, ok_to_fail_fips, oked_to_fail_fips # NOQA diff --git a/ducktape/mark/_mark.py b/ducktape/mark/_mark.py index 634446e53..717564353 100644 --- a/ducktape/mark/_mark.py +++ b/ducktape/mark/_mark.py @@ -125,6 +125,24 @@ def apply(self, seed_context, context_list): return context_list +class OkToFailFIPS(Mark): + """Run the test but categorize status as OPASSFIPS or OFAILFIPS instead of PASS or FAIL.""" + + def __init__(self): + super(OkToFailFIPS, self).__init__() + self.injected_args = None + + @property + def name(self): + return "OK_TO_FAIL_FIPS" + + def apply(self, seed_context, context_list): + assert len(context_list) > 0, "ignore annotation is not being applied to any test cases" + for ctx in context_list: + ctx.ok_to_fail_fips = ctx.ok_to_fail_fips or self.injected_args is None + return context_list + + class Matrix(Mark): """Parametrize with a matrix of arguments. Assume each values in self.injected_args is iterable @@ -242,6 +260,7 @@ def __eq__(self, other): DEFAULTS = Defaults() IGNORE = Ignore() OK_TO_FAIL = OkToFail() +OK_TO_FAIL_FIPS = OkToFailFIPS() ENV = Env() @@ -264,6 +283,11 @@ def oked_to_fail(f): return Mark.marked(f, OK_TO_FAIL) +def oked_to_fail_fips(f): + """Is this function or object decorated with @ok_to_fail_fips?""" + return Mark.marked(f, OK_TO_FAIL_FIPS) + + def is_env(f): return Mark.marked(f, ENV) @@ -459,6 +483,35 @@ def the_test(...): return args[0] +def ok_to_fail_fips(*args, **kwargs): + """ + Test method decorator which signals to the test runner to run test but to set OFAIL_FIPS or OPASS_FIPS. + This mark is only applied if the operating system is actually running in FIPS mode. If not, no mark is made + and test runs as normal + + Example:: + @ok_to_fail_fips + def the_test(...): + ... + """ + def running_fips() -> bool: + fips_file = "/proc/sys/crypto/fips_enabled" + if os.path.exists(fips_file) and os.path.isfile(fips_file): + with open(fips_file, 'r') as f: + contents = f.read().strip() + return contents == '1' + + return False + + if len(args) == 1 and len(kwargs) == 0 and running_fips(): + # this corresponds to the usage of the decorator with no arguments + # @ok_to_fail_fips + # def test_function: + # ... + Mark.mark(args[0], OkToFailFIPS()) + return args[0] + + def env(**kwargs): def environment(f): Mark.mark(f, Env(**kwargs)) diff --git a/ducktape/templates/report/report.css b/ducktape/templates/report/report.css index 8d41cbde2..123c23be5 100644 --- a/ducktape/templates/report/report.css +++ b/ducktape/templates/report/report.css @@ -86,6 +86,14 @@ h1, h2, h3, h4, h5, h6 { background-color: #9cf; } +.ofailfips { + background-color: #ffc +} + +.opassfips { + background-color: #9cf; +} + .testcase { margin-left: 2em; } diff --git a/ducktape/templates/report/report.html b/ducktape/templates/report/report.html index 8009b9e2c..dd8b79e68 100644 --- a/ducktape/templates/report/report.html +++ b/ducktape/templates/report/report.html @@ -14,6 +14,8 @@
+
+
diff --git a/ducktape/tests/reporter.py b/ducktape/tests/reporter.py index 791dc5955..7b824c6b5 100644 --- a/ducktape/tests/reporter.py +++ b/ducktape/tests/reporter.py @@ -26,7 +26,7 @@ from ducktape.utils.terminal_size import get_terminal_size from ducktape.utils.util import ducktape_version -from ducktape.tests.status import PASS, FAIL, IGNORE, FLAKY, OPASS, OFAIL +from ducktape.tests.status import PASS, FAIL, IGNORE, FLAKY, OPASS, OFAIL, OPASSFIPS, OFAILFIPS from ducktape.json_serializable import DucktapeJSONEncoder @@ -117,6 +117,8 @@ def footer_string(self): "ignored: %d" % self.results.num_ignored, "opassed: %d" % self.results.num_opassed, "ofailed: %d" % self.results.num_ofailed, + "opassedfips: %d" % self.results.num_opassedfips, + "ofailedfips: %d" % self.results.num_ofailedfips, "=" * self.width ] @@ -130,6 +132,8 @@ def report_string(self): failed = [] ofail = [] opass = [] + ofailfips = [] + opassfips = [] for result in self.results: if result.test_status == FAIL: failed.append(result) @@ -139,10 +143,14 @@ def report_string(self): opass.append(result) elif result.test_status == OFAIL: ofail.append(result) + elif result.test_status == OPASSFIPS: + opassfips.append(result) + elif result.test_status == OFAILFIPS: + ofailfips.append(result) else: passed.append(result) - ordered_results = passed + ignored + failed + opass + ofail + ordered_results = passed + ignored + failed + opass + ofail + opassfips + ofailfips report_lines = \ [SingleResultReporter(result).result_string() + "\n" + "-" * self.width for result in ordered_results] @@ -205,9 +213,14 @@ def report(self): testsuite['skipped'] += 1 elif result.test_status == OFAIL: testsuite['skipped'] += 1 + elif result.test_status == OPASSFIPS: + testsuite['skipped'] += 1 + elif result.test_status == OFAILFIPS: + testsuite['skipped'] += 1 total = self.results.num_failed + self.results.num_ignored + self.results.num_ofailed + \ - self.results.num_opassed + self.results.num_passed + self.results.num_flaky + self.results.num_opassed + self.results.num_passed + self.results.num_flaky + \ + self.results.num_opassedfips + self.results.num_ofailedfips # Now start building XML document root = ET.Element('testsuites', attrib=dict( name="ducktape", time=str(self.results.run_time_seconds), @@ -230,7 +243,7 @@ def report(self): name=name, classname=test.cls_name, time=str(test.run_time_seconds), status=str(test.test_status), assertions="" )) - if test.test_status == FAIL or test.test_status == OFAIL: + if test.test_status == FAIL or test.test_status == OFAIL or test.test_status == OFAILFIPS: xml_failure = ET.SubElement(xml_testcase, 'failure', attrib=dict( message=test.summary.splitlines()[0] )) @@ -297,6 +310,8 @@ def format_report(self): flaky_result_string = [] opassed_result_string = [] ofailed_result_string = [] + opassedfips_result_string = [] + ofailedfips_result_string = [] for result in self.results: json_string = json.dumps(self.format_result(result)) @@ -319,6 +334,12 @@ def format_report(self): elif result.test_status == OFAIL: ofailed_result_string.append(json_string) ofailed_result_string.append(",") + elif result.test_status == OPASSFIPS: + opassedfips_result_string.append(json_string) + opassedfips_result_string.append(",") + elif result.test_status == OFAILFIPS: + ofailedfips_result_string.append(json_string) + ofailedfips_result_string.append(",") else: raise Exception("Unknown test status in report: {}".format(result.test_status.to_json())) @@ -331,6 +352,8 @@ def format_report(self): 'num_ignored': self.results.num_ignored, 'num_opassed': self.results.num_opassed, 'num_ofailed': self.results.num_ofailed, + 'num_opassedfips': self.results.num_opassedfips, + 'num_ofailedfips': self.results.num_ofailedfips, 'run_time': format_time(self.results.run_time_seconds), 'session': self.results.session_context.session_id, 'passed_tests': "".join(passed_result_string), @@ -339,8 +362,10 @@ def format_report(self): 'ignored_tests': "".join(ignored_result_string), 'ofailed_tests': "".join(ofailed_result_string), 'opassed_tests': "".join(opassed_result_string), + 'ofailedfips_tests': "".join(ofailedfips_result_string), + 'opassedfips_tests': "".join(opassedfips_result_string), 'test_status_names': ",".join(["\'%s\'" % str(status) for status in - [PASS, FAIL, IGNORE, FLAKY, OPASS, OFAIL]]) + [PASS, FAIL, IGNORE, FLAKY, OPASS, OFAIL, OPASSFIPS, OFAILFIPS]]) } html = template % args diff --git a/ducktape/tests/result.py b/ducktape/tests/result.py index 3c90ad25c..48fdc3dfe 100644 --- a/ducktape/tests/result.py +++ b/ducktape/tests/result.py @@ -21,7 +21,7 @@ from ducktape.tests.reporter import SingleResultFileReporter from ducktape.utils.local_filesystem_utils import mkdir_p from ducktape.utils.util import ducktape_version -from ducktape.tests.status import FLAKY, PASS, FAIL, IGNORE, OPASS, OFAIL +from ducktape.tests.status import FLAKY, PASS, FAIL, IGNORE, OPASS, OFAIL, OPASSFIPS, OFAILFIPS class TestResult(object): @@ -174,6 +174,14 @@ def num_opassed(self): def num_ofailed(self): return len([r for r in self._results if r.test_status == OFAIL]) + @property + def num_opassedfips(self): + return len([r for r in self._results if r.test_status == OPASSFIPS]) + + @property + def num_ofailedfips(self): + return len([r for r in self._results if r.test_status == OFAILFIPS]) + @property def run_time_seconds(self): if self.start_time < 0: @@ -232,6 +240,8 @@ def to_json(self): "num_ignored": self.num_ignored, "num_opassed": self.num_opassed, "num_ofailed": self.num_ofailed, + "num_opassedfips": self.num_opassedfips, + "num_ofailedfips": self.num_ofailedfips, "parallelism": parallelism, "results": [r for r in self._results] } diff --git a/ducktape/tests/runner_client.py b/ducktape/tests/runner_client.py index 8abb70423..e445976fa 100644 --- a/ducktape/tests/runner_client.py +++ b/ducktape/tests/runner_client.py @@ -29,7 +29,7 @@ from ducktape.tests.status import FLAKY from ducktape.tests.test import test_logger, TestContext -from ducktape.tests.result import TestResult, IGNORE, PASS, FAIL, OPASS, OFAIL +from ducktape.tests.result import TestResult, IGNORE, PASS, FAIL, OPASS, OFAIL, OPASSFIPS, OFAILFIPS from ducktape.utils.local_filesystem_utils import mkdir_p @@ -185,12 +185,16 @@ def _do_run(self, num_runs): if self.test_context.ok_to_fail: test_status = OPASS + elif self.test_context.ok_to_fail_fips: + test_status = OPASSFIPS else: test_status = PASS except BaseException as e: if self.test_context.ok_to_fail: test_status = OFAIL + elif self.test_context.ok_to_fail_fips: + test_status = OFAILFIPS else: test_status = FAIL err_trace = self._exc_msg(e) @@ -249,6 +253,9 @@ def _check_cluster_utilization(self, result, summary): elif result == OPASS: self.log(logging.INFO, "OFAIL: " + message) result = OFAIL + elif result == OPASSFIPS: + self.log(logging.INFO, "OFAILFIPS: " + message) + result = OFAILFIPS summary += message else: self.log(logging.WARN, message) diff --git a/ducktape/tests/status.py b/ducktape/tests/status.py index 0264af2e9..458832472 100644 --- a/ducktape/tests/status.py +++ b/ducktape/tests/status.py @@ -33,3 +33,5 @@ def to_json(self): IGNORE = TestStatus("ignore") OPASS = TestStatus("opass") OFAIL = TestStatus("ofail") +OPASSFIPS = TestStatus("opassfips") +OFAILFIPS = TestStatus("ofailfips") diff --git a/ducktape/tests/test.py b/ducktape/tests/test.py index 1c0c8dd52..bcdf77127 100644 --- a/ducktape/tests/test.py +++ b/ducktape/tests/test.py @@ -28,7 +28,7 @@ from ducktape.services.service_registry import ServiceRegistry from ducktape.template import TemplateRenderer from ducktape.mark.resource import CLUSTER_SPEC_KEYWORD, CLUSTER_SIZE_KEYWORD -from ducktape.tests.status import FAIL, OFAIL +from ducktape.tests.status import FAIL, OFAIL, OFAILFIPS class Test(TemplateRenderer): @@ -151,7 +151,8 @@ def copy_service_logs(self, test_status): # Gather locations of logs to collect node_logs = [] for log_name in log_dirs.keys(): - if test_status == FAIL or test_status == OFAIL or self.should_collect_log(log_name, service): + if test_status == FAIL or test_status == OFAIL or test_status == OFAILFIPS or \ + self.should_collect_log(log_name, service): node_logs.append(log_dirs[log_name]["path"]) self.test_context.logger.debug("Preparing to copy logs from %s: %s" % @@ -305,6 +306,7 @@ def __init__(self, **kwargs): self.injected_args = kwargs.get("injected_args") self.ignore = kwargs.get("ignore", False) self.ok_to_fail = kwargs.get("ok_to_fail", False) + self.ok_to_fail_fips = kwargs.get("ok_to_fail_fips", False) # cluster_use_metadata is a dict containing information about how this test will use cluster resources self.cluster_use_metadata = copy.copy(kwargs.get("cluster_use_metadata", {})) @@ -321,9 +323,10 @@ def __init__(self, **kwargs): def __repr__(self): return \ "" % \ + "ok_to_fail=%s, ok_to_fail_fips=%s cluster_size=%s, cluster_spec=%s>" % \ (self.module, self.cls_name, self.function_name, str(self.injected_args), str(self.file), - str(self.ignore), str(self.ok_to_fail), str(self.expected_num_nodes), str(self.expected_cluster_spec)) + str(self.ignore), str(self.ok_to_fail), str(self.ok_to_fail_fips), str(self.expected_num_nodes), + str(self.expected_cluster_spec)) def copy(self, **kwargs): """Construct a new TestContext object from another TestContext object