From 1141207aaa2728d7c9ee60dc79a250d5add461fb Mon Sep 17 00:00:00 2001 From: Vivien Nicolas Date: Tue, 14 Feb 2023 19:24:29 +0100 Subject: [PATCH] [matter_yamltests] Add a dedicated class to load the yaml, enforce types, validate keywords, and add some implicit rules checking (#24975) --- scripts/py_matter_yamltests/BUILD.gn | 3 + .../matter_yamltests/__init__.py | 19 + .../matter_yamltests/errors.py | 157 ++++++ .../matter_yamltests/fixes.py | 3 +- .../matter_yamltests/parser.py | 188 +++---- .../matter_yamltests/yaml_loader.py | 235 ++++++++ .../py_matter_yamltests/test_yaml_loader.py | 512 ++++++++++++++++++ scripts/tests/chiptest/__init__.py | 2 + .../suites/certification/Test_TC_LVL_2_3.yaml | 2 +- .../suites/certification/Test_TC_LVL_8_1.yaml | 2 +- 10 files changed, 993 insertions(+), 130 deletions(-) create mode 100644 scripts/py_matter_yamltests/matter_yamltests/errors.py create mode 100644 scripts/py_matter_yamltests/matter_yamltests/yaml_loader.py create mode 100644 scripts/py_matter_yamltests/test_yaml_loader.py diff --git a/scripts/py_matter_yamltests/BUILD.gn b/scripts/py_matter_yamltests/BUILD.gn index 5c1a2c8bab7e7b..f0437ce74350fd 100644 --- a/scripts/py_matter_yamltests/BUILD.gn +++ b/scripts/py_matter_yamltests/BUILD.gn @@ -29,6 +29,7 @@ pw_python_package("matter_yamltests") { "matter_yamltests/__init__.py", "matter_yamltests/constraints.py", "matter_yamltests/definitions.py", + "matter_yamltests/errors.py", "matter_yamltests/fixes.py", "matter_yamltests/parser.py", "matter_yamltests/pics_checker.py", @@ -38,6 +39,7 @@ pw_python_package("matter_yamltests") { "matter_yamltests/pseudo_clusters/clusters/system_commands.py", "matter_yamltests/pseudo_clusters/pseudo_cluster.py", "matter_yamltests/pseudo_clusters/pseudo_clusters.py", + "matter_yamltests/yaml_loader.py", ] python_deps = [ "${chip_root}/scripts/py_matter_idl:matter_idl" ] @@ -46,6 +48,7 @@ pw_python_package("matter_yamltests") { "test_spec_definitions.py", "test_pics_checker.py", "test_pseudo_clusters.py", + "test_yaml_loader.py", ] # TODO: at a future time consider enabling all (* or missing) here to get diff --git a/scripts/py_matter_yamltests/matter_yamltests/__init__.py b/scripts/py_matter_yamltests/matter_yamltests/__init__.py index e69de29bb2d1d6..0aeba98392818f 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/__init__.py +++ b/scripts/py_matter_yamltests/matter_yamltests/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +assert sys.version_info >= ( + 3, 7), "Use Python 3.7 or newer for dictionary order guarantees" diff --git a/scripts/py_matter_yamltests/matter_yamltests/errors.py b/scripts/py_matter_yamltests/matter_yamltests/errors.py new file mode 100644 index 00000000000000..04ca6d7c049eb0 --- /dev/null +++ b/scripts/py_matter_yamltests/matter_yamltests/errors.py @@ -0,0 +1,157 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml + +_ERROR_START_TAG = '__error_start__' +_ERROR_END_TAG = '__error_end__' + + +class TestStepError(Exception): + """Raise when a step is malformed.""" + + def __init__(self, message): + self.step_index = 0 + self.context = None + self.message = message + + def __str__(self): + return self.message + + def update_context(self, context, step_index): + self.context = yaml.dump( + context, + default_flow_style=False, + sort_keys=False + ) + self.step_index = step_index + + def tag_key_with_error(self, content, target_key): + self.__tag_key(content, target_key, _ERROR_START_TAG, _ERROR_END_TAG) + + def __tag_key(self, content, target_key, tag_start, tag_end): + # This method replace the key for the dictionary with the tag provided while preserving the order of the dictionary + reversed_dictionary = {} + + # Build a reversed dictionary, tagging the target key. + for _ in range(len(content)): + key, value = content.popitem() + if key == target_key: + reversed_dictionary[tag_start + key + tag_end] = value + else: + reversed_dictionary[key] = value + + # Revert back the the dictionary to the original order. + for _ in range(len(reversed_dictionary)): + key, value = reversed_dictionary.popitem() + content[key] = value + + +class TestStepKeyError(TestStepError): + """Raise when a key is unknown.""" + + def __init__(self, content, key): + message = f'Unknown key "{key}"' + super().__init__(message) + + self.tag_key_with_error(content, key) + + +class TestStepValueNameError(TestStepError): + """Raise when a value name is unknown.""" + + def __init__(self, content, key, candidate_keys): + message = f'Unknown key: "{key}". Candidates are: "{candidate_keys}"' + for candidate_key in candidate_keys: + if candidate_key.lower() == key.lower(): + message = f'Unknown key: "{key}". Did you mean "{candidate_key}" ?' + break + super().__init__(message) + + self.tag_key_with_error(content, 'name') + + +class TestStepInvalidTypeError(TestStepError): + """Raise when the value for a given key is not of the expected type.""" + + def __init__(self, content, key, expected_type): + if isinstance(expected_type, tuple): + expected_name = '' + for _type in expected_type: + expected_name += _type.__name__ + ',' + expected_name = expected_name[:-1] + else: + expected_name = expected_type.__name__ + received_name = type(content[key]).__name__ + message = f'Unexpected type. Expecting "{expected_name}", got "{received_name}"' + super().__init__(message) + + self.tag_key_with_error(content, key) + + +class TestStepGroupResponseError(TestStepError): + """Raise when a test step targeting a group of nodes expects a response.""" + + def __init__(self, content): + message = 'Group command should not expect a response' + super().__init__(message) + + self.tag_key_with_error(content, 'groupId') + self.tag_key_with_error(content, 'response') + + +class TestStepVerificationStandaloneError(TestStepError): + """Raise when a test step with a verification key is enabled and not interactive.""" + + def __init__(self, content): + message = 'Step using "verification" key should either set "disabled: true" or "PICS: PICS_USER_PROMPT"' + super().__init__(message) + + self.tag_key_with_error(content, 'verification') + + +class TestStepNodeIdAndGroupIdError(TestStepError): + """Raise when a test step contains both "nodeId" and "groupId" keys.""" + + def __init__(self, content): + message = '"nodeId" and "groupId" are mutually exclusive' + super().__init__(message) + + self.tag_key_with_error(content, 'nodeId') + self.tag_key_with_error(content, 'groupId') + + +class TestStepValueAndValuesError(TestStepError): + """Raise when a test step response contains both "value" and "values" keys.""" + + def __init__(self, content): + message = '"value" and "values" are mutually exclusive' + super().__init__(message) + + self.tag_key_with_error(content, 'value') + self.tag_key_with_error(content, 'values') + + +class TestStepWaitResponseError(TestStepError): + """Raise when a test step is waiting for a particular event (e.g an attribute read) using the + wait keyword but also specify a response. + """ + + def __init__(self, content): + message = 'The "wait" key can not be used in conjuction with the "response" key' + super().__init__(message) + + self.tag_key_with_error(content, 'wait') + self.tag_key_with_error(content, 'response') diff --git a/scripts/py_matter_yamltests/matter_yamltests/fixes.py b/scripts/py_matter_yamltests/matter_yamltests/fixes.py index a488d83bfb87d7..0223c2e1a486f1 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/fixes.py +++ b/scripts/py_matter_yamltests/matter_yamltests/fixes.py @@ -98,7 +98,7 @@ def convert_yaml_octet_string_to_bytes(s: str) -> bytes: return binascii.unhexlify(accumulated_hex) -def try_add_yaml_support_for_scientific_notation_without_dot(loader): +def add_yaml_support_for_scientific_notation_without_dot(loader): regular_expression = re.compile(u'''^(?: [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)? |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+) @@ -111,7 +111,6 @@ def try_add_yaml_support_for_scientific_notation_without_dot(loader): u'tag:yaml.org,2002:float', regular_expression, list(u'-+0123456789.')) - return loader # This is a gross hack. The previous runner has a some internal states where an identity match one diff --git a/scripts/py_matter_yamltests/matter_yamltests/parser.py b/scripts/py_matter_yamltests/matter_yamltests/parser.py index 59fa5ecad9f371..5cef3f773405ef 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/parser.py +++ b/scripts/py_matter_yamltests/matter_yamltests/parser.py @@ -17,73 +17,12 @@ from dataclasses import dataclass, field from enum import Enum, auto -import yaml - from . import fixes from .constraints import get_constraints, is_typed_constraint from .definitions import SpecDefinitions +from .errors import TestStepError, TestStepKeyError, TestStepValueNameError from .pics_checker import PICSChecker - -_TESTS_SECTION = [ - 'name', - 'config', - 'tests', - 'PICS', -] - -_TEST_SECTION = [ - 'label', - 'cluster', - 'command', - 'disabled', - 'event', - 'eventNumber', - 'endpoint', - 'identity', - 'fabricFiltered', - 'groupId', - 'verification', - 'nodeId', - 'attribute', - 'PICS', - 'arguments', - 'response', - 'minInterval', - 'maxInterval', - 'timedInteractionTimeoutMs', - 'busyWaitMs', - 'wait', -] - -_TEST_ARGUMENTS_SECTION = [ - 'values', - 'value', -] - -_TEST_RESPONSE_SECTION = [ - 'value', - 'values', - 'error', - 'clusterError', - 'constraints', - 'type', - 'hasMasksSet', - 'contains', - 'saveAs' -] - -_ATTRIBUTE_COMMANDS = [ - 'readAttribute', - 'writeAttribute', - 'subscribeAttribute', - 'waitForReport', -] - -_EVENT_COMMANDS = [ - 'readEvent', - 'subscribeEvent', - 'waitForReport', -] +from .yaml_loader import YamlLoader class PostProcessCheckStatus(Enum): @@ -167,13 +106,6 @@ def _insert(self, state: PostProcessCheckStatus, category: PostProcessCheckType, self.entries.append(log) -def _check_valid_keys(section, valid_keys_dict): - if section: - for key in section: - if key not in valid_keys_dict: - raise KeyError(f'Unknown key: {key}') - - def _value_or_none(data, key): return data[key] if key in data else None @@ -199,8 +131,6 @@ def __init__(self, test: dict, config: dict, definitions: SpecDefinitions, pics_ self._parsing_config_variable_storage = config - _check_valid_keys(test, _TEST_SECTION) - self.label = _value_or_none(test, 'label') self.node_id = _value_or_config(test, 'nodeId', config) self.group_id = _value_or_config(test, 'groupId', config) @@ -221,13 +151,10 @@ def __init__(self, test: dict, config: dict, definitions: SpecDefinitions, pics_ self.wait_for = _value_or_none(test, 'wait') self.event_number = _value_or_none(test, 'eventNumber') - self.is_attribute = self.attribute and ( - self.command in _ATTRIBUTE_COMMANDS or self.wait_for in _ATTRIBUTE_COMMANDS) - self.is_event = self.event and ( - self.command in _EVENT_COMMANDS or self.wait_for in _EVENT_COMMANDS) + self.is_attribute = self.__is_attribute_command() + self.is_event = self.__is_event_command() arguments = _value_or_none(test, 'arguments') - _check_valid_keys(arguments, _TEST_ARGUMENTS_SECTION) self._convert_single_value_to_values(arguments) self.arguments_with_placeholders = arguments @@ -245,7 +172,6 @@ def __init__(self, test: dict, config: dict, definitions: SpecDefinitions, pics_ responses = [responses] for response in responses: - _check_valid_keys(response, _TEST_RESPONSE_SECTION) self._convert_single_value_to_values(response) self.responses_with_placeholders = responses @@ -288,11 +214,6 @@ def __init__(self, test: dict, config: dict, definitions: SpecDefinitions, pics_ self.update_arguments(self.arguments_with_placeholders) self.update_responses(self.responses_with_placeholders) - # The "wait_for" keyword do not support multiple responses. - if len(responses) > 1 and self.wait_for: - raise Exception( - 'The "wait_for" keyword can not be used with multiple expected responses') - # This performs a very basic sanity parse time check of constraints. This parsing happens # again inside post processing response since at that time we will have required variables # to substitute in. This parsing check here has value since some test can take a really @@ -324,7 +245,7 @@ def _convert_single_value_to_values(self, container): # Nothing to do for those keys. pass else: - raise KeyError(f'Unknown key: {key}') + raise TestStepKeyError(item, key) container['values'] = [value] @@ -365,12 +286,8 @@ def _update_with_definition(self, container: dict, mapping_type): else: target_key = value['name'] if mapping_type.get(target_key) is None: - for candidate_key in mapping_type: - if candidate_key.lower() == target_key.lower(): - raise KeyError( - f'"{self.label}": Unknown key: "{target_key}". Did you mean "{candidate_key}" ?') - raise KeyError( - f'"{self.label}": Unknown key: "{target_key}". Candidates are: "{[ key for key in mapping_type]}".') + raise TestStepValueNameError( + value, target_key, [key for key in mapping_type]) mapping = mapping_type[target_key] if key == 'value': @@ -402,6 +319,8 @@ def _update_value_with_definition(self, value, mapping_type): if key == 'FabricIndex' or key == 'fabricIndex': rv[key] = value[key] # int64u else: + if not mapping_type.get(key): + raise TestStepKeyError(value, key) mapping = mapping_type[key] rv[key] = self._update_value_with_definition( value[key], mapping) @@ -428,6 +347,25 @@ def _update_value_with_definition(self, value, mapping_type): return value + def __is_attribute_command(self) -> bool: + commands = { + 'readAttribute', + 'writeAttribute', + 'subscribeAttribute', + 'waitForReport', + } + + return self.attribute and (self.command in commands or self.wait_for in commands) + + def __is_event_command(self) -> bool: + commands = { + 'readEvent', + 'subscribeEvent', + 'waitForReport', + } + + return self.event and (self.command in commands or self.wait_for in commands) + class TestStep: '''A single YAML test action parsed from YAML. @@ -879,14 +817,18 @@ class YamlTests: def __init__(self, parsing_config_variable_storage: dict, definitions: SpecDefinitions, pics_checker: PICSChecker, tests: dict): self._parsing_config_variable_storage = parsing_config_variable_storage enabled_tests = [] - for test in tests: - test_with_placeholders = _TestStepWithPlaceholders( - test, self._parsing_config_variable_storage, definitions, pics_checker) - if test_with_placeholders.is_enabled: - enabled_tests.append(test_with_placeholders) + try: + for step_index, step in enumerate(tests): + test_with_placeholders = _TestStepWithPlaceholders( + step, self._parsing_config_variable_storage, definitions, pics_checker) + if test_with_placeholders.is_enabled: + enabled_tests.append(test_with_placeholders) + except TestStepError as e: + e.update_context(step, step_index) + raise + fixes.try_update_yaml_node_id_test_runner_state( enabled_tests, self._parsing_config_variable_storage) - self._runtime_config_variable_storage = copy.deepcopy( parsing_config_variable_storage) self._tests = enabled_tests @@ -915,15 +857,23 @@ class TestParserConfig: class TestParser: def __init__(self, test_file: str, parser_config: TestParserConfig = TestParserConfig()): - data = self.__load_yaml(test_file) - - _check_valid_keys(data, _TESTS_SECTION) + yaml_loader = YamlLoader() + name, pics, config, tests = yaml_loader.load(test_file) - self.name = _value_or_none(data, 'name') - self.PICS = _value_or_none(data, 'PICS') + self.__apply_config_override(config, parser_config.config_override) + self.__apply_legacy_config(config) - config = data.get('config', {}) - for key, value in parser_config.config_override.items(): + self.name = name + self.PICS = pics + self.tests = YamlTests( + config, + parser_config.definitions, + PICSChecker(parser_config.pics), + tests + ) + + def __apply_config_override(self, config, config_override): + for key, value in config_override.items(): if value is None: continue @@ -931,29 +881,15 @@ def __init__(self, test_file: str, parser_config: TestParserConfig = TestParserC config[key]['defaultValue'] = value else: config[key] = value - self._parsing_config_variable_storage = config + def __apply_legacy_config(self, config): # These are a list of "KnownVariables". These are defaults the codegen used to use. This # is added for legacy support of tests that expect to uses these "defaults". - self.__populate_default_config_if_missing('nodeId', 0x12345) - self.__populate_default_config_if_missing('endpoint', '') - self.__populate_default_config_if_missing('cluster', '') - self.__populate_default_config_if_missing('timeout', '90') - - pics_checker = PICSChecker(parser_config.pics) - tests = _value_or_none(data, 'tests') - self.tests = YamlTests( - self._parsing_config_variable_storage, parser_config.definitions, pics_checker, tests) - - def __populate_default_config_if_missing(self, key, value): - if key not in self._parsing_config_variable_storage: - self._parsing_config_variable_storage[key] = value - - def __load_yaml(self, test_file): - with open(test_file) as f: - loader = yaml.FullLoader - loader = fixes.try_add_yaml_support_for_scientific_notation_without_dot( - loader) - - return yaml.load(f, Loader=loader) - return None + self.__apply_legacy_config_if_missing(config, 'nodeId', 0x12345) + self.__apply_legacy_config_if_missing(config, 'endpoint', '') + self.__apply_legacy_config_if_missing(config, 'cluster', '') + self.__apply_legacy_config_if_missing(config, 'timeout', 90) + + def __apply_legacy_config_if_missing(self, config, key, value): + if key not in config: + config[key] = value diff --git a/scripts/py_matter_yamltests/matter_yamltests/yaml_loader.py b/scripts/py_matter_yamltests/matter_yamltests/yaml_loader.py new file mode 100644 index 00000000000000..543de252dc3820 --- /dev/null +++ b/scripts/py_matter_yamltests/matter_yamltests/yaml_loader.py @@ -0,0 +1,235 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Union + +from .errors import (TestStepError, TestStepGroupResponseError, TestStepInvalidTypeError, TestStepKeyError, + TestStepNodeIdAndGroupIdError, TestStepValueAndValuesError, TestStepVerificationStandaloneError, + TestStepWaitResponseError) +from .fixes import add_yaml_support_for_scientific_notation_without_dot + +try: + from yaml import CSafeLoader as SafeLoader +except: + from yaml import SafeLoader + +import yaml + + +class YamlLoader: + """This class loads a file from the disk and validates that the content is a well formed yaml test.""" + + def load(self, yaml_file: str) -> tuple[str, Union[list, str], dict, list]: + name = '' + pics = None + config = {} + tests = [] + + if yaml_file: + with open(yaml_file) as f: + loader = SafeLoader + add_yaml_support_for_scientific_notation_without_dot(loader) + content = yaml.load(f, Loader=loader) + + self.__check_content(content) + + name = content.get('name', '') + pics = content.get('PICS') + config = content.get('config', {}) + tests = content.get('tests', []) + + return (name, pics, config, tests) + + def __check_content(self, content): + schema = { + 'name': str, + 'PICS': (str, list), + 'config': dict, + 'tests': list, + } + + try: + self.__check(content, schema) + except TestStepError as e: + if 'tests' in content: + # This is a top level error. The content of the tests section + # does not really matter here and dumping it may be counter-productive + # since it can be very long... + content['tests'] = 'Skipped...' + e.update_context(content, 0) + raise + + tests = content.get('tests', []) + for step_index, step in enumerate(tests): + try: + self.__check_test_step(step) + except TestStepError as e: + e.update_context(step, step_index) + raise + + def __check_test_step(self, content): + schema = { + 'label': str, + 'identity': str, + 'nodeId': int, + 'groupId': int, + 'endpoint': int, + 'cluster': str, + 'attribute': str, + 'command': str, + 'event': str, + 'eventNumber': (int, str), # Can be a variable. + 'disabled': bool, + 'fabricFiltered': bool, + 'verification': str, + 'PICS': str, + 'arguments': dict, + 'response': (dict, list), + 'minInterval': int, + 'maxInterval': int, + 'timedInteractionTimeoutMs': int, + 'busyWaitMs': int, + 'wait': str, + } + + self.__check(content, schema) + self.__rule_node_id_and_group_id_are_mutually_exclusive(content) + self.__rule_group_step_should_not_expect_a_response(content) + self.__rule_step_with_verification_should_be_disabled_or_interactive( + content) + self.__rule_wait_should_not_expect_a_response(content) + + if 'arguments' in content: + arguments = content.get('arguments') + self.__check_test_step_arguments(arguments) + + if 'response' in content: + response = content.get('response') + if isinstance(response, list): + [self.__check_test_step_response(x) for x in response] + else: + self.__check_test_step_response(response) + + def __check_test_step_arguments(self, content): + schema = { + 'values': list, + 'value': (type(None), bool, str, int, float, dict, list), + } + + self.__check(content, schema) + + if 'values' in content: + values = content.get('values') + for value in values: + [self.__check_test_step_argument_value(x) for x in values] + + def __check_test_step_argument_value(self, content): + schema = { + 'value': (type(None), bool, str, int, float, dict, list), + 'name': str, + } + + self.__check(content, schema) + + def __check_test_step_response(self, content): + self.__rule_response_value_and_values_are_mutually_exclusive(content) + + if 'values' in content: + self.__check_type('values', content, list) + values = content.get('values') + [self.__check_test_step_response_value( + x, allow_name_key=True) for x in values] + else: + self.__check_test_step_response_value(content) + + def __check_test_step_response_value(self, content, allow_name_key=False): + schema = { + 'value': (type(None), bool, str, int, float, dict, list), + 'name': str, + 'error': str, + 'clusterError': int, + 'constraints': dict, + 'saveAs': str + } + + if allow_name_key: + schema['name'] = str + + self.__check(content, schema) + + if 'constraints' in content: + constraints = content.get('constraints') + self.__check_test_step_response_value_constraints(constraints) + + def __check_test_step_response_value_constraints(self, content): + schema = { + 'hasValue': bool, + 'type': str, + 'minLength': int, + 'maxLength': int, + 'isHexString': bool, + 'startsWith': str, + 'endsWith': str, + 'isUpperCase': bool, + 'isLowerCase': bool, + 'minValue': (int, float, str), # Can be a variable + 'maxValue': (int, float, str), # Can be a variable + 'contains': list, + 'excludes': list, + 'hasMasksSet': list, + 'hasMasksClear': list, + 'notValue': (type(None), bool, str, int, float, list, dict) + } + + self.__check(content, schema) + + def __check(self, content, schema): + for key in content: + if key not in schema: + raise TestStepKeyError(content, key) + + self.__check_type(key, content, schema.get(key)) + + def __check_type(self, key, content, expected_type): + value = content.get(key) + if isinstance(expected_type, tuple) and type(value) not in expected_type: + raise TestStepInvalidTypeError(content, key, expected_type) + elif not isinstance(expected_type, tuple) and type(value) is not expected_type: + raise TestStepInvalidTypeError(content, key, expected_type) + + def __rule_node_id_and_group_id_are_mutually_exclusive(self, content): + if 'nodeId' in content and 'groupId' in content: + raise TestStepNodeIdAndGroupIdError(content) + + def __rule_group_step_should_not_expect_a_response(self, content): + if 'groupId' in content and 'response' in content: + response = content.get('response') + if 'value' in response or 'values' in response: + raise TestStepGroupResponseError(content) + + def __rule_step_with_verification_should_be_disabled_or_interactive(self, content): + if 'verification' in content: + disabled = content.get('disabled') + command = content.get('command') + if disabled != True and command != 'UserPrompt': + raise TestStepVerificationStandaloneError(content) + + def __rule_response_value_and_values_are_mutually_exclusive(self, content): + if 'value' in content and 'values' in content: + raise TestStepValueAndValuesError(content) + + def __rule_wait_should_not_expect_a_response(self, content): + if 'wait' in content and 'response' in content: + raise TestStepWaitResponseError(content) diff --git a/scripts/py_matter_yamltests/test_yaml_loader.py b/scripts/py_matter_yamltests/test_yaml_loader.py new file mode 100644 index 00000000000000..d67dd062667ba1 --- /dev/null +++ b/scripts/py_matter_yamltests/test_yaml_loader.py @@ -0,0 +1,512 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import io +import unittest +from unittest.mock import mock_open, patch + +from matter_yamltests.errors import (TestStepGroupResponseError, TestStepInvalidTypeError, TestStepKeyError, + TestStepNodeIdAndGroupIdError, TestStepValueAndValuesError, + TestStepVerificationStandaloneError) +from matter_yamltests.yaml_loader import YamlLoader + + +def mock_open_with_parameter_content(content): + file_object = mock_open(read_data=content).return_value + file_object.__iter__.return_value = content.splitlines(True) + return file_object + + +@patch('builtins.open', new=mock_open_with_parameter_content) +class TestYamlLoader(unittest.TestCase): + def _get_wrong_values(self, valid_types, spaces=2): + values = [] + + if type(None) not in valid_types: + values.append('') + + if str not in valid_types: + values.append('A Test') + + if bool not in valid_types: + values.append(True) + + if int not in valid_types: + values.append(2) + + if float not in valid_types: + values.append(2.1) + + if dict not in valid_types: + values.append('\n' + (spaces * ' ') + + 'value: True\n' + (spaces * ' ') + 'values: False') + + if list not in valid_types: + values.append('\n' + (spaces * ' ') + + '- value: Test1\n' + (spaces * ' ') + '- value: Test2') + + return values + + def test_missing_file(self): + load = YamlLoader().load + + content = None + + name, pics, config, tests = load(content) + self.assertEqual(name, '') + self.assertEqual(pics, None) + self.assertEqual(config, {}) + self.assertEqual(tests, []) + + def test_empty_file(self): + load = YamlLoader().load + + content = '' + + name, pics, config, tests = load(content) + self.assertEqual(name, '') + self.assertEqual(pics, None) + self.assertEqual(config, {}) + self.assertEqual(tests, []) + + def test_key_unknown(self): + load = YamlLoader().load + + content = ''' + unknown: Test Name + ''' + + self.assertRaises(TestStepKeyError, load, content) + + def test_key_name(self): + load = YamlLoader().load + + content = ''' + name: Test Name + ''' + + name, _, _, _ = load(content) + self.assertEqual(name, 'Test Name') + + def test_key_name_wrong_values(self): + load = YamlLoader().load + + key = 'name' + values = self._get_wrong_values([str]) + [self.assertRaises(TestStepInvalidTypeError, load, + f'{key}: {x}') for x in values] + + def test_key_pics_string(self): + load = YamlLoader().load + + content = ''' + PICS: OO.S + ''' + + _, pics, _, _ = load(content) + self.assertEqual(pics, 'OO.S') + + def test_key_pics_list(self): + load = YamlLoader().load + + content = ''' + PICS: + - OO.S + - OO.C + ''' + + _, pics, _, _ = load(content) + self.assertEqual(pics, ['OO.S', 'OO.C']) + + def test_key_pics_wrong_values(self): + load = YamlLoader().load + + key = 'PICS' + values = self._get_wrong_values([str, list]) + [self.assertRaises(TestStepInvalidTypeError, load, + f'{key}: {x}') for x in values] + + def test_key_config(self): + load = YamlLoader().load + + content = ''' + config: + name: value + name2: value2 + ''' + + _, _, config, _ = load(content) + self.assertEqual(config, {'name': 'value', 'name2': 'value2'}) + + def test_key_config_wrong_values(self): + load = YamlLoader().load + + key = 'config' + values = self._get_wrong_values([dict]) + [self.assertRaises(TestStepInvalidTypeError, load, + f'{key}: {x}') for x in values] + + def test_key_tests(self): + load = YamlLoader().load + + content = ''' + tests: + - label: Test1 + - label: Test2 + ''' + + _, _, _, tests = load(content) + self.assertEqual(tests, [{'label': 'Test1'}, {'label': 'Test2'}]) + + def test_key_tests_wrong_values(self): + load = YamlLoader().load + + key = 'tests' + values = self._get_wrong_values([list]) + [self.assertRaises(TestStepInvalidTypeError, load, + f'{key}: {x}') for x in values] + + def test_key_tests_step_unknown_key(self): + load = YamlLoader().load + + content = ''' + tests: + - unknown: Test2 + ''' + + self.assertRaises(TestStepKeyError, load, content) + + def test_key_tests_step_bool_keys(self): + load = YamlLoader().load + + content = ('tests:\n' + ' - {key}: {value}') + keys = [ + 'disabled', + 'fabricFiltered', + ] + + wrong_values = self._get_wrong_values([bool], spaces=6) + for key in keys: + _, _, _, tests = load(content.format(key=key, value=True)) + self.assertEqual(tests, [{key: True}]) + + for value in wrong_values: + x = content.format(key=key, value=value) + self.assertRaises(TestStepInvalidTypeError, load, x) + + def test_key_tests_step_str_keys(self): + load = YamlLoader().load + + content = ('tests:\n' + ' - {key}: {value}') + keys = [ + 'label', + 'identity', + 'cluster', + 'attribute', + 'command', + 'event', + 'PICS', + 'wait', + ] + + # NOTE: 'verification' is excluded from this list despites beeing a key of type + # str. This is because 'verification' key has a rule that requires it to + # tied with either a 'disabled: True' or a 'command: UserPrompt'. + # As such it has dedicated tests. + + wrong_values = self._get_wrong_values([str], spaces=6) + for key in keys: + _, _, _, tests = load(content.format(key=key, value='a string')) + self.assertEqual(tests, [{key: 'a string'}]) + + for value in wrong_values: + x = content.format(key=key, value=value) + self.assertRaises(TestStepInvalidTypeError, load, x) + + def test_key_tests_step_int_keys(self): + load = YamlLoader().load + + content = ('tests:\n' + ' - {key}: {value}') + keys = [ + 'nodeId', + 'groupId', + 'endpoint', + 'minInterval', + 'maxInterval', + 'timedInteractionTimeoutMs', + 'busyWaitMs', + ] + + wrong_values = self._get_wrong_values([int], spaces=6) + for key in keys: + _, _, _, tests = load(content.format(key=key, value=1)) + self.assertEqual(tests, [{key: 1}]) + + for value in wrong_values: + x = content.format(key=key, value=value) + self.assertRaises(TestStepInvalidTypeError, load, x) + + def test_key_tests_step_dict_keys(self): + load = YamlLoader().load + + content = ('tests:\n' + ' - {key}: {value}') + keys = [ + 'arguments', + ] + + valid_value = ('\n' + ' value: True\n') + wrong_values = self._get_wrong_values([dict], spaces=6) + for key in keys: + _, _, _, tests = load(content.format(key=key, value=valid_value)) + self.assertEqual(tests, [{key: {'value': True}}]) + + for value in wrong_values: + x = content.format(key=key, value=value) + self.assertRaises(TestStepInvalidTypeError, load, x) + + def test_key_tests_step_response_key(self): + load = YamlLoader().load + + content = ('tests:\n' + ' - response: {value}') + + value = ('\n' + ' value: True\n') + _, _, _, tests = load(content.format(value=value)) + self.assertEqual(tests, [{'response': {'value': True}}]) + + value = ('\n' + ' - value: True\n') + _, _, _, tests = load(content.format(value=value)) + self.assertEqual(tests, [{'response': [{'value': True}]}]) + + wrong_values = self._get_wrong_values([dict, list], spaces=6) + for value in wrong_values: + x = content.format(value=value) + self.assertRaises(TestStepInvalidTypeError, load, x) + + def test_key_tests_step_event_number_key(self): + load = YamlLoader().load + + content = ('tests:\n' + ' - eventNumber: {value}') + + _, _, _, tests = load(content.format(value=1)) + self.assertEqual(tests, [{'eventNumber': 1}]) + + _, _, _, tests = load(content.format(value='TestKey')) + self.assertEqual(tests, [{'eventNumber': 'TestKey'}]) + + wrong_values = self._get_wrong_values([str, int], spaces=6) + for value in wrong_values: + x = content.format(value=value) + self.assertRaises(TestStepInvalidTypeError, load, x) + + def test_key_tests_step_verification_key(self): + load = YamlLoader().load + + content = ('tests:\n' + ' - verification: {value}\n' + ' disabled: true') + + _, _, _, tests = load(content.format(value='Test Sentence')) + self.assertEqual( + tests, [{'verification': 'Test Sentence', 'disabled': True}]) + + wrong_values = self._get_wrong_values([str, int], spaces=6) + for value in wrong_values: + x = content.format(value=value) + self.assertRaises(TestStepInvalidTypeError, load, x) + + # TODO + # 'verification', + + def test_key_tests_step_rule_node_id_and_group_id_are_mutually_exclusive(self): + load = YamlLoader().load + + content = ''' + tests: + - label: A Test Name + nodeId: 0 + groupId: 1 + ''' + + self.assertRaises(TestStepNodeIdAndGroupIdError, load, content) + + def test_key_tests_step_rule_group_step_should_not_expect_a_response(self): + load = YamlLoader().load + + content = ''' + tests: + - label: A Test Name + groupId: 1 + response: + value: An expected value + ''' + + self.assertRaises(TestStepGroupResponseError, load, content) + + def test_key_tests_step_rule_step_with_verification_should_be_disabled_or_interactive(self): + load = YamlLoader().load + + content = ''' + tests: + - label: A Test Name + verification: A verification sentence + ''' + + self.assertRaises(TestStepVerificationStandaloneError, load, content) + + content = ''' + tests: + - label: A Test Name + verification: A verification sentence + disabled: false + ''' + + self.assertRaises(TestStepVerificationStandaloneError, load, content) + + content = ''' + tests: + - label: A Test Name + verification: A verification sentence + disabled: true + ''' + + _, _, _, tests = load(content) + self.assertEqual(tests, [ + {'label': 'A Test Name', 'verification': 'A verification sentence', 'disabled': True}]) + + content = ''' + tests: + - label: A Test Name + verification: A verification sentence + command: Something + ''' + + self.assertRaises(TestStepVerificationStandaloneError, load, content) + + content = ''' + tests: + - label: A Test Name + verification: A verification sentence + command: UserPrompt + ''' + + _, _, _, tests = load(content) + self.assertEqual(tests, [ + {'label': 'A Test Name', 'verification': 'A verification sentence', 'command': 'UserPrompt'}]) + + def test_key_tests_step_response_key_value_key(self): + # NOTE: The value key can be of any type. + pass + + def test_key_tests_step_response_key_values_key(self): + load = YamlLoader().load + + content = ('tests:\n' + ' - response:\n' + ' values: {value}') + + _, _, _, tests = load(content.format(value=[])) + self.assertEqual(tests, [{'response': {'values': []}}]) + + wrong_values = self._get_wrong_values([list], spaces=8) + for value in wrong_values: + x = content.format(value=value) + self.assertRaises(TestStepInvalidTypeError, load, x) + + def test_key_tests_step_response_key_error_key(self): + load = YamlLoader().load + + content = ('tests:\n' + ' - response:\n' + ' error: {value}') + + _, _, _, tests = load(content.format(value='AnError')) + self.assertEqual(tests, [{'response': {'error': 'AnError'}}]) + + wrong_values = self._get_wrong_values([str], spaces=8) + for value in wrong_values: + x = content.format(value=value) + self.assertRaises(TestStepInvalidTypeError, load, x) + + def test_key_tests_step_response_key_cluster_error_key(self): + load = YamlLoader().load + + content = ('tests:\n' + ' - response:\n' + ' clusterError: {value}') + + _, _, _, tests = load(content.format(value=1)) + self.assertEqual(tests, [{'response': {'clusterError': 1}}]) + + wrong_values = self._get_wrong_values([int], spaces=8) + for value in wrong_values: + x = content.format(value=value) + self.assertRaises(TestStepInvalidTypeError, load, x) + + def test_key_tests_step_response_key_constraints_key(self): + load = YamlLoader().load + + content = ('tests:\n' + ' - response:\n' + ' constraints: {value}') + + _, _, _, tests = load(content.format(value={})) + self.assertEqual(tests, [{'response': {'constraints': {}}}]) + + wrong_values = self._get_wrong_values([dict], spaces=8) + for value in wrong_values: + x = content.format(value=value) + self.assertRaises(TestStepInvalidTypeError, load, x) + + def test_key_tests_step_response_key_save_as_key(self): + load = YamlLoader().load + + content = ('tests:\n' + ' - response:\n' + ' saveAs: {value}') + + _, _, _, tests = load(content.format(value='AKey')) + self.assertEqual(tests, [{'response': {'saveAs': 'AKey'}}]) + + wrong_values = self._get_wrong_values([str], spaces=8) + for value in wrong_values: + x = content.format(value=value) + self.assertRaises(TestStepInvalidTypeError, load, x) + + def test_rule_response_value_and_values_are_mutually_exclusive(self): + load = YamlLoader().load + + content = ('tests:\n' + ' - response:\n' + ' value: 1\n' + ' values: []') + + self.assertRaises(TestStepValueAndValuesError, load, content) + + # TODO Check constraints + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py index b0f47556f151c1..5baa27da80b54f 100644 --- a/scripts/tests/chiptest/__init__.py +++ b/scripts/tests/chiptest/__init__.py @@ -131,6 +131,8 @@ def _GetInDevelopmentTests() -> Set[str]: Goal is for this set to become empty. """ return { + # Needs group support in repl / The test is incorrect - see https://github.com/CHIP-Specifications/chip-test-plans/issues/2431 for details. + "Test_TC_SC_5_2.yaml", "TestGroupMessaging.yaml", # Needs group support in repl } diff --git a/src/app/tests/suites/certification/Test_TC_LVL_2_3.yaml b/src/app/tests/suites/certification/Test_TC_LVL_2_3.yaml index bda3f6a1b3459e..05f80097b803c1 100644 --- a/src/app/tests/suites/certification/Test_TC_LVL_2_3.yaml +++ b/src/app/tests/suites/certification/Test_TC_LVL_2_3.yaml @@ -29,7 +29,7 @@ config: tests: - label: "Note" - verifaction: | + verification: | For DUT as client test cases, Chip-tool command used below are an example to verify the functionality. For certification test, we expect DUT should have a capability or way to run the equivalent command. disabled: true diff --git a/src/app/tests/suites/certification/Test_TC_LVL_8_1.yaml b/src/app/tests/suites/certification/Test_TC_LVL_8_1.yaml index 4699d7e87c8895..9beb45a4e81f65 100644 --- a/src/app/tests/suites/certification/Test_TC_LVL_8_1.yaml +++ b/src/app/tests/suites/certification/Test_TC_LVL_8_1.yaml @@ -25,7 +25,7 @@ config: tests: - label: "Note" - verifaction: | + verification: | For DUT as client test cases, Chip-tool command used below are an example to verify the functionality. For certification test, we expect DUT should have a capability or way to run the equivalent command. disabled: true