From 1103876e6d1d002f9ce3cfad547b661795ef2463 Mon Sep 17 00:00:00 2001 From: Vivien Nicolas Date: Mon, 24 Jul 2023 15:36:30 +0200 Subject: [PATCH] [CI] Automate Test_TC_IDM_1_2.yaml (#28071) * [chiptool.py] Allow '*' to be used as a top level endpoint value * [matter_yamltests] Add saveResponseAs step level keyword to save the whole response * [matter_yamltests] Add an optional definitions parameters for the methods of pseudo clusters * [matter_yamltests] Add WildcardResponseExtractorCluster * [CI] Automate Test_TC_IDM_1_2.yaml --- .../matter_chip_tool_adapter/encoder.py | 2 + .../matter_yamltests/parser.py | 5 + .../pseudo_clusters/pseudo_clusters.py | 9 +- .../matter_yamltests/runner.py | 2 +- .../matter_yamltests/yaml_loader.py | 1 + scripts/tests/chiptest/__init__.py | 1 + .../wildcard_response_extractor_cluster.py | 128 ++++++ .../suites/certification/Test_TC_IDM_1_2.yaml | 384 ++++++++++++++++++ 8 files changed, 529 insertions(+), 3 deletions(-) create mode 100644 scripts/tests/yaml/extensions/wildcard_response_extractor_cluster.py create mode 100644 src/app/tests/suites/certification/Test_TC_IDM_1_2.yaml diff --git a/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/encoder.py b/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/encoder.py index 8340289c0e2852..2c02811f865845 100644 --- a/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/encoder.py +++ b/examples/chip-tool/py_matter_chip_tool_adapter/matter_chip_tool_adapter/encoder.py @@ -309,6 +309,8 @@ def __maybe_add_endpoint(self, rv, request): endpoint_argument_name = 'endpoint-id-ignored-for-group-commands' endpoint_argument_value = request.endpoint + if endpoint_argument_value == '*': + endpoint_argument_value = 0xFFFF if (request.is_attribute and not request.command == "writeAttribute") or request.is_event or (request.command in _ANY_COMMANDS_LIST and not request.command == "WriteById"): endpoint_argument_name = 'endpoint-ids' diff --git a/scripts/py_matter_yamltests/matter_yamltests/parser.py b/scripts/py_matter_yamltests/matter_yamltests/parser.py index 82b798a85156a4..07090679fd9dbb 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/parser.py +++ b/scripts/py_matter_yamltests/matter_yamltests/parser.py @@ -208,6 +208,7 @@ def __init__(self, test: dict, config: dict, definitions: SpecDefinitions, pics_ self.wait_for = _value_or_none(test, 'wait') self.event_number = _value_or_none(test, 'eventNumber') self.run_if = _value_or_none(test, 'runIf') + self.save_response_as = _value_or_none(test, 'saveResponseAs') self.is_attribute = self.__is_attribute_command() self.is_event = self.__is_event_command() @@ -695,6 +696,9 @@ def post_process_response(self, received_responses): if not isinstance(received_responses, list): received_responses = [received_responses] + if self._test.save_response_as: + self._runtime_config_variable_storage[self._test.save_response_as] = received_responses + if self.wait_for is not None: self._response_cluster_wait_validation(received_responses, result) return result @@ -1112,6 +1116,7 @@ def __init__(self, test_file: str, parser_config: TestParserConfig = TestParserC tests ) self.timeout = config['timeout'] + self.definitions = parser_config.definitions def __apply_config_override(self, config, config_override): for key, value in config_override.items(): diff --git a/scripts/py_matter_yamltests/matter_yamltests/pseudo_clusters/pseudo_clusters.py b/scripts/py_matter_yamltests/matter_yamltests/pseudo_clusters/pseudo_clusters.py index e194615cdea11c..7ab24f086442a9 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/pseudo_clusters/pseudo_clusters.py +++ b/scripts/py_matter_yamltests/matter_yamltests/pseudo_clusters/pseudo_clusters.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import inspect from typing import List from .clusters.commissioner_commands import CommissionerCommands @@ -33,12 +34,16 @@ def supports(self, request) -> bool: def add(self, cluster: PseudoCluster): self.clusters.append(cluster) - async def execute(self, request): + async def execute(self, request, definitions=None): status = {'error': 'FAILURE'} command = self.__get_command(request) if command: - status = await command(request) + if 'definitions' in inspect.signature(command).parameters: + status = await command(request, definitions) + else: + status = await command(request) + # If the command does not returns an error, it is considered a success. if status is None: status = {} diff --git a/scripts/py_matter_yamltests/matter_yamltests/runner.py b/scripts/py_matter_yamltests/matter_yamltests/runner.py index 08de8d758e84fd..c819eea79d9a56 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/runner.py +++ b/scripts/py_matter_yamltests/matter_yamltests/runner.py @@ -186,7 +186,7 @@ async def _run(self, parser: TestParser, config: TestRunnerConfig): start = time.time() if config.pseudo_clusters.supports(request): - responses, logs = await config.pseudo_clusters.execute(request) + responses, logs = await config.pseudo_clusters.execute(request, parser.definitions) else: encoded_request = config.adapter.encode(request) encoded_response = await self.execute(encoded_request) diff --git a/scripts/py_matter_yamltests/matter_yamltests/yaml_loader.py b/scripts/py_matter_yamltests/matter_yamltests/yaml_loader.py index 50a9217ee18b1a..ecdc7af209acbe 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/yaml_loader.py +++ b/scripts/py_matter_yamltests/matter_yamltests/yaml_loader.py @@ -103,6 +103,7 @@ def __check_test_step(self, config: dict, content): 'PICS': str, 'arguments': dict, 'response': (dict, list, str), # Can be a variable + 'saveResponseAs': str, 'minInterval': int, 'maxInterval': int, 'timeout': int, diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py index 87c308c60030f6..26aed1ddc34138 100644 --- a/scripts/tests/chiptest/__init__.py +++ b/scripts/tests/chiptest/__init__.py @@ -148,6 +148,7 @@ def _GetInDevelopmentTests() -> Set[str]: "Test_TC_SMCO_2_4.yaml", # chip-repl does not support timeout (07/20/2023) "Test_TC_SMCO_2_5.yaml", # chip-repl does not support timeout (07/20/2023) "Test_TC_SMCO_2_6.yaml", # chip-repl does not support timeout (07/20/2023) + "Test_TC_IDM_1_2.yaml", # chip-repl does not support AnyCommands (19/07/2023) } diff --git a/scripts/tests/yaml/extensions/wildcard_response_extractor_cluster.py b/scripts/tests/yaml/extensions/wildcard_response_extractor_cluster.py new file mode 100644 index 00000000000000..a0e34cc148d8ac --- /dev/null +++ b/scripts/tests/yaml/extensions/wildcard_response_extractor_cluster.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from matter_yamltests.pseudo_clusters.pseudo_cluster import PseudoCluster + +_VALUE_ARGUMENT_NAME = 'Value' +_CLUSTERS_ARGUMENT_NAME = 'Clusters' + + +class wildcard_response_extractor_cluster(PseudoCluster): + name = 'WildcardResponseExtractorCluster' + + async def GetDefaultEndPointForClusters(self, request, definitions): + entries = self.__get_argument(request, _VALUE_ARGUMENT_NAME) + clusters = self.__get_argument(request, _CLUSTERS_ARGUMENT_NAME) + if entries is None or clusters is None: + return {'error': 'INVALID_ARGUMENT'} + + results = {} + + for cluster in clusters: + cluster_id = definitions.get_cluster_id_by_name(cluster) + results[cluster] = None + + for entry in entries: + server_list = entry.get('value') + if cluster_id in server_list: + results[cluster] = entry.get('endpoint') + break + + return {'value': results} + + async def GetUnsupportedCluster(self, request): + entries = self.__get_argument(request, _VALUE_ARGUMENT_NAME) + if entries is None: + return {'error': 'INVALID_ARGUMENT'} + + cluster_ids = [] + for entry in entries: + server_list = entry.get('value') + for cluster_id in server_list: + if cluster_id not in cluster_ids: + cluster_ids.append(cluster_id) + + unsupported_cluster = None + for cluster_code in range(0xFFFFFFFF): + if cluster_code not in cluster_ids: + unsupported_cluster = f'{cluster_code:#0{10}x}' + break + + return {'value': {'UnsupportedCluster': unsupported_cluster}} + + async def GetUnsupportedCommand(self, request): + entries = self.__get_argument(request, _VALUE_ARGUMENT_NAME) + if entries is None: + return {'error': 'INVALID_ARGUMENT'} + + command_ids = [] + for entry in entries: + commands_list = entry.get('value') + for command_id in commands_list: + if command_id not in command_ids: + command_ids.append(command_id) + + unsupported_command = None + for command_code in range(0xFFFFFFFF): + if command_code not in command_ids: + unsupported_command = f'{command_code:#0{10}x}' + break + + return {'value': {'UnsupportedCommand': unsupported_command}} + + async def GetUnsupportedEndPoint(self, request): + entries = self.__get_argument(request, _VALUE_ARGUMENT_NAME) + if entries is None: + return {'error': 'INVALID_ARGUMENT'} + + endpoint_ids = [] + for entry in entries: + parts_list = entry.get('value') + for endpoint_id in parts_list: + if endpoint_id not in endpoint_ids: + endpoint_ids.append(endpoint_id) + + # Add the endpoint id of the response if needed. + endpoint_id = entry.get('endpoint') + if endpoint_id not in endpoint_ids: + endpoint_ids.append(endpoint_id) + + unsupported_endpoint = None + for endpoint_code in range(0xFFFF): + if endpoint_code not in endpoint_ids: + unsupported_endpoint = endpoint_code + break + + return {'value': {'UnsupportedEndPoint': unsupported_endpoint}} + + def __get_argument(self, request, argument_name): + arguments = request.arguments.get('values') + if arguments is None: + return None + + if not type(arguments) is list: + return None + + for argument in arguments: + name = argument.get('name') + value = argument.get('value') + if name is None or value is None: + return None + + if name == argument_name: + return value + + return None diff --git a/src/app/tests/suites/certification/Test_TC_IDM_1_2.yaml b/src/app/tests/suites/certification/Test_TC_IDM_1_2.yaml new file mode 100644 index 00000000000000..ff7e3d0a241322 --- /dev/null +++ b/src/app/tests/suites/certification/Test_TC_IDM_1_2.yaml @@ -0,0 +1,384 @@ +# Copyright (c) 2021-2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 3.1.2. [TC-IDM-1.2] Invoke Response Action from DUT to TH - [{DUT_Server}] + +PICS: + - MCORE.IDM.S + +config: + nodeId: 0x12344321 + InvokeRequestMessage.Cluster: 0x00000006 # OnOff + InvokeRequestMessage.Command: 0x00000000 # Off + InvokeRequestMessage.EndPoint: 1 + InvokeRequestMessage.Payload: {} + +tests: + - label: "Wait for the commissioned device to be retrieved" + cluster: "DelayCommands" + command: "WaitForCommissionee" + arguments: + values: + - name: "nodeId" + value: nodeId + + #### Test Setup + + - label: "Get the list of available endpoints on the device" + cluster: "Descriptor" + command: "readAttribute" + attribute: "PartsList" + endpoint: 0 + saveResponseAs: AvailableDeviceEndPoints + + - label: + "Get an unsupported endpoint id from the list of available device + endpoints" + cluster: WildcardResponseExtractorCluster + command: GetUnsupportedEndPoint + arguments: + values: + - name: Value + value: AvailableDeviceEndPoints + response: + values: + - name: UnsupportedEndPoint + saveAs: PIXIT.IDM.UnsupportedEndPoint + + - label: "Get the list of available clusters on the device (all endpoints)" + cluster: "Descriptor" + command: "readAttribute" + attribute: "ServerList" + endpoint: "*" + saveResponseAs: AvailableDeviceClusters + + - label: + "Extract the endpoint values for the AccessControl, + GeneralCommissioning, AdministratorCommissioning and + OperationalCredentials cluster" + cluster: WildcardResponseExtractorCluster + command: GetDefaultEndPointForClusters + arguments: + values: + - name: Value + value: AvailableDeviceClusters + - name: Clusters + value: + [ + "AccessControl", + "GeneralCommissioning", + "AdministratorCommissioning", + "OperationalCredentials", + ] + response: + values: + - name: AccessControl + saveAs: PIXIT.Cluster.AccessControl.EndPoint + - name: GeneralCommissioning + saveAs: PIXIT.Cluster.GeneralCommissioning.EndPoint + - name: AdministratorCommissioning + saveAs: PIXIT.Cluster.AdministratorCommissioning.EndPoint + - name: OperationalCredentials + saveAs: PIXIT.Cluster.OperationalCredentials.EndPoint + + - label: + "Get an unsupported cluster id from the list of available device + clusters" + cluster: WildcardResponseExtractorCluster + command: GetUnsupportedCluster + arguments: + values: + - name: Value + value: AvailableDeviceClusters + response: + values: + - name: UnsupportedCluster + saveAs: PIXIT.IDM.UnsupportedCluster + + - label: + "Get the list of available cluster commands on the device (all + endpoints)" + cluster: "AnyCommands" + command: "ReadById" + endpoint: "*" + arguments: + values: + - name: "ClusterId" + value: "*" + - name: "AttributeId" + value: 0x0000FFF9 + saveResponseAs: AvailableDeviceCommands + + - label: + "Get an unsupported command id from the list of available device + commands" + cluster: WildcardResponseExtractorCluster + command: GetUnsupportedCommand + arguments: + values: + - name: Value + value: AvailableDeviceCommands + response: + values: + - name: UnsupportedCommand + saveAs: PIXIT.IDM.UnsupportedCommand + + - label: "Read the fabric index from the alpha fabric" + cluster: "Operational Credentials" + command: "readAttribute" + endpoint: PIXIT.Cluster.OperationalCredentials.EndPoint + attribute: "CurrentFabricIndex" + response: + saveAs: alphaIndex + + - label: "Read the commissioner node ID from the alpha fabric" + cluster: "CommissionerCommands" + command: "GetCommissionerNodeId" + response: + values: + - name: "nodeId" + saveAs: commissionerNodeIdAlpha + + ##### Test Implementation + + - label: + "TH sends the Invoke Request Message to the DUT with the path that + indicates a specific endpoint that is unsupported." + cluster: "AnyCommands" + command: "CommandById" + endpoint: PIXIT.IDM.UnsupportedEndPoint + arguments: + values: + - name: "ClusterId" + value: InvokeRequestMessage.Cluster + - name: "CommandId" + value: InvokeRequestMessage.Command + - name: "Payload" + value: InvokeRequestMessage.Payload + response: + error: UNSUPPORTED_ENDPOINT + + - label: + "TH sends the Invoke Request Message to the DUT with the path that + indicates a specific cluster that is unsupported." + cluster: "AnyCommands" + command: "CommandById" + endpoint: InvokeRequestMessage.EndPoint + arguments: + values: + - name: "ClusterId" + value: PIXIT.IDM.UnsupportedCluster + - name: "CommandId" + value: InvokeRequestMessage.Command + - name: "Payload" + value: InvokeRequestMessage.Payload + response: + error: UNSUPPORTED_CLUSTER + + - label: + "TH sends the Invoke Request Message to the DUT with the path that + indicates a specific command that is unsupported." + cluster: "AnyCommands" + command: "CommandById" + endpoint: InvokeRequestMessage.EndPoint + arguments: + values: + - name: "ClusterId" + value: InvokeRequestMessage.Cluster + - name: "CommandId" + value: PIXIT.IDM.UnsupportedCommand + - name: "Payload" + value: InvokeRequestMessage.Payload + response: + error: UNSUPPORTED_COMMAND + + - label: + "Setup the TH such that it should not have the privilege for the + cluster in the path." + cluster: "AccessControl" + command: "writeAttribute" + attribute: "ACL" + endpoint: PIXIT.Cluster.AccessControl.EndPoint + arguments: + value: + [ + { + "fabricIndex": alphaIndex, + "Privilege": 5, + "AuthMode": 2, + "Subjects": [commissionerNodeIdAlpha], + "Targets": + [ + { + "Cluster": 31, + "Endpoint": 0, + "DeviceType": null, + }, + ], + }, + ] + + - label: + "TH sends the Invoke Request Message to the DUT with a valid + CommandDataIB" + cluster: "AnyCommands" + command: "CommandById" + endpoint: InvokeRequestMessage.EndPoint + arguments: + values: + - name: "ClusterId" + value: InvokeRequestMessage.Cluster + - name: "CommandId" + value: InvokeRequestMessage.Command + - name: "Payload" + value: InvokeRequestMessage.Payload + response: + error: UNSUPPORTED_ACCESS + + - label: + "TH sends the Invoke Request Message to the DUT with a valid and + fabric-scoped CommandDataIB" + cluster: "General Commissioning" + command: "CommissioningComplete" + endpoint: PIXIT.Cluster.GeneralCommissioning.EndPoint + response: + error: UNSUPPORTED_ACCESS + + - label: + "Setup the TH such that it should have the privilege for the cluster + in the path." + cluster: "AccessControl" + command: "writeAttribute" + endpoint: PIXIT.Cluster.AccessControl.EndPoint + attribute: "ACL" + arguments: + value: + [ + { + "fabricIndex": alphaIndex, + "Privilege": 5, + "AuthMode": 2, + "Subjects": [commissionerNodeIdAlpha], + "Targets": null, + }, + ] + + - label: + "(OPTIONAL) TH sends the Invoke Request Message to the DUT with the + command which requires a data response to be sent back." + cluster: "General Commissioning" + command: "ArmFailSafe" + endpoint: PIXIT.Cluster.GeneralCommissioning.EndPoint + arguments: + values: + - name: "ExpiryLengthSeconds" + value: 1000 + - name: "Breadcrumb" + value: 1 + response: + values: + - name: "ErrorCode" + value: 0 + + - label: + "TH sends the Invoke Request Message to the DUT with a valid + CommandDataIB and SuppressResponse set to True" + disabled: true + verification: | + Out of Scope for V1.0 + https://github.com/project-chip/connectedhomeip/issues/8043 + cluster: InvokeRequestMessage.Cluster + command: InvokeRequestMessage.Command + endpoint: InvokeRequestMessage.EndPoint + arguments: + values: + - name: "Payload" + value: PIXIT.InvokeRequestMessage.Payload + + - label: + "Setup the TH such that it should not have the privilege for the + cluster in the path." + cluster: "AccessControl" + command: "writeAttribute" + attribute: "ACL" + endpoint: PIXIT.Cluster.AccessControl.EndPoint + arguments: + value: + [ + { + "fabricIndex": alphaIndex, + "Privilege": 5, + "AuthMode": 2, + "Subjects": [commissionerNodeIdAlpha], + "Targets": + [ + { + "Cluster": 31, + "Endpoint": 0, + "DeviceType": null, + }, + ], + }, + ] + + - label: + "TH sends a Invoke Request Message to the DUT with the TimedRequest + set as TRUE.(There should be no previous Timed Invoke action.)" + cluster: "AnyCommands" + command: "CommandById" + endpoint: InvokeRequestMessage.EndPoint + timedInteractionTimeoutMs: 1000 + arguments: + values: + - name: "ClusterId" + value: InvokeRequestMessage.Cluster + - name: "CommandId" + value: InvokeRequestMessage.Command + - name: "Payload" + value: InvokeRequestMessage.Payload + response: + error: UNSUPPORTED_ACCESS + + - label: + "Setup the TH such that it should have the privilege for the cluster + in the path." + cluster: "AccessControl" + command: "writeAttribute" + endpoint: PIXIT.Cluster.AccessControl.EndPoint + attribute: "ACL" + arguments: + value: + [ + { + "fabricIndex": alphaIndex, + "Privilege": 5, + "AuthMode": 2, + "Subjects": [commissionerNodeIdAlpha], + "Targets": null, + }, + ] + + - label: + "TH sends Invoke Request Message to the DUT with the command in the + path that requires a Timed Invoke transaction to invoke and this + action is not part of a Timed Invoke transaction" + cluster: "AdministratorCommissioning" + command: "OpenBasicCommissioningWindow" + endpoint: PIXIT.Cluster.AdministratorCommissioning.EndPoint + arguments: + values: + - name: "CommissioningTimeout" + value: 180 + response: + error: NEEDS_TIMED_INTERACTION