diff --git a/.flake8 b/.flake8 index eb983226f07b05..fe0075bf560c49 100644 --- a/.flake8 +++ b/.flake8 @@ -90,6 +90,7 @@ exclude = third_party src/controller/python/chip/yaml/__init__.py src/controller/python/chip/yaml/format_converter.py src/controller/python/chip/yaml/runner.py + src/controller/python/py_matter_yamltest_repl_adapter/matter_yamltest_repl_adapter/runner.py src/lib/asn1/gen_asn1oid.py src/pybindings/pycontroller/build-chip-wheel.py src/pybindings/pycontroller/pychip/__init__.py diff --git a/scripts/py_matter_yamltests/matter_yamltests/runner.py b/scripts/py_matter_yamltests/matter_yamltests/runner.py index 66855a528bc4a7..4c5db7a9d2b8d8 100644 --- a/scripts/py_matter_yamltests/matter_yamltests/runner.py +++ b/scripts/py_matter_yamltests/matter_yamltests/runner.py @@ -175,7 +175,9 @@ async def _run(self, parser: TestParser, config: TestRunnerConfig): if config.pseudo_clusters.supports(request): responses, logs = await config.pseudo_clusters.execute(request) else: - responses, logs = config.adapter.decode(await self.execute(config.adapter.encode(request))) + encoded_request = config.adapter.encode(request) + encoded_response = await self.execute(encoded_request) + responses, logs = config.adapter.decode(encoded_response) duration = round((time.time() - start) * 1000, 2) test_duration += duration diff --git a/scripts/tests/chiptest/yamltest_with_chip_repl_tester.py b/scripts/tests/chiptest/yamltest_with_chip_repl_tester.py index 625b4d95b149fa..db06b31769d030 100644 --- a/scripts/tests/chiptest/yamltest_with_chip_repl_tester.py +++ b/scripts/tests/chiptest/yamltest_with_chip_repl_tester.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import atexit import logging import os @@ -54,9 +55,27 @@ os.path.join(_DEFAULT_CHIP_ROOT, "src/app/zap-templates/zcl/data-model/")) -def StackShutdown(): - certificateAuthorityManager.Shutdown() - builtins.chipStack.Shutdown() +async def execute_test(yaml, runner): + # Executing and validating test + for test_step in yaml.tests: + if not test_step.is_pics_enabled: + continue + test_action = runner.encode(test_step) + if test_action is None: + raise Exception( + f'Failed to encode test step {test_step.label}') + + response = await runner.execute(test_action) + decoded_response = runner.decode(response) + post_processing_result = test_step.post_process_response( + decoded_response) + if not post_processing_result.is_success(): + logging.warning(f"Test step failure in 'test_step.label'") + for entry in post_processing_result.entries: + if entry.state == PostProcessCheckStatus.SUCCESS: + continue + logging.warning("%s: %s", entry.state, entry.message) + raise Exception(f'Test step failed {test_step.label}') @click.command() @@ -118,27 +137,8 @@ def _StackShutDown(): runner = ReplTestRunner( clusters_definitions, certificate_authority_manager, dev_ctrl) - # Executing and validating test - for test_step in yaml.tests: - if not test_step.is_pics_enabled: - continue - test_action = runner.encode(test_step) - # TODO if test_action is None we should see if it is a pseudo cluster. - if test_action is None: - raise Exception( - f'Failed to encode test step {test_step.label}') - - response = runner.execute(test_action) - decoded_response = runner.decode(response) - post_processing_result = test_step.post_process_response( - decoded_response) - if not post_processing_result.is_success(): - logging.warning(f"Test step failure in 'test_step.label'") - for entry in post_processing_result.entries: - if entry.state == PostProcessCheckStatus.SUCCESS: - continue - logging.warning("%s: %s", entry.state, entry.message) - raise Exception(f'Test step failed {test_step.label}') + asyncio.run(execute_test(yaml, runner)) + except Exception: print(traceback.format_exc()) exit(-2) diff --git a/scripts/tests/yaml/relative_importer.py b/scripts/tests/yaml/relative_importer.py index bf1168a8349f86..3893354bb806a9 100644 --- a/scripts/tests/yaml/relative_importer.py +++ b/scripts/tests/yaml/relative_importer.py @@ -21,6 +21,7 @@ os.path.join(os.path.dirname(__file__), '..', '..', '..')) SCRIPT_PATH = os.path.join(DEFAULT_CHIP_ROOT, 'scripts') EXAMPLES_PATH = os.path.join(DEFAULT_CHIP_ROOT, 'examples') +REPL_PATH = os.path.join(DEFAULT_CHIP_ROOT, 'src', 'controller', 'python') try: import matter_idl # noqa: F401 @@ -41,3 +42,8 @@ import matter_placeholder_adapter # noqa: F401 except ModuleNotFoundError: sys.path.append(os.path.join(EXAMPLES_PATH, 'placeholder', 'py_matter_placeholder_adapter')) + +try: + import matter_yamltest_repl_adapter # noqa: F401 +except ModuleNotFoundError: + sys.path.append(os.path.join(REPL_PATH, 'py_matter_yamltest_repl_adapter')) diff --git a/scripts/tests/yaml/runner.py b/scripts/tests/yaml/runner.py index ec1ae340027ebb..9889976cf9e002 100755 --- a/scripts/tests/yaml/runner.py +++ b/scripts/tests/yaml/runner.py @@ -88,6 +88,16 @@ def websocket_runner_options(f): return f +def chip_repl_runner_options(f): + f = click.option('--repl_storage_path', type=str, default='/tmp/repl-storage.json', + help='Path to persistent storage configuration file.')(f) + f = click.option('--commission_on_network_dut', type=bool, default=False, + help='Prior to running test should we try to commission DUT on network.')(f) + f = click.option('--runner', type=str, default=None, show_default=True, + help='The runner to run the test with.')(f) + return f + + @dataclass class ParserGroup: builder_config: TestParserBuilderConfig @@ -202,6 +212,10 @@ def __add_custom_params(self, ctx): 'server_name': 'chip-app2', 'server_arguments': '--interactive', }, + 'chip-repl': { + 'adapter': 'matter_yamltest_repl_adapter.adapter', + 'runner': 'matter_yamltest_repl_adapter.runner', + }, }, max_content_width=120, ) @@ -281,6 +295,21 @@ def websocket(parser_group: ParserGroup, adapter: str, stop_on_error: bool, stop return runner.run(parser_group.builder_config, runner_config) +@runner_base.command() +@test_runner_options +@chip_repl_runner_options +@pass_parser_group +def chip_repl(parser_group: ParserGroup, adapter: str, stop_on_error: bool, stop_on_warning: bool, stop_at_number: int, show_adapter_logs: bool, show_adapter_logs_on_error: bool, runner: str, repl_storage_path: str, commission_on_network_dut: bool): + """Run the test suite using chip-repl.""" + adapter = __import__(adapter, fromlist=[None]).Adapter(parser_group.builder_config.parser_config.definitions) + runner_options = TestRunnerOptions(stop_on_error, stop_on_warning, stop_at_number) + runner_hooks = TestRunnerLogger(show_adapter_logs, show_adapter_logs_on_error) + runner_config = TestRunnerConfig(adapter, parser_group.pseudo_clusters, runner_options, runner_hooks) + + runner = __import__(runner, fromlist=[None]).Runner(repl_storage_path, commission_on_network_dut) + return runner.run(parser_group.builder_config, runner_config) + + @runner_base.command() @test_runner_options @websocket_runner_options diff --git a/src/controller/python/chip/yaml/runner.py b/src/controller/python/chip/yaml/runner.py index 5cf58f16ee94dd..bd5dbf99ded8a7 100644 --- a/src/controller/python/chip/yaml/runner.py +++ b/src/controller/python/chip/yaml/runner.py @@ -15,7 +15,6 @@ # limitations under the License. # -import asyncio as asyncio import logging import queue from abc import ABC, abstractmethod @@ -116,7 +115,7 @@ def pics_enabled(self): return self._pics_enabled @abstractmethod - def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: + async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: pass @@ -127,8 +126,8 @@ def __init__(self, test_step): if not _PSEUDO_CLUSTERS.supports(test_step): raise ActionCreationError(f'Default cluster {test_step.cluster} {test_step.command}, not supported') - def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: - resp = asyncio.run(_PSEUDO_CLUSTERS.execute(self._test_step)) + async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: + resp = await _PSEUDO_CLUSTERS.execute(self._test_step) return _ActionResult(status=_ActionStatus.SUCCESS, response=None) @@ -186,17 +185,17 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext): else: self._request_object = command_object - def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: + async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: try: if self._group_id: resp = dev_ctrl.SendGroupCommand( self._group_id, self._request_object, busyWaitMs=self._busy_wait_ms) else: - resp = asyncio.run(dev_ctrl.SendCommand( + resp = await dev_ctrl.SendCommand( self._node_id, self._endpoint, self._request_object, timedRequestTimeoutMs=self._interation_timeout_ms, - busyWaitMs=self._busy_wait_ms)) + busyWaitMs=self._busy_wait_ms) except chip.interaction_model.InteractionModelError as error: return _ActionResult(status=_ActionStatus.ERROR, response=error) @@ -249,11 +248,11 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext): raise UnexpectedActionCreationError( f'ReadAttribute doesnt have valid attribute_type. {self.label}') - def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: + async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: try: - raw_resp = asyncio.run(dev_ctrl.ReadAttribute(self._node_id, - [(self._endpoint, self._request_object)], - fabricFiltered=self._fabric_filtered)) + raw_resp = await dev_ctrl.ReadAttribute(self._node_id, + [(self._endpoint, self._request_object)], + fabricFiltered=self._fabric_filtered) except chip.interaction_model.InteractionModelError as error: return _ActionResult(status=_ActionStatus.ERROR, response=error) except ChipStackError as error: @@ -317,12 +316,12 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext): raise UnexpectedActionCreationError( f'ReadEvent should not contain arguments. {self.label}') - def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: + async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: try: urgent = 0 request = [(self._endpoint, self._request_object, urgent)] - resp = asyncio.run(dev_ctrl.ReadEvent(self._node_id, events=request, eventNumberFilter=self._event_number_filter, - fabricFiltered=self._fabric_filtered)) + resp = await dev_ctrl.ReadEvent(self._node_id, events=request, eventNumberFilter=self._event_number_filter, + fabricFiltered=self._fabric_filtered) except chip.interaction_model.InteractionModelError as error: return _ActionResult(status=_ActionStatus.ERROR, response=error) @@ -358,7 +357,7 @@ def __init__(self, test_step): # Timeout is provided in seconds we need to conver to milliseconds. self._timeout_ms = request_data_as_dict['timeout'] * 1000 - def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: + async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: try: if self._expire_existing_session: dev_ctrl.ExpireSessions(self._node_id) @@ -437,12 +436,11 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext): f'SubscribeAttribute action does not have max_interval {self.label}') self._max_interval = test_step.max_interval - def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: + async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: try: - subscription = asyncio.run( - dev_ctrl.ReadAttribute(self._node_id, [(self._endpoint, self._request_object)], - reportInterval=(self._min_interval, self._max_interval), - keepSubscriptions=False)) + subscription = await dev_ctrl.ReadAttribute(self._node_id, [(self._endpoint, self._request_object)], + reportInterval=(self._min_interval, self._max_interval), + keepSubscriptions=False) except chip.interaction_model.InteractionModelError as error: return _ActionResult(status=_ActionStatus.ERROR, response=error) @@ -490,14 +488,13 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext): f'SubscribeEvent action does not have max_interval {self.label}') self._max_interval = test_step.max_interval - def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: + async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: try: urgent = 0 request = [(self._endpoint, self._request_object, urgent)] - subscription = asyncio.run( - dev_ctrl.ReadEvent(self._node_id, events=request, eventNumberFilter=self._event_number_filter, - reportInterval=(self._min_interval, self._max_interval), - keepSubscriptions=False)) + subscription = await dev_ctrl.ReadEvent(self._node_id, events=request, eventNumberFilter=self._event_number_filter, + reportInterval=(self._min_interval, self._max_interval), + keepSubscriptions=False) except chip.interaction_model.InteractionModelError as error: return _ActionResult(status=_ActionStatus.ERROR, response=error) @@ -571,16 +568,15 @@ def __init__(self, test_step, cluster: str, context: _ExecutionContext): # Create a cluster object for the request from the provided YAML data. self._request_object = attribute(request_data) - def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: + async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: try: if self._group_id: resp = dev_ctrl.WriteGroupAttribute(self._group_id, [(self._request_object,)], busyWaitMs=self._busy_wait_ms) else: - resp = asyncio.run( - dev_ctrl.WriteAttribute(self._node_id, [(self._endpoint, self._request_object)], - timedRequestTimeoutMs=self._interation_timeout_ms, - busyWaitMs=self._busy_wait_ms)) + resp = await dev_ctrl.WriteAttribute(self._node_id, [(self._endpoint, self._request_object)], + timedRequestTimeoutMs=self._interation_timeout_ms, + busyWaitMs=self._busy_wait_ms) except chip.interaction_model.InteractionModelError as error: return _ActionResult(status=_ActionStatus.ERROR, response=error) @@ -624,7 +620,7 @@ def __init__(self, test_step, context: _ExecutionContext): if self._output_queue is None: raise UnexpectedActionCreationError(f'Could not find output queue') - def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: + async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: try: # While there should be a timeout here provided by the test, the current codegen version # of YAML tests doesn't have a per test step timeout, only a global timeout for the @@ -662,7 +658,7 @@ def __init__(self, test_step): else: raise UnexpectedActionCreationError(f'Unexpected CommisionerCommand {test_step.command}') - def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: + async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: if self._command == 'GetCommissionerNodeId': return _ActionResult(status=_ActionStatus.SUCCESS, response=_GetCommissionerNodeIdResult(dev_ctrl.nodeId)) @@ -713,7 +709,7 @@ def __init__(self, test_step): super().__init__(test_step) self.filterType, self.filter = DiscoveryCommandAction._filter_for_step(test_step) - def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: + async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: devices = dev_ctrl.DiscoverCommissionableNodes( filterType=self.filterType, filter=self.filter, stopOnFirst=True, timeoutSecond=5) @@ -737,7 +733,7 @@ def __init__(self, test_step, cluster, command): self.cluster = cluster self.command = command - def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: + async def run_action(self, dev_ctrl: ChipDeviceController) -> _ActionResult: raise Exception(f"NOT YET IMPLEMENTED: {self.cluster}::{self.command}") @@ -753,7 +749,8 @@ def __init__(self, test_spec_definition, certificate_authority_manager, alpha_de self._certificate_authority_manager = certificate_authority_manager self._dev_ctrls = {} - alpha_dev_ctrl.InitGroupTestingData() + if alpha_dev_ctrl is not None: + alpha_dev_ctrl.InitGroupTestingData() self._dev_ctrls['alpha'] = alpha_dev_ctrl def _invoke_action_factory(self, test_step, cluster: str): @@ -1012,9 +1009,9 @@ def _get_dev_ctrl(self, action: BaseAction): return dev_ctrl - def execute(self, action: BaseAction): + async def execute(self, action: BaseAction): dev_ctrl = self._get_dev_ctrl(action) - return action.run_action(dev_ctrl) + return await action.run_action(dev_ctrl) def shutdown(self): for subscription in self._context.subscriptions: diff --git a/src/controller/python/py_matter_yamltest_repl_adapter/matter_yamltest_repl_adapter/__init__.py b/src/controller/python/py_matter_yamltest_repl_adapter/matter_yamltest_repl_adapter/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/src/controller/python/py_matter_yamltest_repl_adapter/matter_yamltest_repl_adapter/adapter.py b/src/controller/python/py_matter_yamltest_repl_adapter/matter_yamltest_repl_adapter/adapter.py new file mode 100644 index 00000000000000..916a30ea003853 --- /dev/null +++ b/src/controller/python/py_matter_yamltest_repl_adapter/matter_yamltest_repl_adapter/adapter.py @@ -0,0 +1,33 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from chip.yaml.runner import ReplTestRunner +from matter_yamltests.adapter import TestAdapter + + +class Adapter(TestAdapter): + def __init__(self, specifications): + self._adapter = ReplTestRunner(specifications, None, None) + + def encode(self, request): + return self._adapter.encode(request) + + def decode(self, response): + # TODO We should provide more meaningful logs here, but to adhere to + # abstract function definition we do need to return list here. + logs = [] + decoded_response = self._adapter.decode(response) + if len(decoded_response) == 0: + decoded_response = [{}] + return decoded_response, logs diff --git a/src/controller/python/py_matter_yamltest_repl_adapter/matter_yamltest_repl_adapter/runner.py b/src/controller/python/py_matter_yamltest_repl_adapter/matter_yamltest_repl_adapter/runner.py new file mode 100644 index 00000000000000..83974aa09ab1e0 --- /dev/null +++ b/src/controller/python/py_matter_yamltest_repl_adapter/matter_yamltest_repl_adapter/runner.py @@ -0,0 +1,81 @@ +# Copyright (c) 2023 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# isort: off + +from chip import ChipDeviceCtrl # Needed before chip.FabricAdmin +import chip.FabricAdmin # Needed before chip.CertificateAuthority + +# isort: on + +import chip.CertificateAuthority +import chip.logging +import chip.native +from chip.ChipStack import * +from chip.yaml.runner import ReplTestRunner +from matter_yamltests.runner import TestRunner + + +class Runner(TestRunner): + def __init__(self, repl_storage_path: str, commission_on_network_dut: bool): + self._repl_runner = None + self._chip_stack = None + self._certificate_authority_manager = None + self._repl_storage_path = repl_storage_path + self._commission_on_network_dut = commission_on_network_dut + + async def start(self): + chip.native.Init() + chip.logging.RedirectToPythonLogging() + chip_stack = ChipStack(self._repl_storage_path) + certificate_authority_manager = chip.CertificateAuthority.CertificateAuthorityManager( + chip_stack, chip_stack.GetStorageManager()) + certificate_authority_manager.LoadAuthoritiesFromStorage() + + commission_device = False + if len(certificate_authority_manager.activeCaList) == 0: + if self._commission_on_network_dut == False: + raise Exception( + 'Provided repl storage does not contain certificate. Without commission_on_network_dut, there is no reachable DUT') + certificate_authority_manager.NewCertificateAuthority() + commission_device = True + + if len(certificate_authority_manager.activeCaList[0].adminList) == 0: + certificate_authority_manager.activeCaList[0].NewFabricAdmin( + vendorId=0xFFF1, fabricId=1) + + ca_list = certificate_authority_manager.activeCaList + + dev_ctrl = ca_list[0].adminList[0].NewController() + if commission_device: + # These magic values are the defaults expected for YAML tests + dev_ctrl.CommissionWithCode('MT:-24J0AFN00KA0648G00', 0x12344321) + + self._chip_stack = chip_stack + self._certificate_authority_manager = certificate_authority_manager + self._repl_runner = ReplTestRunner(None, certificate_authority_manager, dev_ctrl) + + async def stop(self): + if self._repl_runner: + self._repl_runner.shutdown() + if self._certificate_authority_manager: + self._certificate_authority_manager.Shutdown() + if self._chip_stack: + self._chip_stack.Shutdown() + self._repl_runner = None + self._chip_stack = None + self._certificate_authority_manager = None + + async def execute(self, request): + return await self._repl_runner.execute(request)