From 644008067385aef3b8aade90d8cc62d239abfde2 Mon Sep 17 00:00:00 2001 From: Mathieu Kardous Date: Tue, 6 Aug 2024 14:22:04 -0400 Subject: [PATCH] Add ICDM 5.1 Automated Test Script --- .github/workflows/tests.yaml | 92 ++++++++---- src/app/tests/suites/manualTests.json | 3 +- src/python_testing/TC_ICDM_5_1.py | 202 ++++++++++++++++++++++++++ 3 files changed, 267 insertions(+), 30 deletions(-) create mode 100644 src/python_testing/TC_ICDM_5_1.py diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 70f40efc502234..02c2902edb479b 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -17,13 +17,14 @@ name: Tests on: push: branches-ignore: - - 'dependabot/**' + - "dependabot/**" pull_request: merge_group: workflow_dispatch: concurrency: - group: ${{ github.ref }}-${{ github.workflow }}-${{ (github.event_name == + group: + ${{ github.ref }}-${{ github.workflow }}-${{ (github.event_name == 'pull_request' && github.event.number) || (github.event_name == 'workflow_dispatch' && github.run_number) || github.sha }} cancel-in-progress: true @@ -42,7 +43,9 @@ jobs: env: BUILD_VARIANT: ${{matrix.build_variant}} CHIP_TOOL_VARIANT: ${{matrix.chip_tool}} - TSAN_OPTIONS: "halt_on_error=1 suppressions=scripts/tests/chiptest/tsan-linux-suppressions.txt" + TSAN_OPTIONS: + "halt_on_error=1 + suppressions=scripts/tests/chiptest/tsan-linux-suppressions.txt" LSAN_OPTIONS: detect_leaks=1 if: github.actor != 'restyled-io[bot]' @@ -50,7 +53,8 @@ jobs: container: image: ghcr.io/project-chip/chip-build:54 - options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0 + options: + --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0 net.ipv4.conf.all.forwarding=1 net.ipv6.conf.all.forwarding=1" steps: @@ -68,9 +72,12 @@ jobs: - name: Checkout submodules & Bootstrap uses: ./.github/actions/checkout-submodules-and-bootstrap with: - platform: linux - bootstrap-log-name: bootstrap-logs-linux-${{ matrix.build_variant }}${{ matrix.chip_tool }} - - name: Try to ensure the directories for core dumping exist and we + platform: linux + bootstrap-log-name: + bootstrap-logs-linux-${{ matrix.build_variant }}${{ + matrix.chip_tool }} + - name: + Try to ensure the directories for core dumping exist and we can write them. run: | mkdir /tmp/cores || true @@ -231,7 +238,9 @@ jobs: --copy-artifacts-to objdir-clone \ " - - name: Run Tests using the python parser sending commands to chip-tool + - name: + Run Tests using the python parser sending commands to + chip-tool run: | ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ @@ -252,7 +261,9 @@ jobs: --network-manager-app ./out/linux-x64-network-manager-${BUILD_VARIANT}/matter-network-manager-app \ " - - name: Run purposeful failure tests using the python parser sending commands to chip-tool + - name: + Run purposeful failure tests using the python parser sending + commands to chip-tool run: | ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ @@ -317,7 +328,9 @@ jobs: uses: actions/upload-artifact@v4 if: ${{ failure() && !env.ACT }} with: - name: crash-core-linux-${{ matrix.build_variant }}${{ matrix.chip_tool }} + name: + crash-core-linux-${{ matrix.build_variant }}${{ + matrix.chip_tool }} path: /tmp/cores/ # Cores are big; don't hold on to them too long. retention-days: 5 @@ -325,7 +338,9 @@ jobs: uses: actions/upload-artifact@v4 if: ${{ failure() && !env.ACT }} with: - name: crash-objdir-linux-${{ matrix.build_variant }}${{ matrix.chip_tool }} + name: + crash-objdir-linux-${{ matrix.build_variant }}${{ + matrix.chip_tool }} path: objdir-clone/ # objdirs are big; don't hold on to them too long. retention-days: 5 @@ -341,7 +356,9 @@ jobs: BUILD_VARIANT: ${{matrix.build_variant}} CHIP_TOOL_VARIANT: ${{matrix.chip_tool}} TSAN_OPTIONS: "halt_on_error=1" - LSAN_OPTIONS: detect_leaks=1 suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt + LSAN_OPTIONS: + detect_leaks=1 + suppressions=scripts/tests/chiptest/lsan-mac-suppressions.txt if: github.actor != 'restyled-io[bot]' runs-on: macos-13 @@ -362,9 +379,10 @@ jobs: - name: Checkout submodules & Bootstrap uses: ./.github/actions/checkout-submodules-and-bootstrap with: - platform: darwin - bootstrap-log-name: bootstrap-logs-darwin-${{ matrix.build_variant }}${{ matrix.chip_tool }} - + platform: darwin + bootstrap-log-name: + bootstrap-logs-darwin-${{ matrix.build_variant }}${{ + matrix.chip_tool }} - name: Build Apps run: | @@ -385,7 +403,9 @@ jobs: --copy-artifacts-to objdir-clone \ " - - name: Run Tests using the python parser sending commands to chip-tool + - name: + Run Tests using the python parser sending commands to + chip-tool run: | ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ @@ -407,7 +427,9 @@ jobs: --network-manager-app ./out/darwin-x64-network-manager-${BUILD_VARIANT}/matter-network-manager-app \ " - - name: Run purposeful failure tests using the python parser sending commands to chip-tool + - name: + Run purposeful failure tests using the python parser sending + commands to chip-tool run: | ./scripts/run_in_build_env.sh \ "./scripts/tests/run_test_suite.py \ @@ -426,7 +448,9 @@ jobs: uses: actions/upload-artifact@v4 if: ${{ failure() && !env.ACT }} with: - name: crash-core-darwin-${{ matrix.build_variant }}${{ matrix.chip_tool }} + name: + crash-core-darwin-${{ matrix.build_variant }}${{ + matrix.chip_tool }} path: /cores/ # Cores are big; don't hold on to them too long. retention-days: 5 @@ -434,13 +458,17 @@ jobs: uses: actions/upload-artifact@v4 if: ${{ failure() && !env.ACT }} with: - name: crash-log-darwin-${{ matrix.build_variant }}${{ matrix.chip_tool }} + name: + crash-log-darwin-${{ matrix.build_variant }}${{ + matrix.chip_tool }} path: ~/Library/Logs/DiagnosticReports/ - name: Uploading objdir for debugging uses: actions/upload-artifact@v4 if: ${{ failure() && !env.ACT }} with: - name: crash-objdir-darwin-${{ matrix.build_variant }}${{ matrix.chip_tool }} + name: + crash-objdir-darwin-${{ matrix.build_variant }}${{ + matrix.chip_tool }} path: objdir-clone/ # objdirs are big; don't hold on to them too long. retention-days: 5 @@ -449,14 +477,17 @@ jobs: name: REPL Tests - Linux env: - TSAN_OPTIONS: "halt_on_error=1 suppressions=scripts/tests/chiptest/tsan-linux-suppressions.txt" + TSAN_OPTIONS: + "halt_on_error=1 + suppressions=scripts/tests/chiptest/tsan-linux-suppressions.txt" if: github.actor != 'restyled-io[bot]' runs-on: ubuntu-latest container: image: ghcr.io/project-chip/chip-build:66 - options: --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0 + options: + --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0 net.ipv4.conf.all.forwarding=0 net.ipv6.conf.all.forwarding=0" steps: @@ -465,9 +496,12 @@ jobs: - name: Checkout submodules & Bootstrap uses: ./.github/actions/checkout-submodules-and-bootstrap with: - platform: linux - bootstrap-log-name: bootstrap-logs-linux-${{ matrix.build_variant }}${{ matrix.chip_tool }} - - name: Try to ensure the directories for core dumping exist and we + platform: linux + bootstrap-log-name: + bootstrap-logs-linux-${{ matrix.build_variant }}${{ + matrix.chip_tool }} + - name: + Try to ensure the directories for core dumping exist and we can write them. run: | mkdir /tmp/cores || true @@ -552,6 +586,7 @@ jobs: scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/python_testing/TC_ICDM_2_1.py' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/python_testing/TC_ICDM_3_1.py' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/python_testing/TC_ICDM_3_3.py' + scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/python_testing/TC_ICDM_5_1.py' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/python_testing/TC_ICDManagementCluster.py' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/python_testing/TC_IDM_1_2.py' scripts/run_in_python_env.sh out/venv './scripts/tests/run_python_test.py --load-from-env /tmp/test_env.yaml --script src/python_testing/TC_IDM_1_4.py' @@ -623,7 +658,6 @@ jobs: scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/test_testing/test_TC_SC_7_1.py' scripts/run_in_python_env.sh out/venv 'python3 ./src/python_testing/test_testing/TestDecorators.py' - - name: Uploading core files uses: actions/upload-artifact@v4 if: ${{ failure() && !env.ACT }} @@ -670,8 +704,10 @@ jobs: - name: Checkout submodules & Bootstrap uses: ./.github/actions/checkout-submodules-and-bootstrap with: - platform: darwin - bootstrap-log-name: bootstrap-logs-darwin-${{ matrix.build_variant }}${{ matrix.chip_tool }} + platform: darwin + bootstrap-log-name: + bootstrap-logs-darwin-${{ matrix.build_variant }}${{ + matrix.chip_tool }} - name: Build Python REPL and example apps run: | diff --git a/src/app/tests/suites/manualTests.json b/src/app/tests/suites/manualTests.json index bbb86f7092b715..09ef8e3bfaf9f9 100644 --- a/src/app/tests/suites/manualTests.json +++ b/src/app/tests/suites/manualTests.json @@ -119,8 +119,7 @@ "Identify": ["Test_TC_I_3_2"], "IcdManagement": [ "Test_TC_ICDM_3_2", - "Test_TC_ICDM_4_1", - "Test_TC_ICDM_5_1" + "Test_TC_ICDM_4_1" ], "IlluminanceMeasurement": [], "InteractionDataModel": [ diff --git a/src/python_testing/TC_ICDM_5_1.py b/src/python_testing/TC_ICDM_5_1.py new file mode 100644 index 00000000000000..987bbfbfb4e613 --- /dev/null +++ b/src/python_testing/TC_ICDM_5_1.py @@ -0,0 +1,202 @@ +# +# Copyright (c) 2023 Project CHIP Authors +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# See https://github.com/project-chip/connectedhomeip/blob/master/docs/testing/python.md#defining-the-ci-test-arguments +# for details about the block below. +# +# === BEGIN CI TEST ARGUMENTS === +# test-runner-runs: run1 +# test-runner-run/run1/app: ${LIT_ICD_APP} +# test-runner-run/run1/factoryreset: True +# test-runner-run/run1/quiet: True +# test-runner-run/run1/app-args: --discriminator 1234 --KVS kvs1 --trace-to json:${TRACE_APP}.json +# test-runner-run/run1/script-args: --storage-path admin_storage.json --commissioning-method on-network --discriminator 1234 --passcode 20202021 --PICS src/app/tests/suites/certification/ci-pics-values --trace-to json:${TRACE_TEST_JSON}.json --trace-to perfetto:${TRACE_TEST_PERFETTO}.perfetto +# === END CI TEST ARGUMENTS === + +import logging +from dataclasses import dataclass + +import chip.clusters as Clusters +from matter_testing_support import MatterBaseTest, TestStep, async_test_body, default_matter_test_main +from mobly import asserts +from chip.interaction_model import InteractionModelError, Status +from mdns_discovery import mdns_discovery + +cluster = Clusters.Objects.IcdManagement +commands = cluster.Commands +attributes = cluster.Attributes +modes = cluster.Enums.OperatingModeEnum +ClientTypeEnum = cluster.Enums.ClientTypeEnum + + +@dataclass +class Client: + checkInNodeID: int + subjectId: int + key: bytes + clientType: ClientTypeEnum + + +logger = logging.getLogger(__name__) +kRootEndpointId = 0 + +client1 = Client( + checkInNodeID=1, + subjectId=1, + key=b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + clientType=ClientTypeEnum.kEphemeral +) + + +class TC_ICDM_5_1(MatterBaseTest): + + # + # Class Helper functions + # + + async def _read_icdm_attribute_expect_success(self, attribute): + return await self.read_single_attribute_check_success(endpoint=kRootEndpointId, cluster=cluster, attribute=attribute) + + async def _send_single_icdm_command(self, command): + return await self.send_single_cmd(command, endpoint=kRootEndpointId) + + async def _get_icd_txt_record(self): + discovery = mdns_discovery.MdnsDiscovery() + localCompressedNodeId = hex(self.default_controller.GetCompressedFabricId()) + dutNodeId = hex(self.dut_node_id) + + services = await discovery.get_all_services(log_output=False) + icdTxtRecord = modes.kUnknownEnumValue.value + + for service in services[mdns_discovery.MdnsServiceType.OPERATIONAL.value]: + + # Generate hex for the compressedFabric and NodeId + serviceName = service.instance_name.split("-") + compressedFabric = hex(int(serviceName[0], 16)) + nodeId = hex(int(serviceName[1], 16)) + + # Find the correct MdnsServiceEntry + if localCompressedNodeId == compressedFabric and dutNodeId == nodeId: + icdTxtRecord = service.txt_record['ICD'] + + return icdTxtRecord + + # + # Test Harness Helpers + # + + def desc_TC_ICDM_5_1(self) -> str: + """Returns a description of this test""" + return "[TC-ICDM-5.1] Operating Mode with DUT as Server" + + def steps_TC_ICDM_5_1(self) -> list[TestStep]: + steps = [ + TestStep(0, "Commissioning, already done", is_commissioning=True), + TestStep(1, "TH reads from the DUT the RegisteredClients attribute"), + TestStep("2a", "TH reads from the DUT the OperatingMode attribute."), + TestStep("2b", "Verify that the ICD DNS-SD TXT key is present."), + TestStep("3a", "TH sends RegisterClient command."), + TestStep("3b", "TH reads from the DUT the OperatingMode attribute."), + TestStep("3c", "Verify that mDNS is advertising ICD key."), + TestStep(4, "TH sends UnregisterClient command with CheckInNodeID1."), + TestStep("5a", "TH reads from the DUT the OperatingMode attribute."), + TestStep("5b", "Verify that the ICD DNS-SD TXT key is present."), + ] + return steps + + def pics_TC_ICDM_5_1(self) -> list[str]: + """ This function returns a list of PICS for this test case that must be True for the test to be run""" + pics = [ + "ICDM.S", + "ICDM.S.F02", + ] + return pics + + # + # ICDM 5.1 Test Body + # + + @async_test_body + async def test_TC_ICDM_5_1(self): + + # Commissioning + self.step(0) + + try: + self.step(1) + registeredClients = await self._read_icdm_attribute_expect_success( + attributes.RegisteredClients) + + for client in registeredClients: + try: + await self._send_single_icdm_command(commands.UnregisterClient(checkInNodeID=client.checkInNodeID)) + except InteractionModelError as e: + asserts.assert_equal( + e.status, Status.Success, "Unexpected error returned") + + self.step("2a") + operatingMode = await self._read_icdm_attribute_expect_success(attributes.OperatingMode) + asserts.assert_equal(operatingMode, modes.kSit.value) + + self.step("2b") + icdTxtRecord = await self._get_icd_txt_record() + asserts.assert_equal(int(icdTxtRecord), modes.kSit.value, "OperatingMode Is not in SIT mode.") + + self.step("3a") + try: + await self._send_single_icdm_command(commands.RegisterClient(checkInNodeID=client1.checkInNodeID, monitoredSubject=client1.subjectId, key=client1.key, clientType=client1.clientType)) + except InteractionModelError as e: + asserts.assert_equal( + e.status, Status.Success, "Unexpected error returned") + + self.step("3b") + operatingMode = await self._read_icdm_attribute_expect_success(attributes.OperatingMode) + asserts.assert_equal(operatingMode, modes.kLit.value) + + self.step("3c") + icdTxtRecord = await self._get_icd_txt_record() + asserts.assert_equal(int(icdTxtRecord), modes.kLit.value, "OperatingMode Is not in Lit mode.") + + self.step(4) + try: + await self._send_single_icdm_command(commands.UnregisterClient(checkInNodeID=client1.checkInNodeID)) + except InteractionModelError as e: + asserts.assert_equal( + e.status, Status.Success, "Unexpected error returned") + + self.step("5a") + operatingMode = await self._read_icdm_attribute_expect_success(attributes.OperatingMode) + asserts.assert_equal(operatingMode, modes.kSit.value) + + self.step("5b") + icdTxtRecord = await self._get_icd_txt_record() + asserts.assert_equal(int(icdTxtRecord), modes.kSit.value, "OperatingMode Is not in SIT mode.") + + finally: + registeredClients = await self._read_icdm_attribute_expect_success( + attributes.RegisteredClients) + + for client in registeredClients: + try: + await self._send_single_icdm_command(commands.UnregisterClient(checkInNodeID=client.checkInNodeID)) + except InteractionModelError as e: + asserts.assert_equal( + e.status, Status.Success, "Unexpected error returned") + + +if __name__ == "__main__": + default_matter_test_main()