From d82a873b8ec3e2afcd24b399df19818ca9314d3f Mon Sep 17 00:00:00 2001 From: Cecille Freeman Date: Thu, 20 Jul 2023 21:30:44 -0400 Subject: [PATCH 1/6] Basic comp tests: Update to Test plan style Also add in the missing bits. NOTE: Still missing the topology test - it's getting close to the deadline and I don't want to rush here and make errors. --- .../TC_DeviceBasicComposition.py | 74 +++++++++++++++++-- 1 file changed, 66 insertions(+), 8 deletions(-) diff --git a/src/python_testing/TC_DeviceBasicComposition.py b/src/python_testing/TC_DeviceBasicComposition.py index 1c143723b1c0d1..4496d22950ed97 100644 --- a/src/python_testing/TC_DeviceBasicComposition.py +++ b/src/python_testing/TC_DeviceBasicComposition.py @@ -251,16 +251,47 @@ def fail_current_test(self, msg: Optional[str] = None): asserts.fail(msg) # ======= START OF ACTUAL TESTS ======= - def test_endpoint_zero_present(self): - logging.info("Validating that the Root Node endpoint is present (EP0)") + def test_TC_SM_1_1(self): + self.print_step(1, "Perform a wildcard read of attributes on all endpoints - already done") + self.print_step(2, "Verify that endpoint 0 exists") if 0 not in self.endpoints: self.record_error(self.get_test_name(), location=AttributePathLocation(endpoint_id=0), problem="Did not find Endpoint 0.", spec_location="Endpoint Composition") self.fail_current_test() - def test_descriptor_present_on_each_endpoint(self): - logging.info("Validating each endpoint has a descriptor cluster") + self.print_step(3, "Verify that endpoint 0 descriptor cluster includes the root node device type") + if not Clusters.Descriptor in self.endpoints[0]: + self.record_error(self.get_test_name(), location=AttributePathLocation(endpoint_id=0), + problem="No descriptor cluster on Endpoint 0", spec_location="Root node device type") + self.fail_current_test() + root_dev_type = Clusters.Descriptor.Structs.DeviceTypeStruct(deviceType=0x16, revision=1) + if root_dev_type not in self.endpoints[0][Clusters.Descriptor][Clusters.Descriptor.Attributes.DeviceTypeList]: + self.record_error(self.get_test_name(), location=AttributePathLocation(endpoint_id=0), + problem="Root node device type not listed on endpoint 0", spec_location="Root node device type") + self.fail_current_test() + self.print_step(4, "Verify that the root node device type does not appear in any of the non-zero endpoints") + for endpoint_id, endpoint in self.endpoints.items(): + if endpoint_id == 0: + continue + if root_dev_type in endpoint[Clusters.Descriptor][Clusters.Descriptor.Attributes.DeviceTypeList]: + self.record_error(self.get_test_name(), location=AttributePathLocation(endpoint_id=endpoint_id), + problem=f'Root node device type listed on endpoint {endpoint_id}', spec_location="Root node device type") + self.fail_current_test() + + self.print_step(5, "Verify the existence of all the root node clusters on EP0") + root = self.endpoints[0] + required_clusters = [Clusters.BasicInformation, Clusters.AccessControl, Clusters.GroupKeyManagement, + Clusters.GeneralCommissioning, Clusters.AdministratorCommissioning, Clusters.OperationalCredentials, Clusters.GeneralDiagnostics] + for c in required_clusters: + if c not in root: + self.record_error(self.get_test_name(), location=AttributePathLocation(endpoint_id=0), + problem=f'Root node does not contain required cluster {c}', spec_location="Root node device type") + self.fail_current_test() + + def test_DT_1_1(self): + self.print_step(1, "Perform a wildcard read of attributes on all endpoints - already done") + self.print_step(2, "Verify that each endpoint includes a descriptor cluster") success = True for endpoint_id, endpoint in self.endpoints.items(): has_descriptor = (Clusters.Descriptor in endpoint) @@ -273,8 +304,10 @@ def test_descriptor_present_on_each_endpoint(self): if not success: self.fail_current_test("At least one endpoint was missing the descriptor cluster.") - def test_global_attributes_present_on_each_cluster(self): - logging.info("Validating each cluster has the mandatory global attributes") + def test_IDM_10_1(self): + self.print_step(1, "Perform a wildcard read of attributes on all endpoints - already done") + + self.print_step(2, "Validate all global attributes are present") @dataclass class RequiredMandatoryAttribute: @@ -297,6 +330,7 @@ class RequiredMandatoryAttribute: validator=check_list_of_ints_in_range(0, 0xFFFF_FFFF)), ] + self.print_step(3, "Validate all reported attributes match AttributeList") success = True for endpoint_id, endpoint in self.endpoints_tlv.items(): for cluster_id, cluster in endpoint.items(): @@ -359,13 +393,37 @@ class RequiredMandatoryAttribute: # Warn only for now # TODO: Fail in the future continue + for attribute_id in cluster: + if attribute_id not in attribute_list: + attribute_string = self.cluster_mapper.get_attribute_string(cluster_id, attribute_id) + location = AttributePathLocation(endpoint_id, cluster_id, attribute_id) + self.record_error(self.get_test_name(), location=location, + problem=f'Found attribute {attribute_string} on {location.as_cluster_string(self.cluster_mapper)} not listed in attribute list', spec_location="AttributeList Attribute") + success = False if not success: self.fail_current_test( "At least one cluster was missing a mandatory global attribute or had differences between claimed attributes supported and actual.") - def test_all_attribute_strings_valid(self): - asserts.skip("TODO: Validate every string in the attribute tree is valid UTF-8 and has no nulls") + def test_IDM_11_1(self): + success = True + for endpoint_id, endpoint in self.endpoints_tlv.items(): + for cluster_id, cluster in endpoint.items(): + for attribute_id, attribute in cluster.items(): + if cluster_id not in Clusters.ClusterObjects.ALL_ATTRIBUTES or attribute_id not in Clusters.ClusterObjects.ALL_ATTRIBUTES[cluster_id]: + continue + if Clusters.ClusterObjects.ALL_ATTRIBUTES[cluster_id][attribute_id].attribute_type.Type is not str: + continue + try: + cluster[attribute_id].encode('utf-8', errors='strict') + except UnicodeError: + location = AttributePathLocation(endpoint_id, cluster_id, attribute_id) + attribute_string = self.cluster_mapper.get_attribute_string(cluster_id, attribute_id) + self.record_error(self.get_test_name( + ), location=location, problem=f'Attribute {attribute_string} on {location.as_cluster_string(self.cluster_mapper)} is invalid UTF-8', spec_location="Data types - Character String") + success = False + if not success: + self.fail_current_test("At least one attribute string was not valid UTF-8") def test_all_event_strings_valid(self): asserts.skip("TODO: Validate every string in the read events is valid UTF-8 and has no nulls") From 333d9d5c9e7ad7c6f7f1892f75eb5c8d500f99f8 Mon Sep 17 00:00:00 2001 From: Cecille Freeman Date: Thu, 20 Jul 2023 23:07:20 -0400 Subject: [PATCH 2/6] Device composition: duplicate device types --- .../TC_DeviceBasicComposition.py | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/src/python_testing/TC_DeviceBasicComposition.py b/src/python_testing/TC_DeviceBasicComposition.py index 4496d22950ed97..d038168a04b563 100644 --- a/src/python_testing/TC_DeviceBasicComposition.py +++ b/src/python_testing/TC_DeviceBasicComposition.py @@ -22,6 +22,7 @@ import pathlib import sys from dataclasses import dataclass +from itertools import islice from pprint import pprint from typing import Any, Callable, Optional @@ -167,7 +168,7 @@ async def setup_class(self): dev_ctrl = self.default_controller self.problems = [] - do_test_over_pase = self.user_params.get("use_pase_only", True) + do_test_over_pase = self.user_params.get("use_pase_only", False) dump_device_composition_path: Optional[str] = self.user_params.get("dump_device_composition_path", None) if do_test_over_pase: @@ -448,6 +449,45 @@ def test_all_endpoints_have_valid_composition(self): def test_topology_is_valid(self): asserts.skip("TODO: Make a test that verifies each endpoint only lists direct descendants, except Root Node and Aggregator endpoints that list all their descendants") + def test_dup_feature_flag(self): + device_types = {} + dups = {} + for endpoint_id, endpoint in self.endpoints.items(): + device_types[endpoint_id] = [i.deviceType for i in endpoint[Clusters.Descriptor] + [Clusters.Descriptor.Attributes.DeviceTypeList]] + dups[endpoint_id] = False + + for i, (endpoint_id1, device_types1) in enumerate(device_types.items()): + for endpoint_id2, device_types2 in islice(device_types.items(), i+1, None): + have_dup = list(set(device_types1).intersection(device_types2)) + if have_dup: + dups[endpoint_id1] = True + dups[endpoint_id2] = True + + success = True + for endpoint_id, endpoint in self.endpoints.items(): + if not dups[endpoint_id]: + continue + # All dup endpoints require the taglist feature flag and attribute in the descriptor cluster + feature_flag = endpoint[Clusters.Descriptor][Clusters.Descriptor.Attributes.FeatureMap] + # This should come from the python objects, but I'm not seeing it right now. + if not feature_flag & 0x01: + location = AttributePathLocation(endpoint_id, Clusters.Descriptor.id, + Clusters.Descriptor.Attributes.FeatureMap.attribute_id) + self.record_error(self.get_test_name(), location=location, + problem=f'{endpoint_id} has duplicate device types with another endpoint, but taglist feature flag is not set on the descriptor cluster', spec_location="Base device type") + success = False + # TODO: add check for the attribute, once that lands in the SDK + # if Clusters.Descriptor.Attributes.TagList not in endpoint[Clusters.Descriptor]: + # location = AttributePathLocation(endpoint_id, Clusters.Descriptor.id, + # Clusters.Descriptor.Attributes.TagList.attribute_id) + # self.record_error(self.get_test_name(), location=location, + # problem=f'{endpoint_id} has duplicate device types with another endpoint, but taglist attribute is not present in the descriptor cluster', spec_location="Base device type") + # success = False + + if not success: + self.fail_current_test("At least one descriptor cluster is missing the tag list attribute or feature flag") + if __name__ == "__main__": default_matter_test_main() From 1dd886bb0d8f6a2effda2f31972aebd04387d89c Mon Sep 17 00:00:00 2001 From: Cecille Freeman Date: Fri, 21 Jul 2023 13:17:45 -0400 Subject: [PATCH 3/6] Appease the python overlords --- src/python_testing/TC_DeviceBasicComposition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_testing/TC_DeviceBasicComposition.py b/src/python_testing/TC_DeviceBasicComposition.py index d038168a04b563..cab061451b95fd 100644 --- a/src/python_testing/TC_DeviceBasicComposition.py +++ b/src/python_testing/TC_DeviceBasicComposition.py @@ -261,7 +261,7 @@ def test_TC_SM_1_1(self): self.fail_current_test() self.print_step(3, "Verify that endpoint 0 descriptor cluster includes the root node device type") - if not Clusters.Descriptor in self.endpoints[0]: + if Clusters.Descriptor not in self.endpoints[0]: self.record_error(self.get_test_name(), location=AttributePathLocation(endpoint_id=0), problem="No descriptor cluster on Endpoint 0", spec_location="Root node device type") self.fail_current_test() From 6c612c58577c49f9eb57eb90c4ab342a39637e46 Mon Sep 17 00:00:00 2001 From: Cecille Freeman Date: Fri, 21 Jul 2023 16:58:12 -0400 Subject: [PATCH 4/6] Change back default --- src/python_testing/TC_DeviceBasicComposition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python_testing/TC_DeviceBasicComposition.py b/src/python_testing/TC_DeviceBasicComposition.py index cab061451b95fd..0d26a0bd721562 100644 --- a/src/python_testing/TC_DeviceBasicComposition.py +++ b/src/python_testing/TC_DeviceBasicComposition.py @@ -168,7 +168,7 @@ async def setup_class(self): dev_ctrl = self.default_controller self.problems = [] - do_test_over_pase = self.user_params.get("use_pase_only", False) + do_test_over_pase = self.user_params.get("use_pase_only", True) dump_device_composition_path: Optional[str] = self.user_params.get("dump_device_composition_path", None) if do_test_over_pase: From f22afde745f4dcdf34ea209edfd124b689ab79bd Mon Sep 17 00:00:00 2001 From: Cecille Freeman Date: Fri, 21 Jul 2023 17:24:14 -0400 Subject: [PATCH 5/6] Revert "Device composition: duplicate device types" This reverts commit 333d9d5c9e7ad7c6f7f1892f75eb5c8d500f99f8. --- .../TC_DeviceBasicComposition.py | 40 ------------------- 1 file changed, 40 deletions(-) diff --git a/src/python_testing/TC_DeviceBasicComposition.py b/src/python_testing/TC_DeviceBasicComposition.py index 0d26a0bd721562..1a011fd531d1e4 100644 --- a/src/python_testing/TC_DeviceBasicComposition.py +++ b/src/python_testing/TC_DeviceBasicComposition.py @@ -22,7 +22,6 @@ import pathlib import sys from dataclasses import dataclass -from itertools import islice from pprint import pprint from typing import Any, Callable, Optional @@ -449,45 +448,6 @@ def test_all_endpoints_have_valid_composition(self): def test_topology_is_valid(self): asserts.skip("TODO: Make a test that verifies each endpoint only lists direct descendants, except Root Node and Aggregator endpoints that list all their descendants") - def test_dup_feature_flag(self): - device_types = {} - dups = {} - for endpoint_id, endpoint in self.endpoints.items(): - device_types[endpoint_id] = [i.deviceType for i in endpoint[Clusters.Descriptor] - [Clusters.Descriptor.Attributes.DeviceTypeList]] - dups[endpoint_id] = False - - for i, (endpoint_id1, device_types1) in enumerate(device_types.items()): - for endpoint_id2, device_types2 in islice(device_types.items(), i+1, None): - have_dup = list(set(device_types1).intersection(device_types2)) - if have_dup: - dups[endpoint_id1] = True - dups[endpoint_id2] = True - - success = True - for endpoint_id, endpoint in self.endpoints.items(): - if not dups[endpoint_id]: - continue - # All dup endpoints require the taglist feature flag and attribute in the descriptor cluster - feature_flag = endpoint[Clusters.Descriptor][Clusters.Descriptor.Attributes.FeatureMap] - # This should come from the python objects, but I'm not seeing it right now. - if not feature_flag & 0x01: - location = AttributePathLocation(endpoint_id, Clusters.Descriptor.id, - Clusters.Descriptor.Attributes.FeatureMap.attribute_id) - self.record_error(self.get_test_name(), location=location, - problem=f'{endpoint_id} has duplicate device types with another endpoint, but taglist feature flag is not set on the descriptor cluster', spec_location="Base device type") - success = False - # TODO: add check for the attribute, once that lands in the SDK - # if Clusters.Descriptor.Attributes.TagList not in endpoint[Clusters.Descriptor]: - # location = AttributePathLocation(endpoint_id, Clusters.Descriptor.id, - # Clusters.Descriptor.Attributes.TagList.attribute_id) - # self.record_error(self.get_test_name(), location=location, - # problem=f'{endpoint_id} has duplicate device types with another endpoint, but taglist attribute is not present in the descriptor cluster', spec_location="Base device type") - # success = False - - if not success: - self.fail_current_test("At least one descriptor cluster is missing the tag list attribute or feature flag") - if __name__ == "__main__": default_matter_test_main() From 0b4a78af09ee7e96abeb19ac47b2dba626a1979b Mon Sep 17 00:00:00 2001 From: Cecille Freeman Date: Fri, 21 Jul 2023 17:40:01 -0400 Subject: [PATCH 6/6] Apply suggestions from code review --- src/python_testing/TC_DeviceBasicComposition.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/python_testing/TC_DeviceBasicComposition.py b/src/python_testing/TC_DeviceBasicComposition.py index 1a011fd531d1e4..3b03a05e1a6dba 100644 --- a/src/python_testing/TC_DeviceBasicComposition.py +++ b/src/python_testing/TC_DeviceBasicComposition.py @@ -252,6 +252,7 @@ def fail_current_test(self, msg: Optional[str] = None): # ======= START OF ACTUAL TESTS ======= def test_TC_SM_1_1(self): + ROOT_NODE_DEVICE_TYPE = 0x16 self.print_step(1, "Perform a wildcard read of attributes on all endpoints - already done") self.print_step(2, "Verify that endpoint 0 exists") if 0 not in self.endpoints: @@ -264,8 +265,10 @@ def test_TC_SM_1_1(self): self.record_error(self.get_test_name(), location=AttributePathLocation(endpoint_id=0), problem="No descriptor cluster on Endpoint 0", spec_location="Root node device type") self.fail_current_test() - root_dev_type = Clusters.Descriptor.Structs.DeviceTypeStruct(deviceType=0x16, revision=1) - if root_dev_type not in self.endpoints[0][Clusters.Descriptor][Clusters.Descriptor.Attributes.DeviceTypeList]: + + listed_device_types = [i.deviceType for i in self.endpoints[0] + [Clusters.Descriptor][Clusters.Descriptor.Attributes.DeviceTypeList]] + if ROOT_NODE_DEVICE_TYPE not in listed_device_types: self.record_error(self.get_test_name(), location=AttributePathLocation(endpoint_id=0), problem="Root node device type not listed on endpoint 0", spec_location="Root node device type") self.fail_current_test() @@ -274,7 +277,9 @@ def test_TC_SM_1_1(self): for endpoint_id, endpoint in self.endpoints.items(): if endpoint_id == 0: continue - if root_dev_type in endpoint[Clusters.Descriptor][Clusters.Descriptor.Attributes.DeviceTypeList]: + listed_device_types = [i.deviceType for i in endpoint[Clusters.Descriptor] + [Clusters.Descriptor.Attributes.DeviceTypeList]] + if ROOT_NODE_DEVICE_TYPE in listed_device_types: self.record_error(self.get_test_name(), location=AttributePathLocation(endpoint_id=endpoint_id), problem=f'Root node device type listed on endpoint {endpoint_id}', spec_location="Root node device type") self.fail_current_test()