From b7ab7c6259d342cc9f0cb048bf1accf2e9b0f09a Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley Date: Wed, 24 May 2023 15:58:46 +0100 Subject: [PATCH 01/45] Since jstasiak/zeroconf#666 (0.32.0) it needs a much much smaller monkeypatch to be able to advertise the overlong service type "_nmos-registration._tcp" --- nmostesting/NMOSTesting.py | 2 +- nmostesting/mocks/Auth.py | 2 +- nmostesting/suites/IS0401Test.py | 6 +++++- nmostesting/suites/IS0402Test.py | 2 +- nmostesting/suites/IS0403Test.py | 2 +- nmostesting/suites/IS0902Test.py | 2 +- nmostesting/suites/IS1001Test.py | 2 +- requirements.txt | 2 +- 8 files changed, 12 insertions(+), 8 deletions(-) diff --git a/nmostesting/NMOSTesting.py b/nmostesting/NMOSTesting.py index d5340423..a4d320f9 100644 --- a/nmostesting/NMOSTesting.py +++ b/nmostesting/NMOSTesting.py @@ -58,7 +58,7 @@ from .mocks.Registry import NUM_REGISTRIES, REGISTRIES, REGISTRY_API from .mocks.System import NUM_SYSTEMS, SYSTEMS, SYSTEM_API from .mocks.Auth import AUTH_API, PRIMARY_AUTH, SECONDARY_AUTH -from zeroconf_monkey import Zeroconf +from zeroconf import Zeroconf # Make ANSI escape character sequences (for producing coloured terminal text) work under Windows try: diff --git a/nmostesting/mocks/Auth.py b/nmostesting/mocks/Auth.py index 11b0724b..4b43921f 100644 --- a/nmostesting/mocks/Auth.py +++ b/nmostesting/mocks/Auth.py @@ -29,7 +29,7 @@ from ..Config import PORT_BASE, KEYS_MOCKS, ENABLE_HTTPS, CERT_TRUST_ROOT_CA, JWKS_URI, REDIRECT_URI, SCOPE, CACHE_PATH from ..TestHelper import get_default_ip, get_mocks_hostname, load_resolved_schema, check_content_type from ..IS10Utils import IS10Utils -from zeroconf_monkey import ServiceInfo +from zeroconf import ServiceInfo from enum import Enum from werkzeug.serving import make_server from http import HTTPStatus diff --git a/nmostesting/suites/IS0401Test.py b/nmostesting/suites/IS0401Test.py index 33502f43..64c53e67 100644 --- a/nmostesting/suites/IS0401Test.py +++ b/nmostesting/suites/IS0401Test.py @@ -24,7 +24,7 @@ from copy import deepcopy from collections import defaultdict from pathlib import Path -from zeroconf_monkey import ServiceBrowser, ServiceInfo, Zeroconf +from zeroconf import ServiceBrowser, ServiceInfo, Zeroconf from .. import Config as CONFIG from ..MdnsListener import MdnsListener @@ -32,6 +32,10 @@ from ..IS04Utils import IS04Utils from ..TestHelper import get_default_ip, is_ip_address, load_resolved_schema, check_content_type +# monkey patch zeroconf to allow us to advertise "_nmos-registration._tcp" +from zeroconf import service_type_name +service_type_name.__kwdefaults__['strict'] = False + NODE_API_KEY = "node" RECEIVER_CAPS_KEY = "receiver-caps" CAPS_REGISTER_KEY = "caps-register" diff --git a/nmostesting/suites/IS0402Test.py b/nmostesting/suites/IS0402Test.py index 80a8b84d..6ed57380 100644 --- a/nmostesting/suites/IS0402Test.py +++ b/nmostesting/suites/IS0402Test.py @@ -22,7 +22,7 @@ from time import sleep from jsonschema import ValidationError from urllib.parse import urlparse -from zeroconf_monkey import ServiceBrowser, Zeroconf +from zeroconf import ServiceBrowser, Zeroconf from .. import Config as CONFIG from ..MdnsListener import MdnsListener diff --git a/nmostesting/suites/IS0403Test.py b/nmostesting/suites/IS0403Test.py index 12bfa354..b457f754 100644 --- a/nmostesting/suites/IS0403Test.py +++ b/nmostesting/suites/IS0403Test.py @@ -15,7 +15,7 @@ import time import socket -from zeroconf_monkey import ServiceBrowser, Zeroconf +from zeroconf import ServiceBrowser, Zeroconf from ..MdnsListener import MdnsListener from ..GenericTest import GenericTest, NMOS_WIKI_URL from ..IS04Utils import IS04Utils diff --git a/nmostesting/suites/IS0902Test.py b/nmostesting/suites/IS0902Test.py index 40a984d8..df3e3d40 100644 --- a/nmostesting/suites/IS0902Test.py +++ b/nmostesting/suites/IS0902Test.py @@ -14,7 +14,7 @@ import time import socket -from zeroconf_monkey import ServiceInfo, Zeroconf +from zeroconf import ServiceInfo, Zeroconf from .. import Config as CONFIG from ..MdnsListener import MdnsListener diff --git a/nmostesting/suites/IS1001Test.py b/nmostesting/suites/IS1001Test.py index 0fc527e7..ad208059 100644 --- a/nmostesting/suites/IS1001Test.py +++ b/nmostesting/suites/IS1001Test.py @@ -21,7 +21,7 @@ from ..GenericTest import GenericTest, NMOSTestException, NMOSInitException from .. import Config as CONFIG -from zeroconf_monkey import ServiceBrowser, Zeroconf +from zeroconf import ServiceBrowser, Zeroconf from ..MdnsListener import MdnsListener from ..TestHelper import check_content_type diff --git a/requirements.txt b/requirements.txt index 7f46f0a9..5a51fa50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ flask>=2.0.0 wtforms jsonschema -zeroconf-monkey>=1.0.0 +zeroconf>=0.32.0 requests netifaces gitpython From 858ec155284b2f67adf4941138947bbdc035b1b2 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Tue, 18 Jul 2023 11:01:51 +0100 Subject: [PATCH 02/45] Fixed error reporting --- nmostesting/suites/IS1201Test.py | 34 +++++++++++++++++++------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index b5262abd..ac2ce646 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -200,7 +200,8 @@ def send_command(self, test, command_handle, command_json): parsed_message = json.loads(message) if parsed_message["messageType"] == MessageTypes.CommandResponse: - self._validate_schema(parsed_message, + self._validate_schema(test, + parsed_message, self.schemas["command-response-message"], context="command-response-message: ") if NMOSUtils.compare_api_version(parsed_message["protocolVersion"], @@ -219,7 +220,8 @@ def send_command(self, test, command_handle, command_json): raise NMOSTestException(test.FAIL(response["result"])) results.append(response) if parsed_message["messageType"] == MessageTypes.Error: - self._validate_schema(parsed_message, + self._validate_schema(test, + parsed_message, self.schemas["error-message"], context="error-message: ") raise NMOSTestException(test.FAIL(parsed_message, "https://specs.amwa.tv/is-12/branches/{}" @@ -251,7 +253,8 @@ def get_manager(self, test, class_id_str): class_descriptor = self.classes_descriptors[class_id_str] for value in response["result"]["value"]: - self._validate_schema(value, + self._validate_schema(test, + value, self.datatype_schemas["NcBlockMemberDescriptor"], context="NcBlockMemberDescriptor: ") @@ -324,15 +327,15 @@ def validate_descriptor(self, test, reference, descriptor, context=""): + str(descriptor))) return - def _validate_schema(self, payload, schema, context=""): + def _validate_schema(self, test, payload, schema, context=""): """Delegates to validate_schema. Raises NMOSTestExceptions on error""" try: # Validate the JSON schema is correct self.validate_schema(payload, schema) except ValidationError as e: - raise NMOSTestException(context + "Schema validation error: " + e.message) + raise NMOSTestException(test.FAIL(context + "Schema validation error: " + e.message)) except SchemaError as e: - raise NMOSTestException(context + "Schema error: " + e.message) + raise NMOSTestException(test.FAIL(context + "Schema error: " + e.message)) return @@ -370,7 +373,7 @@ def validate_model_definitions(self, class_manager_oid, property_id, schema_name descriptor = descriptors[key] # Validate the JSON schema is correct - self._validate_schema(descriptor, self.datatype_schemas[schema_name]) + self._validate_schema(test, descriptor, self.datatype_schemas[schema_name]) # Validate the descriptor is correct self.validate_descriptor(test, reference_descriptors[key], descriptor) @@ -641,12 +644,12 @@ def test_12(self, test): expected_status=NcMethodStatus.Readonly, is12_error=False) - def validate_property_type(self, value, type, is_nullable, datatype_schemas, context=""): + def validate_property_type(self, test, value, type, is_nullable, datatype_schemas, context=""): if value is None: if is_nullable: return else: - raise NMOSTestException(context + "Non-nullable property set to null.") + raise NMOSTestException(test.FAIL(context + "Non-nullable property set to null.")) if self.is12_utils.primitive_to_python_type(type): # Special case: if this is a floating point value it @@ -656,9 +659,9 @@ def validate_property_type(self, value, type, is_nullable, datatype_schemas, con return if not isinstance(value, self.is12_utils.primitive_to_python_type(type)): - raise NMOSTestException(context + str(value) + " is not of type " + str(type)) + raise NMOSTestException(test.FAIL(context + str(value) + " is not of type " + str(type))) else: - self._validate_schema(value, datatype_schemas[type], context) + self._validate_schema(test, value, datatype_schemas[type], context) return @@ -678,13 +681,15 @@ def validate_object_properties(self, test, reference_class_descriptor, oid, data # validate property type if class_property['isSequence']: for property_value in response["result"]["value"]: - self.validate_property_type(property_value, + self.validate_property_type(test, + property_value, class_property['typeName'], class_property['isNullable'], datatype_schemas, context=context + class_property["name"] + ": ") else: - self.validate_property_type(response["result"]["value"], + self.validate_property_type(test, + response["result"]["value"], class_property['typeName'], class_property['isNullable'], datatype_schemas, @@ -704,7 +709,8 @@ def validate_block(self, test, block_id, class_descriptors, datatype_schemas, co manager_cache = [] for child_object in response["result"]["value"]: - self._validate_schema(child_object, + self._validate_schema(test, + child_object, datatype_schemas["NcBlockMemberDescriptor"], context="NcBlockMemberDescriptor: ") From 473262941b1d0c16e9f4f73485c081685697836f Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Tue, 18 Jul 2023 14:57:35 +0100 Subject: [PATCH 03/45] Added Set user label test --- nmostesting/suites/IS1201Test.py | 133 ++++++++++++++++++++----------- 1 file changed, 87 insertions(+), 46 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index ac2ce646..82b836ea 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -339,21 +339,42 @@ def _validate_schema(self, test, payload, schema, context=""): return - def get_class_manager_descriptors(self, test, class_manager_oid, property_id): + def _get_property(self, test, oid, property_id): + """Get property from object. Raises NMOSTestException on error""" command_handle = self.get_command_handle() version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) - get_descriptors_command = \ + get_property_command = \ self.is12_utils.create_generic_get_command_JSON(version, command_handle, - class_manager_oid, + oid, property_id) - response = self.send_command(test, command_handle, get_descriptors_command) + response = self.send_command(test, command_handle, get_property_command) + + return response["result"]["value"] + + def _set_property(self, test, oid, property_id, argument): + """Get property from object. Raises NMOSTestException on error""" + command_handle = self.get_command_handle() + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + + set_property_command = \ + self.is12_utils.create_generic_set_command_JSON(version, + command_handle, + oid, + property_id, + argument) + response = self.send_command(test, command_handle, set_property_command) + + return response["result"] + + def get_class_manager_descriptors(self, test, class_manager_oid, property_id): + response = self._get_property(test, class_manager_oid, property_id) # Create descriptor dictionary from response array # Use identity as key if present, otherwise use name def key_lambda(identity, name): return ".".join(map(str, identity)) if identity else name - descriptors = {key_lambda(r.get('identity'), r['name']): r for r in response["result"]["value"]} + descriptors = {key_lambda(r.get('identity'), r['name']): r for r in response} return descriptors @@ -440,18 +461,12 @@ def test_03(self, test): self.create_ncp_socket(test) - command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) - get_role_command = \ - self.is12_utils.create_generic_get_command_JSON(version, - command_handle, - self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['ROLE']) - - response = self.send_command(test, command_handle, get_role_command) + role = self._get_property(test, + self.is12_utils.ROOT_BLOCK_OID, + self.is12_utils.PROPERTY_IDS['NCOBJECT']['ROLE']) - if response["result"]["value"] != "root": - return test.FAIL("Unexpected role in Root Block: " + response["result"]["value"], + if role != "root": + return test.FAIL("Unexpected role in Root Block: " + role, "https://specs.amwa.tv/ms-05-02/branches/{}" "/docs/Blocks.html" .format(self.apis[CONTROL_API_KEY]["spec_branch"])) @@ -666,21 +681,12 @@ def validate_property_type(self, test, value, type, is_nullable, datatype_schema return def validate_object_properties(self, test, reference_class_descriptor, oid, datatype_schemas, context): - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) - for class_property in reference_class_descriptor['properties']: - command_handle = self.get_command_handle() - get_property_command = \ - self.is12_utils.create_generic_get_command_JSON(version, - command_handle, - oid, - class_property['id']) - # get property - response = self.send_command(test, command_handle, get_property_command) + response = self._get_property(test, oid, class_property['id']) # validate property type if class_property['isSequence']: - for property_value in response["result"]["value"]: + for property_value in response: self.validate_property_type(test, property_value, class_property['typeName'], @@ -689,7 +695,7 @@ def validate_object_properties(self, test, reference_class_descriptor, oid, data context=context + class_property["name"] + ": ") else: self.validate_property_type(test, - response["result"]["value"], + response, class_property['typeName'], class_property['isNullable'], datatype_schemas, @@ -767,7 +773,7 @@ def validate_block(self, test, block_id, class_descriptors, datatype_schemas, co context=context + child_object['role'] + ': ') return - def validate_device_model(self, test): + def validate_device_model_properties(self, test): if not self.device_model_validated: self.create_ncp_socket(test) @@ -795,11 +801,11 @@ def validate_device_model(self, test): return def test_13(self, test): - """Validate device model against discovered classes and datatypes""" + """Validate device model properties against discovered classes and datatypes""" # Referencing the Google sheet # MS-05-02 (34) All workers MUST inherit from NcWorker # MS-05-02 (35) All managers MUST inherit from NcManager - self.validate_device_model(test) + self.validate_device_model_properties(test) return test.PASS() @@ -810,7 +816,7 @@ def test_14(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html try: - self.validate_device_model(test) + self.validate_device_model_properties(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -830,7 +836,7 @@ def test_15(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html try: - self.validate_device_model(test) + self.validate_device_model_properties(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -850,7 +856,7 @@ def test_16(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html try: - self.validate_device_model(test) + self.validate_device_model_properties(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -870,7 +876,7 @@ def test_17(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html try: - self.validate_device_model(test) + self.validate_device_model_properties(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -893,21 +899,14 @@ def test_18(self, test): device_manager = self.get_manager(test, DEVICE_MANAGER_CLS_ID) # Check MS-05-02 Version - command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) property_id = self.is12_utils.PROPERTY_IDS['NCDEVICEMANAGER']['NCVERSION'] - get_descriptors_command = \ - self.is12_utils.create_generic_get_command_JSON(version, - command_handle, - device_manager['oid'], - property_id) - response = self.send_command(test, command_handle, get_descriptors_command) + version = self._get_property(test, device_manager['oid'], property_id) - if self.is12_utils.compare_api_version(response['result']['value'], self.apis[MS05_API_KEY]["version"]): + if self.is12_utils.compare_api_version(version, self.apis[MS05_API_KEY]["version"]): return test.FAIL("Unexpected version. Expected: " + self.apis[MS05_API_KEY]["version"] - + ". Actual: " + str(response['result']['value'])) + + ". Actual: " + str(version)) return test.PASS() @@ -922,7 +921,7 @@ def test_19(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html try: - self.validate_device_model(test) + self.validate_device_model_properties(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -937,3 +936,45 @@ def test_19(self, test): return test.UNCLEAR("No non-standard classes found.") return test.PASS() + + def test_20(self, test): + """Set user label on Root Block""" + # Referencing the Google sheet + # MS-05-02 (39) Generic getter and setter + # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#generic-getter-and-setter + + link = "https://specs.amwa.tv/ms-05-02/branches/{}" \ + "/docs/NcObject.html#generic-getter-and-setter" \ + .format(self.apis[MS05_API_KEY]["spec_branch"]) + + # Attempt to set labels + self.create_ncp_socket(test) + + property_id = self.is12_utils.PROPERTY_IDS['NCOBJECT']['USER_LABEL'] + + old_user_label = self._get_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id) + + # Set user label + new_user_label = "NMOS Testing Tool" + self._set_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id, new_user_label) + + # Check user label + label = self._get_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id) + if label != new_user_label: + if label == old_user_label: + return test.FAIL("Unable to set user label", link) + else: + return test.FAIL("Unexpected user label: " + str(label), link) + + # Reset user label + self._set_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id, old_user_label) + + # Check user label + label = self._get_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id) + if label != old_user_label: + if label == new_user_label: + return test.FAIL("Unable to set user label", link) + else: + return test.FAIL("Unexpected user label: " + str(label), link) + + return test.PASS() From 655b4a1e47347833340212220093408cabe91781 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Tue, 18 Jul 2023 16:08:39 +0100 Subject: [PATCH 04/45] Validate touchpoints --- nmostesting/suites/IS1201Test.py | 100 ++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 22 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 82b836ea..a15566c3 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -56,6 +56,10 @@ def set_up_tests(self): self.organization_id_error = False self.organization_id_error_msg = "" self.device_model_validated = False + self.touchpoints_validated = False + self.touchpoints_error = False + self.touchpoints_error_msg = "" + self.oid_cache = [] def tear_down_tests(self): @@ -702,6 +706,52 @@ def validate_object_properties(self, test, reference_class_descriptor, oid, data context=context + class_property["name"] + ": ") return + def check_unique_roles(self, role, role_cache): + """Check role is unique within containing Block""" + if role in role_cache: + self.unique_roles_error = True + else: + role_cache.append(role) + + def check_unique_oid(self, oid): + """Check oid is globally unique""" + if oid in self.oid_cache: + self.unique_oids_error = True + else: + self.oid_cache.append(oid) + + def check_manager(self, class_id, owner, class_descriptors, manager_cache): + """Check manager is singleton and that it inherits from NcManager""" + # detemine the standard base class name + base_id = self.is12_utils.get_base_class_id(class_id) + base_class_name = class_descriptors[base_id]["name"] + + # manager checks + if self.is12_utils.is_manager(class_id): + if owner != self.is12_utils.ROOT_BLOCK_OID: + self.managers_members_root_block_error = True + if base_class_name in manager_cache: + self.managers_are_singletons_error = True + else: + manager_cache.append(base_class_name) + + def check_touchpoints(self, test, oid, datatype_schemas): + """Touchpoint checks""" + touchpoints = self._get_property(test, + oid, + self.is12_utils.PROPERTY_IDS["NCOBJECT"]["TOUCHPOINTS"]) + if touchpoints is not None: + self.touchpoints_validated = True + try: + for touchpoint in touchpoints: + self._validate_schema(test, + touchpoint, + datatype_schemas["NcTouchpointNmos"], + context="NcTouchpointNmos: ") + except NMOSTestException as e: + self.touchpoints_error = True + self.touchpoints_error_msg = e.args[0].detail + def validate_block(self, test, block_id, class_descriptors, datatype_schemas, context=""): command_handle = self.get_command_handle() version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) @@ -720,36 +770,19 @@ def validate_block(self, test, block_id, class_descriptors, datatype_schemas, co datatype_schemas["NcBlockMemberDescriptor"], context="NcBlockMemberDescriptor: ") - # Check role is unique within containing Block - if child_object['role'] in role_cache: - self.unique_roles_error = True - else: - role_cache.append(child_object['role']) + self.check_unique_roles(child_object['role'], role_cache) - # Check oid is globally unique - if child_object['oid'] in self.oid_cache: - self.unique_oids_error = True - else: - self.oid_cache.append(child_object['oid']) + self.check_unique_oid(child_object['oid']) # check for non-standard classes if self.is12_utils.is_non_standard_class(child_object['classId']): self.organization_id_detected = True - # detemine the standard base class name - base_id = self.is12_utils.get_base_class_id(child_object['classId']) - base_class_name = class_descriptors[base_id]["name"] + self.check_manager(child_object['classId'], child_object["owner"], class_descriptors, manager_cache) - class_identifier = ".".join(map(str, child_object['classId'])) + self.check_touchpoints(test, child_object['oid'], datatype_schemas) - # manager checks - if self.is12_utils.is_manager(child_object['classId']): - if child_object["owner"] != self.is12_utils.ROOT_BLOCK_OID: - self.managers_members_root_block_error = True - if base_class_name in manager_cache: - self.managers_are_singletons_error = True - else: - manager_cache.append(base_class_name) + class_identifier = ".".join(map(str, child_object['classId'])) if class_identifier: self.validate_object_properties(test, @@ -978,3 +1011,26 @@ def test_20(self, test): return test.FAIL("Unexpected user label: " + str(label), link) return test.PASS() + + def test_21(self, test): + """Validate touchpoints""" + # Referencing the Google sheet + # MS-05-02 (39) For general NMOS contexts (IS-04, IS-05 and IS-07) the NcTouchpointNmos datatype MUST be used + # which has a resource of type NcTouchpointResourceNmos. + # For IS-08 Audio Channel Mapping the NcTouchpointResourceNmosChannelMapping datatype MUST be used + # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#touchpoints + try: + self.validate_device_model_properties(test) + except NMOSTestException as e: + # Couldn't validate model so can't perform test + return test.UNCLEAR(e.args[0].detail, e.args[0].link) + + if self.touchpoints_error: + return test.FAIL(self.touchpoints_error_msg, + "https://specs.amwa.tv/ms-05-02/branches/{}" + "/docs/NcObject.html#touchpoints" + .format(self.apis[MS05_API_KEY]["spec_branch"])) + + if not self.touchpoints_validated: + return test.UNCLEAR("No Touchpoints found.") + return test.PASS() From fb85eb33eeb3db9f59aff4983b70988fe2394615 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Tue, 18 Jul 2023 16:37:12 +0100 Subject: [PATCH 05/45] Validate NcTouchpointNmos or NcTouchpointNmosChannelMapping depending on contextNamespace --- nmostesting/suites/IS1201Test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index a15566c3..61d516ca 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -744,9 +744,12 @@ def check_touchpoints(self, test, oid, datatype_schemas): self.touchpoints_validated = True try: for touchpoint in touchpoints: + schema = datatype_schemas["NcTouchpointNmos"] \ + if touchpoint["contextNamespace"] == "x-nmos" \ + else datatype_schemas["NcTouchpointNmosChannelMapping"] self._validate_schema(test, touchpoint, - datatype_schemas["NcTouchpointNmos"], + schema, context="NcTouchpointNmos: ") except NMOSTestException as e: self.touchpoints_error = True @@ -971,7 +974,7 @@ def test_19(self, test): return test.PASS() def test_20(self, test): - """Set user label on Root Block""" + """Get/Set properties on Root Block""" # Referencing the Google sheet # MS-05-02 (39) Generic getter and setter # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#generic-getter-and-setter From fa0a51bb70ee2dea3a6be3dad1a64140e6759faa Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Wed, 19 Jul 2023 15:24:21 +0100 Subject: [PATCH 06/45] Sequence command JSON helper functions --- nmostesting/IS12Utils.py | 42 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index e2d0673f..1984d874 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -64,7 +64,11 @@ def protocol_definitions(self): self.METHOD_IDS = { 'NCOBJECT': { 'GENERIC_GET': {'level': 1, 'index': 1}, - 'GENERIC_SET': {'level': 1, 'index': 2} + 'GENERIC_SET': {'level': 1, 'index': 2}, + 'GET_SEQUENCE_ITEM': {'level': 1, 'index': 3}, + 'SET_SEQUENCE_ITEM': {'level': 1, 'index': 4}, + 'ADD_SEQUENCE_ITEM': {'level': 1, 'index': 5}, + 'REMOVE_SEQUENCE_ITEM': {'level': 1, 'index': 6} }, 'NCBLOCK': { 'GET_MEMBERS_DESCRIPTOR': {'level': 2, 'index': 1} @@ -136,6 +140,42 @@ def create_get_member_descriptors_JSON(self, version, handle, oid): self.METHOD_IDS["NCBLOCK"]["GET_MEMBERS_DESCRIPTOR"], {'recurse': False}) + def create_get_sequence_item_command_JSON(self, version, handle, oid, property_id, index): + """Create message that will request the sequence item value given an oid and index""" + + return self.create_command_JSON(version, + handle, + oid, + self.METHOD_IDS["NCOBJECT"]["GET_SEQUENCE_ITEM"], + {'id': property_id, 'index': index}) + + def create_set_sequence_item_command_JSON(self, version, handle, oid, property_id, index, value): + """Create message that will add a sequence item value""" + + return self.create_command_JSON(version, + handle, + oid, + self.METHOD_IDS["NCOBJECT"]["SET_SEQUENCE_ITEM"], + {'id': property_id, 'index': index, 'value': value}) + + def create_add_sequence_item_command_JSON(self, version, handle, oid, property_id, value): + """Create message that will add a sequence item value""" + + return self.create_command_JSON(version, + handle, + oid, + self.METHOD_IDS["NCOBJECT"]["ADD_SEQUENCE_ITEM"], + {'id': property_id, 'value': value}) + + def create_remove_sequence_item_command_JSON(self, version, handle, oid, property_id, index): + """Create message that will request the sequence item value given an oid and index""" + + return self.create_command_JSON(version, + handle, + oid, + self.METHOD_IDS["NCOBJECT"]["REMOVE_SEQUENCE_ITEM"], + {'id': property_id, 'index': index}) + def model_primitive_to_JSON(self, type): """Convert MS-05 primitive type to corresponding JSON type""" From 99354899526127f375184da55bc10bbdb8c7c3ad Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Wed, 19 Jul 2023 15:25:02 +0100 Subject: [PATCH 07/45] Check sequence manipulation functions --- nmostesting/suites/IS1201Test.py | 276 +++++++++++++++++++++++++++---- 1 file changed, 247 insertions(+), 29 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 61d516ca..7cb8219f 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -52,13 +52,13 @@ def set_up_tests(self): self.unique_oids_error = False self.managers_are_singletons_error = False self.managers_members_root_block_error = False - self.organization_id_detected = False - self.organization_id_error = False - self.organization_id_error_msg = "" self.device_model_validated = False - self.touchpoints_validated = False - self.touchpoints_error = False - self.touchpoints_error_msg = "" + self.organization_metadata = {"checked": False, "error": False, "error_msg": ""} + self.touchpoints_metadata = {"checked": False, "error": False, "error_msg": ""} + self.get_sequence_item_metadata = {"checked": False, "error": False, "error_msg": ""} + self.set_sequence_item_metadata = {"checked": False, "error": False, "error_msg": ""} + self.add_sequence_item_metadata = {"checked": False, "error": False, "error_msg": ""} + self.remove_sequence_item_metadata = {"checked": False, "error": False, "error_msg": ""} self.oid_cache = [] @@ -372,6 +372,67 @@ def _set_property(self, test, oid, property_id, argument): return response["result"] + def _get_sequence_item(self, test, oid, property_id, index): + """Get value from sequence property. Raises NMOSTestException on error""" + command_handle = self.get_command_handle() + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + + get_sequence_item_command = \ + self.is12_utils.create_get_sequence_item_command_JSON(version, + command_handle, + oid, + property_id, + index) + response = self.send_command(test, command_handle, get_sequence_item_command) + + return response["result"]["value"] + + def _set_sequence_item(self, test, oid, property_id, index, value): + """Add value to a sequence property. Raises NMOSTestException on error""" + command_handle = self.get_command_handle() + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + + add_sequence_item_command = \ + self.is12_utils.create_set_sequence_item_command_JSON(version, + command_handle, + oid, + property_id, + index, + value) + response = self.send_command(test, command_handle, add_sequence_item_command) + + return response["result"] + + def _add_sequence_item(self, test, oid, property_id, value): + """Add value to a sequence property. Raises NMOSTestException on error""" + command_handle = self.get_command_handle() + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + + add_sequence_item_command = \ + self.is12_utils.create_add_sequence_item_command_JSON(version, + command_handle, + oid, + property_id, + value) + response = self.send_command(test, command_handle, add_sequence_item_command) + + return response["result"] + + def _remove_sequence_item(self, test, oid, property_id, index): + """Get value from sequence property. Raises NMOSTestException on error""" + command_handle = self.get_command_handle() + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + + get_sequence_item_command = \ + self.is12_utils.create_remove_sequence_item_command_JSON(version, + command_handle, + oid, + property_id, + index) + response = self.send_command(test, command_handle, get_sequence_item_command) + + return response["result"] + def get_class_manager_descriptors(self, test, class_manager_oid, property_id): response = self._get_property(test, class_manager_oid, property_id) @@ -684,6 +745,97 @@ def validate_property_type(self, test, value, type, is_nullable, datatype_schema return + def check_get_sequence_item(self, test, oid, sequence_values, property_metadata, context=""): + try: + # GetSequenceItem + self.get_sequence_item_metadata["checked"] = True + sequence_index = 0 + for property_value in sequence_values: + value = self._get_sequence_item(test, oid, property_metadata['id'], sequence_index) + if property_value != value: + self.get_sequence_item_metadata["error"] = True + self.get_sequence_item_metadata["error_msg"] += \ + context + property_metadata["name"] \ + + ": Expected: " + str(property_value) + ", Actual: " + str(value) \ + + " at index " + sequence_index + ", " + sequence_index += 1 + return True + except NMOSTestException as e: + self.get_sequence_item_metadata["error"] = True + self.get_sequence_item_metadata["error_msg"] += \ + context + property_metadata["name"] + ": " + str(e.args[0].detail) + ", " + return False + + def check_add_sequence_item(self, test, oid, property_metadata, sequence_length, context=""): + try: + self.add_sequence_item_metadata["checked"] = True + # Add a value to the end of the sequence + new_item = self._get_sequence_item(test, oid, property_metadata['id'], index=0) + + self._add_sequence_item(test, oid, property_metadata['id'], new_item) + + # check the value + value = self._get_sequence_item(test, oid, property_metadata['id'], index=sequence_length) + if value != new_item: + self.add_sequence_item_metadata["error"] = True + self.add_sequence_item_metadata["error_msg"] += \ + context + property_metadata["name"] \ + + ": Expected: " + str(new_item) + ", Actual: " + str(value) + ", " + return True + except NMOSTestException as e: + self.add_sequence_item_metadata["error"] = True + self.add_sequence_item_metadata["error_msg"] += \ + context + property_metadata["name"] + ": " + str(e.args[0].detail) + ", " + return False + + def check_set_sequence_item(self, test, oid, property_metadata, sequence_length, context=""): + try: + self.set_sequence_item_metadata["checked"] = True + new_value = self._get_sequence_item(test, oid, property_metadata['id'], index=sequence_length - 1) + + # set to another value + self._set_sequence_item(test, oid, property_metadata['id'], index=sequence_length, value=new_value) + + # check the value + value = self._get_sequence_item(test, oid, property_metadata['id'], index=sequence_length) + if value != new_value: + self.set_sequence_item_metadata["error"] = True + self.set_sequence_item_metadata["error_msg"] += \ + context + property_metadata["name"] \ + + ": Expected: " + str(new_value) + ", Actual: " + str(value) + ", " + return True + except NMOSTestException as e: + self.set_sequence_item_metadata["error"] = True + self.set_sequence_item_metadata["error_msg"] += \ + context + property_metadata["name"] + ": " + str(e.args[0].detail) + ", " + return False + + def check_remove_sequence_item(self, test, oid, property_metadata, sequence_length, context=""): + try: + # remove item + self.remove_sequence_item_metadata["checked"] = True + self._remove_sequence_item(test, oid, property_metadata['id'], index=sequence_length) + return True + except NMOSTestException as e: + self.remove_sequence_item_metadata["error"] = True + self.remove_sequence_item_metadata["error_msg"] += \ + context + property_metadata["name"] + ": " + str(e.args[0].detail) + ", " + return False + + def check_sequence_methods(self, test, oid, sequence_values, property_metadata, context=""): + """Check that sequence manipulation methods work correctly""" + self.check_get_sequence_item(test, oid, sequence_values, property_metadata, context) + + if not property_metadata['isReadOnly']: + sequence_length = len(sequence_values) + + if not self.check_add_sequence_item(test, oid, property_metadata, sequence_length, context=context): + return + + self.check_set_sequence_item(test, oid, property_metadata, sequence_length, context=context) + + self.check_remove_sequence_item(test, oid, property_metadata, sequence_length, context) + def validate_object_properties(self, test, reference_class_descriptor, oid, datatype_schemas, context): for class_property in reference_class_descriptor['properties']: response = self._get_property(test, oid, class_property['id']) @@ -697,6 +849,7 @@ def validate_object_properties(self, test, reference_class_descriptor, oid, data class_property['isNullable'], datatype_schemas, context=context + class_property["name"] + ": ") + self.check_sequence_methods(test, oid, response, class_property, context=context) else: self.validate_property_type(test, response, @@ -735,13 +888,13 @@ def check_manager(self, class_id, owner, class_descriptors, manager_cache): else: manager_cache.append(base_class_name) - def check_touchpoints(self, test, oid, datatype_schemas): + def check_touchpoints(self, test, oid, datatype_schemas, context): """Touchpoint checks""" touchpoints = self._get_property(test, oid, self.is12_utils.PROPERTY_IDS["NCOBJECT"]["TOUCHPOINTS"]) if touchpoints is not None: - self.touchpoints_validated = True + self.touchpoints_metadata["checked"] = True try: for touchpoint in touchpoints: schema = datatype_schemas["NcTouchpointNmos"] \ @@ -750,10 +903,10 @@ def check_touchpoints(self, test, oid, datatype_schemas): self._validate_schema(test, touchpoint, schema, - context="NcTouchpointNmos: ") + context=context + schema["title"] + ": ") except NMOSTestException as e: - self.touchpoints_error = True - self.touchpoints_error_msg = e.args[0].detail + self.touchpoints_metadata["error"] = True + self.touchpoints_metadata["error_msg"] = context + str(e.args[0].detail) def validate_block(self, test, block_id, class_descriptors, datatype_schemas, context=""): command_handle = self.get_command_handle() @@ -779,11 +932,12 @@ def validate_block(self, test, block_id, class_descriptors, datatype_schemas, co # check for non-standard classes if self.is12_utils.is_non_standard_class(child_object['classId']): - self.organization_id_detected = True + self.organization_metadata["checked"] = True self.check_manager(child_object['classId'], child_object["owner"], class_descriptors, manager_cache) - self.check_touchpoints(test, child_object['oid'], datatype_schemas) + self.check_touchpoints(test, child_object['oid'], datatype_schemas, + context=context + child_object['role'] + ': ') class_identifier = ".".join(map(str, child_object['classId'])) @@ -795,8 +949,8 @@ def validate_block(self, test, block_id, class_descriptors, datatype_schemas, co context=context + child_object['role'] + ': ') else: # Not a standard or non-standard class - self.organization_id_error = True - self.organization_id_error_msg = child_object['role'] + ': ' \ + self.organization_metadata["error"] = True + self.organization_metadata["error_msg"] = child_object['role'] + ': ' \ + "Non-standard class id does not contain authority key: " \ + str(child_object['classId']) + ". " @@ -809,7 +963,7 @@ def validate_block(self, test, block_id, class_descriptors, datatype_schemas, co context=context + child_object['role'] + ': ') return - def validate_device_model_properties(self, test): + def validate_device_model(self, test): if not self.device_model_validated: self.create_ncp_socket(test) @@ -841,7 +995,7 @@ def test_13(self, test): # Referencing the Google sheet # MS-05-02 (34) All workers MUST inherit from NcWorker # MS-05-02 (35) All managers MUST inherit from NcManager - self.validate_device_model_properties(test) + self.validate_device_model(test) return test.PASS() @@ -852,7 +1006,7 @@ def test_14(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html try: - self.validate_device_model_properties(test) + self.validate_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -872,7 +1026,7 @@ def test_15(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html try: - self.validate_device_model_properties(test) + self.validate_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -892,7 +1046,7 @@ def test_16(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html try: - self.validate_device_model_properties(test) + self.validate_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -912,7 +1066,7 @@ def test_17(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html try: - self.validate_device_model_properties(test) + self.validate_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -957,18 +1111,18 @@ def test_19(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html try: - self.validate_device_model_properties(test) + self.validate_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) - if self.organization_id_error: - return test.FAIL(self.organization_id_error_msg, + if self.organization_metadata["error"]: + return test.FAIL(self.organization_metadata["error_msg"], "https://specs.amwa.tv/ms-05-02/branches/{}" "/docs/Framework.html#ncclassid" .format(self.apis[MS05_API_KEY]["spec_branch"])) - if not self.organization_id_detected: + if not self.organization_metadata["checked"]: return test.UNCLEAR("No non-standard classes found.") return test.PASS() @@ -1023,17 +1177,81 @@ def test_21(self, test): # For IS-08 Audio Channel Mapping the NcTouchpointResourceNmosChannelMapping datatype MUST be used # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#touchpoints try: - self.validate_device_model_properties(test) + self.validate_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) - if self.touchpoints_error: - return test.FAIL(self.touchpoints_error_msg, + if self.touchpoints_metadata["error"]: + return test.FAIL(self.touchpoints_metadata["error_msg"], "https://specs.amwa.tv/ms-05-02/branches/{}" "/docs/NcObject.html#touchpoints" .format(self.apis[MS05_API_KEY]["spec_branch"])) - if not self.touchpoints_validated: + if not self.touchpoints_metadata["checked"]: return test.UNCLEAR("No Touchpoints found.") return test.PASS() + + def test_22(self, test): + """Get Sequence Item""" + try: + self.validate_device_model(test) + except NMOSTestException as e: + # Couldn't validate model so can't perform test + return test.UNCLEAR(e.args[0].detail, e.args[0].link) + + if self.get_sequence_item_metadata["error"]: + return test.FAIL(self.get_sequence_item_metadata["error_msg"]) + + if not self.get_sequence_item_metadata["checked"]: + return test.UNCLEAR("GetSequenceItem not tested.") + + return test.PASS() + + def test_23(self, test): + """Set Sequence Item""" + try: + self.validate_device_model(test) + except NMOSTestException as e: + # Couldn't validate model so can't perform test + return test.UNCLEAR(e.args[0].detail, e.args[0].link) + + if self.set_sequence_item_metadata["error"]: + return test.FAIL(self.set_sequence_item_metadata["error_msg"]) + + if not self.set_sequence_item_metadata["checked"]: + return test.UNCLEAR("SetSequenceItem not tested.") + + return test.PASS() + + def test_24(self, test): + """Add Sequence Item""" + try: + self.validate_device_model(test) + except NMOSTestException as e: + # Couldn't validate model so can't perform test + return test.UNCLEAR(e.args[0].detail, e.args[0].link) + + if self.add_sequence_item_metadata["error"]: + return test.FAIL(self.add_sequence_item_metadata["error_msg"]) + + if not self.add_sequence_item_metadata["checked"]: + return test.UNCLEAR("AddSequenceItem not tested.") + + return test.PASS() + + def test_25(self, test): + """Remove Sequence Item""" + try: + self.validate_device_model(test) + except NMOSTestException as e: + # Couldn't validate model so can't perform test + return test.UNCLEAR(e.args[0].detail, e.args[0].link) + + if self.remove_sequence_item_metadata["error"]: + return test.FAIL(self.remove_sequence_item_metadata["error_msg"]) + + if not self.remove_sequence_item_metadata["checked"]: + return test.UNCLEAR("RemoveSequenceItem not tested.") + + return test.PASS() From 19011066297a6c27548f62f51bf1c01a72fc6ffc Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Thu, 20 Jul 2023 15:07:52 +0100 Subject: [PATCH 08/45] Build device model object tree when validating device model Check FindMembersByRole --- nmostesting/IS12Utils.py | 48 ++++++++++++++++++++++++- nmostesting/suites/IS1201Test.py | 61 +++++++++++++++++++++++++++++--- 2 files changed, 104 insertions(+), 5 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index 1984d874..e4285276 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -71,7 +71,8 @@ def protocol_definitions(self): 'REMOVE_SEQUENCE_ITEM': {'level': 1, 'index': 6} }, 'NCBLOCK': { - 'GET_MEMBERS_DESCRIPTOR': {'level': 2, 'index': 1} + 'GET_MEMBERS_DESCRIPTOR': {'level': 2, 'index': 1}, + 'FIND_MEMBERS_BY_PATH': {'level': 2, 'index': 2} }, 'NCCLASSMANAGER': { 'GET_CONTROL_CLASS': {'level': 3, 'index': 1} @@ -98,6 +99,11 @@ def protocol_definitions(self): } } + self.CLASS_IDS = { + 'NCOBJECT': [1], + 'NCBLOCK': [1, 1] + } + def create_command_JSON(self, version, handle, oid, method_id, arguments): """Create command JSON for generic get of a property""" return { @@ -176,6 +182,15 @@ def create_remove_sequence_item_command_JSON(self, version, handle, oid, propert self.METHOD_IDS["NCOBJECT"]["REMOVE_SEQUENCE_ITEM"], {'id': property_id, 'index': index}) + def create_find_members_by_path_command_JSON(self, version, handle, oid, role_path): + """Create JSON message for FindMembersByPath method from NcBlock""" + + return self.create_command_JSON(version, + handle, + oid, + self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_PATH"], + {'path': role_path}) + def model_primitive_to_JSON(self, type): """Convert MS-05 primitive type to corresponding JSON type""" @@ -302,3 +317,34 @@ def is_block(self, class_id): def is_manager(self, class_id): """ Check class id to determine if this is a manager """ return len(class_id) > 1 and class_id[0] == 1 and class_id[1] == 3 + + +class NcObject(): + def __init__(self, class_id, oid, role): + self.class_id = class_id + self.oid = oid + self.role = role + self.child_objects = {} + + def add_child_object(self, nc_object): + self.child_objects[nc_object.oid] = nc_object + + def get_role_paths(self, root=True): + role_paths = [[self.role]] if not root else [] + for _, child_object in self.child_objects.items(): + child_paths = child_object.get_role_paths(False) + for child_path in child_paths: + role_path = [self.role] if not root else [] + role_path += child_path + role_paths.append(role_path) + return role_paths + + def find_members_by_path(self, role_path): + query_role = role_path[0] + for _, child_object in self.child_objects.items(): + if child_object.role == query_role: + if len(role_path[1:]): + return child_object.find_members_by_path(role_path[1:]) + else: + return child_object + return None diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 7cb8219f..1b18c3fe 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -20,7 +20,7 @@ from ..Config import WS_MESSAGE_TIMEOUT from ..GenericTest import GenericTest, NMOSTestException -from ..IS12Utils import IS12Utils, NcMethodStatus, MessageTypes +from ..IS12Utils import IS12Utils, NcMethodStatus, MessageTypes, NcObject from ..NMOSUtils import NMOSUtils from ..TestHelper import WebsocketWorker, load_resolved_schema from ..TestResult import Test @@ -46,6 +46,7 @@ def __init__(self, apis, **kwargs): self.ncp_websocket = None self.load_reference_resources() self.command_handle = 0 + self.root_block = None def set_up_tests(self): self.unique_roles_error = False @@ -433,6 +434,20 @@ def _remove_sequence_item(self, test, oid, property_id, index): return response["result"] + def _find_members_by_path(self, test, oid, role_path): + """Query members based on role path. Raises NMOSTestException on error""" + command_handle = self.get_command_handle() + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + + get_sequence_item_command = \ + self.is12_utils.create_find_members_by_path_command_JSON(version, + command_handle, + oid, + role_path) + response = self.send_command(test, command_handle, get_sequence_item_command) + + return response["result"]["value"] + def get_class_manager_descriptors(self, test, class_manager_oid, property_id): response = self._get_property(test, class_manager_oid, property_id) @@ -908,7 +923,7 @@ def check_touchpoints(self, test, oid, datatype_schemas, context): self.touchpoints_metadata["error"] = True self.touchpoints_metadata["error_msg"] = context + str(e.args[0].detail) - def validate_block(self, test, block_id, class_descriptors, datatype_schemas, context=""): + def validate_block(self, test, block_id, class_descriptors, datatype_schemas, block, context=""): command_handle = self.get_command_handle() version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) @@ -921,6 +936,8 @@ def validate_block(self, test, block_id, class_descriptors, datatype_schemas, co manager_cache = [] for child_object in response["result"]["value"]: + child_block = NcObject(child_object['classId'], child_object['oid'], child_object['role']) + self._validate_schema(test, child_object, datatype_schemas["NcBlockMemberDescriptor"], @@ -960,7 +977,10 @@ def validate_block(self, test, block_id, class_descriptors, datatype_schemas, co child_object['oid'], class_descriptors, datatype_schemas, + child_block, context=context + child_object['role'] + ': ') + + block.add_child_object(child_block) return def validate_device_model(self, test): @@ -981,13 +1001,15 @@ def validate_device_model(self, test): datatype_descriptors=datatype_descriptors, schema_path=os.path.join(self.apis[CONTROL_API_KEY]["spec_path"], 'APIs/tmp_schemas/')) + self.root_block = NcObject(self.is12_utils.CLASS_IDS["NCBLOCK"], self.is12_utils.ROOT_BLOCK_OID, "root") + self.validate_block(test, self.is12_utils.ROOT_BLOCK_OID, class_descriptors, - datatype_schemas) + datatype_schemas, + self.root_block) self.device_model_validated = True - return def test_13(self, test): @@ -1255,3 +1277,34 @@ def test_25(self, test): return test.UNCLEAR("RemoveSequenceItem not tested.") return test.PASS() + + def test_26(self, test): + """Find member by role""" + try: + self.validate_device_model(test) + except NMOSTestException as e: + # Couldn't validate model so can't perform test + return test.UNCLEAR(e.args[0].detail, e.args[0].link) + + role_paths = self.root_block.get_role_paths() + + for role_path in role_paths: + # Get ground truth data from local device model object tree + child_object = self.root_block.find_members_by_path(role_path) + + queried_members = self._find_members_by_path(test, self.root_block.oid, role_path) + + for queried_member in queried_members: + self._validate_schema(test, + queried_member, + self.datatype_schemas["NcBlockMemberDescriptor"], + context="NcBlockMemberDescriptor: ") + + queried_member_oids = [m['oid'] for m in queried_members] + + if child_object.oid not in queried_member_oids: + return test.FAIL("Unsuccessful attempt to find member by role path: " + str(role_path)) + + if len(queried_members) > 1: + return test.FAIL("Incorrect member found by role path: " + str(role_path)) + return test.PASS() From 3761baba3b30ee6b33de07a2db71063c3e76ba60 Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley <31761158+garethsb@users.noreply.github.com> Date: Wed, 28 Jun 2023 15:07:26 +0100 Subject: [PATCH 09/45] Merge pull request #812 from AMWA-TV/zeroconf-marmoset [IS-04-01] Much smaller zeroconf monkey patch (cherry picked from commit aa58b5ed32e8252249b3345c9b58fb66943cbd9d) --- nmostesting/NMOSTesting.py | 2 +- nmostesting/mocks/Auth.py | 2 +- nmostesting/suites/IS0401Test.py | 6 +++++- nmostesting/suites/IS0402Test.py | 2 +- nmostesting/suites/IS0403Test.py | 2 +- nmostesting/suites/IS0902Test.py | 2 +- nmostesting/suites/IS1001Test.py | 2 +- requirements.txt | 2 +- 8 files changed, 12 insertions(+), 8 deletions(-) diff --git a/nmostesting/NMOSTesting.py b/nmostesting/NMOSTesting.py index 4f1216d4..da89a372 100644 --- a/nmostesting/NMOSTesting.py +++ b/nmostesting/NMOSTesting.py @@ -58,7 +58,7 @@ from .mocks.Registry import NUM_REGISTRIES, REGISTRIES, REGISTRY_API from .mocks.System import NUM_SYSTEMS, SYSTEMS, SYSTEM_API from .mocks.Auth import AUTH_API, PRIMARY_AUTH, SECONDARY_AUTH -from zeroconf_monkey import Zeroconf +from zeroconf import Zeroconf # Make ANSI escape character sequences (for producing coloured terminal text) work under Windows try: diff --git a/nmostesting/mocks/Auth.py b/nmostesting/mocks/Auth.py index 11b0724b..4b43921f 100644 --- a/nmostesting/mocks/Auth.py +++ b/nmostesting/mocks/Auth.py @@ -29,7 +29,7 @@ from ..Config import PORT_BASE, KEYS_MOCKS, ENABLE_HTTPS, CERT_TRUST_ROOT_CA, JWKS_URI, REDIRECT_URI, SCOPE, CACHE_PATH from ..TestHelper import get_default_ip, get_mocks_hostname, load_resolved_schema, check_content_type from ..IS10Utils import IS10Utils -from zeroconf_monkey import ServiceInfo +from zeroconf import ServiceInfo from enum import Enum from werkzeug.serving import make_server from http import HTTPStatus diff --git a/nmostesting/suites/IS0401Test.py b/nmostesting/suites/IS0401Test.py index ec5622fe..1fde8b8c 100644 --- a/nmostesting/suites/IS0401Test.py +++ b/nmostesting/suites/IS0401Test.py @@ -24,7 +24,7 @@ from copy import deepcopy from collections import defaultdict from pathlib import Path -from zeroconf_monkey import ServiceBrowser, ServiceInfo, Zeroconf +from zeroconf import ServiceBrowser, ServiceInfo, Zeroconf from .. import Config as CONFIG from ..MdnsListener import MdnsListener @@ -32,6 +32,10 @@ from ..IS04Utils import IS04Utils from ..TestHelper import get_default_ip, is_ip_address, load_resolved_schema, check_content_type +# monkey patch zeroconf to allow us to advertise "_nmos-registration._tcp" +from zeroconf import service_type_name +service_type_name.__kwdefaults__['strict'] = False + NODE_API_KEY = "node" RECEIVER_CAPS_KEY = "receiver-caps" CAPS_REGISTER_KEY = "caps-register" diff --git a/nmostesting/suites/IS0402Test.py b/nmostesting/suites/IS0402Test.py index 80a8b84d..6ed57380 100644 --- a/nmostesting/suites/IS0402Test.py +++ b/nmostesting/suites/IS0402Test.py @@ -22,7 +22,7 @@ from time import sleep from jsonschema import ValidationError from urllib.parse import urlparse -from zeroconf_monkey import ServiceBrowser, Zeroconf +from zeroconf import ServiceBrowser, Zeroconf from .. import Config as CONFIG from ..MdnsListener import MdnsListener diff --git a/nmostesting/suites/IS0403Test.py b/nmostesting/suites/IS0403Test.py index 12bfa354..b457f754 100644 --- a/nmostesting/suites/IS0403Test.py +++ b/nmostesting/suites/IS0403Test.py @@ -15,7 +15,7 @@ import time import socket -from zeroconf_monkey import ServiceBrowser, Zeroconf +from zeroconf import ServiceBrowser, Zeroconf from ..MdnsListener import MdnsListener from ..GenericTest import GenericTest, NMOS_WIKI_URL from ..IS04Utils import IS04Utils diff --git a/nmostesting/suites/IS0902Test.py b/nmostesting/suites/IS0902Test.py index 40a984d8..df3e3d40 100644 --- a/nmostesting/suites/IS0902Test.py +++ b/nmostesting/suites/IS0902Test.py @@ -14,7 +14,7 @@ import time import socket -from zeroconf_monkey import ServiceInfo, Zeroconf +from zeroconf import ServiceInfo, Zeroconf from .. import Config as CONFIG from ..MdnsListener import MdnsListener diff --git a/nmostesting/suites/IS1001Test.py b/nmostesting/suites/IS1001Test.py index 0fc527e7..ad208059 100644 --- a/nmostesting/suites/IS1001Test.py +++ b/nmostesting/suites/IS1001Test.py @@ -21,7 +21,7 @@ from ..GenericTest import GenericTest, NMOSTestException, NMOSInitException from .. import Config as CONFIG -from zeroconf_monkey import ServiceBrowser, Zeroconf +from zeroconf import ServiceBrowser, Zeroconf from ..MdnsListener import MdnsListener from ..TestHelper import check_content_type diff --git a/requirements.txt b/requirements.txt index 7f46f0a9..5a51fa50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ flask>=2.0.0 wtforms jsonschema -zeroconf-monkey>=1.0.0 +zeroconf>=0.32.0 requests netifaces gitpython From 49f968a46ff225f2f78ee481f9fc8554fa0d9b66 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Fri, 21 Jul 2023 09:48:41 +0100 Subject: [PATCH 10/45] Recursively check each block's role paths --- nmostesting/IS12Utils.py | 8 +++---- nmostesting/suites/IS1201Test.py | 39 ++++++++++++++++++++------------ 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index e4285276..276acfda 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -324,14 +324,14 @@ def __init__(self, class_id, oid, role): self.class_id = class_id self.oid = oid self.role = role - self.child_objects = {} + self.child_objects = [] def add_child_object(self, nc_object): - self.child_objects[nc_object.oid] = nc_object + self.child_objects.append(nc_object) def get_role_paths(self, root=True): role_paths = [[self.role]] if not root else [] - for _, child_object in self.child_objects.items(): + for child_object in self.child_objects: child_paths = child_object.get_role_paths(False) for child_path in child_paths: role_path = [self.role] if not root else [] @@ -341,7 +341,7 @@ def get_role_paths(self, root=True): def find_members_by_path(self, role_path): query_role = role_path[0] - for _, child_object in self.child_objects.items(): + for child_object in self.child_objects: if child_object.role == query_role: if len(role_path[1:]): return child_object.find_members_by_path(role_path[1:]) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 1b18c3fe..3b8ec835 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -1278,21 +1278,19 @@ def test_25(self, test): return test.PASS() - def test_26(self, test): - """Find member by role""" - try: - self.validate_device_model(test) - except NMOSTestException as e: - # Couldn't validate model so can't perform test - return test.UNCLEAR(e.args[0].detail, e.args[0].link) + def do_role_path_test(self, test, block): + # Get ground truth role paths + role_paths = block.get_role_paths() - role_paths = self.root_block.get_role_paths() + for child_object in block.child_objects: + if self.is12_utils.is_block(child_object.class_id): + self.do_role_path_test(test, child_object) for role_path in role_paths: # Get ground truth data from local device model object tree - child_object = self.root_block.find_members_by_path(role_path) + expected_member = block.find_members_by_path(role_path) - queried_members = self._find_members_by_path(test, self.root_block.oid, role_path) + queried_members = self._find_members_by_path(test, block.oid, role_path) for queried_member in queried_members: self._validate_schema(test, @@ -1300,11 +1298,24 @@ def test_26(self, test): self.datatype_schemas["NcBlockMemberDescriptor"], context="NcBlockMemberDescriptor: ") + if len(queried_members) != 1: + raise NMOSTestException(test.FAIL("Incorrect member found by role path: " + str(role_path))) + queried_member_oids = [m['oid'] for m in queried_members] - if child_object.oid not in queried_member_oids: - return test.FAIL("Unsuccessful attempt to find member by role path: " + str(role_path)) + if expected_member.oid not in queried_member_oids: + raise NMOSTestException(test.FAIL("Unsuccessful attempt to find member by role path: " + + str(role_path))) + + def test_26(self, test): + """Find member by role""" + try: + self.validate_device_model(test) + except NMOSTestException as e: + # Couldn't validate model so can't perform test + return test.UNCLEAR(e.args[0].detail, e.args[0].link) + + # Recursively check each block in Device Model + self.do_role_path_test(test, self.root_block) - if len(queried_members) > 1: - return test.FAIL("Incorrect member found by role path: " + str(role_path)) return test.PASS() From 954d123c0ddc432c1064eec128903eb733ff2b85 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Fri, 21 Jul 2023 14:14:53 +0100 Subject: [PATCH 11/45] find members by role test --- nmostesting/IS12Utils.py | 33 +++++++++++- nmostesting/suites/IS1201Test.py | 92 +++++++++++++++++++++++++++++--- 2 files changed, 116 insertions(+), 9 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index 276acfda..e4c973b6 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -72,7 +72,8 @@ def protocol_definitions(self): }, 'NCBLOCK': { 'GET_MEMBERS_DESCRIPTOR': {'level': 2, 'index': 1}, - 'FIND_MEMBERS_BY_PATH': {'level': 2, 'index': 2} + 'FIND_MEMBERS_BY_PATH': {'level': 2, 'index': 2}, + 'FIND_MEMBERS_BY_ROLE': {'level': 2, 'index': 3} }, 'NCCLASSMANAGER': { 'GET_CONTROL_CLASS': {'level': 3, 'index': 1} @@ -191,6 +192,19 @@ def create_find_members_by_path_command_JSON(self, version, handle, oid, role_pa self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_PATH"], {'path': role_path}) + def create_find_members_by_role_command_JSON(self, version, handle, oid, role, + case_sensitive, match_whole_string, recurse): + """Create JSON message for FindMembersByPath method from NcBlock""" + + return self.create_command_JSON(version, + handle, + oid, + self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_ROLE"], + {'role': role, + 'caseSensitive': case_sensitive, + 'matchWholeString': match_whole_string, + 'recurse': recurse}) + def model_primitive_to_JSON(self, type): """Convert MS-05 primitive type to corresponding JSON type""" @@ -348,3 +362,20 @@ def find_members_by_path(self, role_path): else: return child_object return None + + def find_members_by_role(self, role, case_sensitive=False, match_whole_string=False, recurse=False): + def match(query_role, role, case_sensitive, match_whole_string): + if case_sensitive: + return query_role == role if match_whole_string else query_role in role + return query_role.lower() == role.lower() if match_whole_string else query_role.lower() in role.lower() + + query_results = [] + for child_object in self.child_objects: + if match(role, child_object.role, case_sensitive, match_whole_string): + query_results.append(child_object) + if recurse: + query_results += child_object.find_members_by_role(role, + case_sensitive, + match_whole_string, + recurse) + return query_results diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 3b8ec835..2283e56b 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -16,12 +16,12 @@ import os import time +from itertools import product from jsonschema import ValidationError, SchemaError from ..Config import WS_MESSAGE_TIMEOUT from ..GenericTest import GenericTest, NMOSTestException from ..IS12Utils import IS12Utils, NcMethodStatus, MessageTypes, NcObject -from ..NMOSUtils import NMOSUtils from ..TestHelper import WebsocketWorker, load_resolved_schema from ..TestResult import Test @@ -209,7 +209,7 @@ def send_command(self, test, command_handle, command_json): parsed_message, self.schemas["command-response-message"], context="command-response-message: ") - if NMOSUtils.compare_api_version(parsed_message["protocolVersion"], + if IS12Utils.compare_api_version(parsed_message["protocolVersion"], self.apis[CONTROL_API_KEY]["version"]): raise NMOSTestException(test.FAIL("Incorrect protocol version. Expected " + self.apis[CONTROL_API_KEY]["version"] @@ -439,12 +439,29 @@ def _find_members_by_path(self, test, oid, role_path): command_handle = self.get_command_handle() version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) - get_sequence_item_command = \ + find_members_by_path_command = \ self.is12_utils.create_find_members_by_path_command_JSON(version, command_handle, oid, role_path) - response = self.send_command(test, command_handle, get_sequence_item_command) + response = self.send_command(test, command_handle, find_members_by_path_command) + + return response["result"]["value"] + + def _find_members_by_role(self, test, oid, role, case_sensitive, match_whole_string, recurse): + """Query members based on role. Raises NMOSTestException on error""" + command_handle = self.get_command_handle() + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + + find_members_by_role_command = \ + self.is12_utils.create_find_members_by_role_command_JSON(version, + command_handle, + oid, + role, + case_sensitive, + match_whole_string, + recurse) + response = self.send_command(test, command_handle, find_members_by_role_command) return response["result"]["value"] @@ -1279,13 +1296,14 @@ def test_25(self, test): return test.PASS() def do_role_path_test(self, test, block): - # Get ground truth role paths - role_paths = block.get_role_paths() - + # Recurse through the child blocks for child_object in block.child_objects: if self.is12_utils.is_block(child_object.class_id): self.do_role_path_test(test, child_object) + # Get ground truth role paths + role_paths = block.get_role_paths() + for role_path in role_paths: # Get ground truth data from local device model object tree expected_member = block.find_members_by_path(role_path) @@ -1308,7 +1326,7 @@ def do_role_path_test(self, test, block): + str(role_path))) def test_26(self, test): - """Find member by role""" + """Find member by path""" try: self.validate_device_model(test) except NMOSTestException as e: @@ -1319,3 +1337,61 @@ def test_26(self, test): self.do_role_path_test(test, self.root_block) return test.PASS() + + def do_role_test(self, test, block): + # Recurse through the child blocks + for child_object in block.child_objects: + if self.is12_utils.is_block(child_object.class_id): + self.do_role_test(test, child_object) + + role_paths = IS12Utils.sampled_list(block.get_role_paths()) + # Generate every combination of case_sensitive, match_whole_string and recurse + truth_table = IS12Utils.sampled_list(list(product([False, True], repeat=3))) + search_conditions = [] + for state in truth_table: + search_conditions += [{"case_sensitive": state[0], "match_whole_string": state[1], "recurse": state[2]}] + + for role_path in role_paths: + role = role_path[-1] + # Case sensitive role, case insensitive role, CS role substring and CI role substring + query_strings = [role, role.upper(), role[-4:], role[-4:].upper()] + + for condition in search_conditions: + for query_string in query_strings: + # Get ground truth result + expected_results = \ + block.find_members_by_role(query_string, + case_sensitive=condition["case_sensitive"], + match_whole_string=condition["match_whole_string"], + recurse=condition["recurse"]) + actual_results = self._find_members_by_role(test, + block.oid, + query_string, + case_sensitive=condition["case_sensitive"], + match_whole_string=condition["match_whole_string"], + recurse=condition["recurse"]) + + expected_results_oids = [m.oid for m in expected_results] + + if len(actual_results) != len(expected_results): + raise NMOSTestException(test.FAIL("Expected " + + str(len(expected_results)) + + ", but got " + + str(len(actual_results)))) + + for actual_result in actual_results: + if actual_result["oid"] not in expected_results_oids: + raise NMOSTestException(test.FAIL("Unexpected search result. " + str(actual_result))) + + def test_27(self, test): + """Find member by role""" + try: + self.validate_device_model(test) + except NMOSTestException as e: + # Couldn't validate model so can't perform test + return test.UNCLEAR(e.args[0].detail, e.args[0].link) + + # Recursively check each block in Device Model + self.do_role_test(test, self.root_block) + + return test.PASS() From e6b7f5caa1bb8e5e43fd32c3bcc212b257da352a Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Fri, 21 Jul 2023 16:13:22 +0100 Subject: [PATCH 12/45] Find members by class id test --- nmostesting/IS12Utils.py | 36 +++++++++++++- nmostesting/suites/IS1201Test.py | 80 +++++++++++++++++++++++++++++--- 2 files changed, 107 insertions(+), 9 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index e4c973b6..b44286d9 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -73,7 +73,8 @@ def protocol_definitions(self): 'NCBLOCK': { 'GET_MEMBERS_DESCRIPTOR': {'level': 2, 'index': 1}, 'FIND_MEMBERS_BY_PATH': {'level': 2, 'index': 2}, - 'FIND_MEMBERS_BY_ROLE': {'level': 2, 'index': 3} + 'FIND_MEMBERS_BY_ROLE': {'level': 2, 'index': 3}, + 'FIND_MEMBERS_BY_CLASS_ID': {'level': 2, 'index': 4} }, 'NCCLASSMANAGER': { 'GET_CONTROL_CLASS': {'level': 3, 'index': 1} @@ -102,7 +103,11 @@ def protocol_definitions(self): self.CLASS_IDS = { 'NCOBJECT': [1], - 'NCBLOCK': [1, 1] + 'NCBLOCK': [1, 1], + 'NCWORKER': [1, 2], + 'NCMANAGER': [1, 3], + 'NCDEVICEMANAGER': [1, 3, 1], + 'NCCLASSMANAGER': [1, 3, 2] } def create_command_JSON(self, version, handle, oid, method_id, arguments): @@ -205,6 +210,17 @@ def create_find_members_by_role_command_JSON(self, version, handle, oid, role, 'matchWholeString': match_whole_string, 'recurse': recurse}) + def create_find_members_by_class_id_command_JSON(self, version, handle, oid, class_id, include_derived, recurse): + """Create JSON message for FindMembersByClassId method from NcBlock""" + + return self.create_command_JSON(version, + handle, + oid, + self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_CLASS_ID"], + {'id': class_id, + 'includeDerived': include_derived, + 'recurse': recurse}) + def model_primitive_to_JSON(self, type): """Convert MS-05 primitive type to corresponding JSON type""" @@ -379,3 +395,19 @@ def match(query_role, role, case_sensitive, match_whole_string): match_whole_string, recurse) return query_results + + def find_members_by_class_id(self, class_id, include_derived=False, recurse=False): + def match(query_class_id, class_id, include_derived): + if query_class_id == (class_id[:len(query_class_id)] if include_derived else class_id): + return True + return False + + query_results = [] + for child_object in self.child_objects: + if match(class_id, child_object.class_id, include_derived): + query_results.append(child_object) + if recurse: + query_results += child_object.find_members_by_class_id(class_id, + include_derived, + recurse) + return query_results diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 2283e56b..d40ac41c 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -465,6 +465,22 @@ def _find_members_by_role(self, test, oid, role, case_sensitive, match_whole_str return response["result"]["value"] + def _find_members_by_class_id(self, test, oid, class_id, include_derived, recurse): + """Query members based on class id. Raises NMOSTestException on error""" + command_handle = self.get_command_handle() + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + + find_members_by_class_id_command = \ + self.is12_utils.create_find_members_by_class_id_command_JSON(version, + command_handle, + oid, + class_id, + include_derived, + recurse) + response = self.send_command(test, command_handle, find_members_by_class_id_command) + + return response["result"]["value"] + def get_class_manager_descriptors(self, test, class_manager_oid, property_id): response = self._get_property(test, class_manager_oid, property_id) @@ -1295,11 +1311,11 @@ def test_25(self, test): return test.PASS() - def do_role_path_test(self, test, block): + def do_find_member_by_path_test(self, test, block): # Recurse through the child blocks for child_object in block.child_objects: if self.is12_utils.is_block(child_object.class_id): - self.do_role_path_test(test, child_object) + self.do_find_member_by_path_test(test, child_object) # Get ground truth role paths role_paths = block.get_role_paths() @@ -1334,15 +1350,15 @@ def test_26(self, test): return test.UNCLEAR(e.args[0].detail, e.args[0].link) # Recursively check each block in Device Model - self.do_role_path_test(test, self.root_block) + self.do_find_member_by_path_test(test, self.root_block) return test.PASS() - def do_role_test(self, test, block): + def do_find_member_by_role_test(self, test, block): # Recurse through the child blocks for child_object in block.child_objects: if self.is12_utils.is_block(child_object.class_id): - self.do_role_test(test, child_object) + self.do_find_member_by_role_test(test, child_object) role_paths = IS12Utils.sampled_list(block.get_role_paths()) # Generate every combination of case_sensitive, match_whole_string and recurse @@ -1384,7 +1400,7 @@ def do_role_test(self, test, block): raise NMOSTestException(test.FAIL("Unexpected search result. " + str(actual_result))) def test_27(self, test): - """Find member by role""" + """Find members by role""" try: self.validate_device_model(test) except NMOSTestException as e: @@ -1392,6 +1408,56 @@ def test_27(self, test): return test.UNCLEAR(e.args[0].detail, e.args[0].link) # Recursively check each block in Device Model - self.do_role_test(test, self.root_block) + self.do_find_member_by_role_test(test, self.root_block) + + return test.PASS() + + def do_find_members_by_class_id_test(self, test, block): + # Recurse through the child blocks + for child_object in block.child_objects: + if self.is12_utils.is_block(child_object.class_id): + self.do_find_members_by_class_id_test(test, child_object) + + class_ids = [class_id for _, class_id in self.is12_utils.CLASS_IDS.items()] + + truth_table = IS12Utils.sampled_list(list(product([False, True], repeat=2))) + search_conditions = [] + for state in truth_table: + search_conditions += [{"include_derived": state[0], "recurse": state[1]}] + + for class_id in class_ids: + for condition in search_conditions: + # Recursively check each block in Device Model + expected_results = block.find_members_by_class_id(class_id, + condition["include_derived"], + condition["recurse"]) + + actual_results = self._find_members_by_class_id(test, + block.oid, + class_id, + condition["include_derived"], + condition["recurse"]) + + expected_results_oids = [m.oid for m in expected_results] + + if len(actual_results) != len(expected_results): + raise NMOSTestException(test.FAIL("Expected " + + str(len(expected_results)) + + ", but got " + + str(len(actual_results)))) + + for actual_result in actual_results: + if actual_result["oid"] not in expected_results_oids: + raise NMOSTestException(test.FAIL("Unexpected search result. " + str(actual_result))) + + def test_28(self, test): + """Find members by class id""" + try: + self.validate_device_model(test) + except NMOSTestException as e: + # Couldn't validate model so can't perform test + return test.UNCLEAR(e.args[0].detail, e.args[0].link) + + self.do_find_members_by_class_id_test(test, self.root_block) return test.PASS() From 94aaedd602edce7fe92dcca45b929052779f0c76 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Mon, 24 Jul 2023 14:59:31 +0100 Subject: [PATCH 13/45] GetMemberDescriptors test Renumbered tests --- nmostesting/IS12Utils.py | 20 +- nmostesting/suites/IS1201Test.py | 567 +++++++++++++++++-------------- 2 files changed, 335 insertions(+), 252 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index b44286d9..4be3f73c 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -92,6 +92,10 @@ def protocol_definitions(self): 'TOUCHPOINTS': {'level': 1, 'index': 7}, 'RUNTIME_PROPERTY_CONSTRAINTS': {'level': 1, 'index': 8} }, + 'NCBLOCK': { + 'ENABLED': {'level': 2, 'index': 1}, + 'MEMBERS': {'level': 2, 'index': 2} + }, 'NCCLASSMANAGER': { 'CONTROL_CLASSES': {'level': 3, 'index': 1}, 'DATATYPES': {'level': 3, 'index': 2} @@ -143,14 +147,14 @@ def create_generic_set_command_JSON(self, version, handle, oid, property_id, val self.METHOD_IDS["NCOBJECT"]["GENERIC_SET"], {'id': property_id, 'value': value}) - def create_get_member_descriptors_JSON(self, version, handle, oid): + def create_get_member_descriptors_JSON(self, version, handle, oid, recurse): """Create message that will request the member descriptors of the object with the given oid""" return self.create_command_JSON(version, handle, oid, self.METHOD_IDS["NCBLOCK"]["GET_MEMBERS_DESCRIPTOR"], - {'recurse': False}) + {'recurse': recurse}) def create_get_sequence_item_command_JSON(self, version, handle, oid, property_id, index): """Create message that will request the sequence item value given an oid and index""" @@ -355,10 +359,14 @@ def __init__(self, class_id, oid, role): self.oid = oid self.role = role self.child_objects = [] + self.member_descriptors = [] def add_child_object(self, nc_object): self.child_objects.append(nc_object) + def add_member_descriptors(self, member_descriptors): + self.member_descriptors = member_descriptors + def get_role_paths(self, root=True): role_paths = [[self.role]] if not root else [] for child_object in self.child_objects: @@ -369,6 +377,14 @@ def get_role_paths(self, root=True): role_paths.append(role_path) return role_paths + def get_member_descriptors(self, recurse=False): + query_results = [] + query_results += self.member_descriptors + if recurse: + for child_object in self.child_objects: + query_results += child_object.get_member_descriptors(recurse) + return query_results + def find_members_by_path(self, role_path): query_role = role_path[0] for child_object in self.child_objects: diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index d40ac41c..acaa30a7 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -245,19 +245,16 @@ def send_command(self, test, command_handle, command_json): def get_manager(self, test, class_id_str): """Get Manager from Root Block. Returns [Manager]. Raises NMOSTestException on error""" - command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) - get_member_descriptors_command = \ - self.is12_utils.create_get_member_descriptors_JSON(version, command_handle, self.is12_utils.ROOT_BLOCK_OID) - - response = self.send_command(test, command_handle, get_member_descriptors_command) + response = self._get_property(test, + self.is12_utils.ROOT_BLOCK_OID, + self.is12_utils.PROPERTY_IDS['NCBLOCK']['MEMBERS']) manager_found = False manager = None class_descriptor = self.classes_descriptors[class_id_str] - for value in response["result"]["value"]: + for value in response: self._validate_schema(test, value, self.datatype_schemas["NcBlockMemberDescriptor"], @@ -434,6 +431,18 @@ def _remove_sequence_item(self, test, oid, property_id, index): return response["result"] + def _get_member_descriptors(self, test, oid, recurse): + """Get BlockMemberDescritors for this block. Raises NMOSTestException on error""" + command_handle = self.get_command_handle() + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + + get_member_descriptors_command = \ + self.is12_utils.create_get_member_descriptors_JSON(version, command_handle, oid, recurse) + + response = self.send_command(test, command_handle, get_member_descriptors_command) + + return response["result"]["value"] + def _find_members_by_path(self, test, oid, role_path): """Query members based on role path. Raises NMOSTestException on error""" command_handle = self.get_command_handle() @@ -597,181 +606,6 @@ def test_04(self, test): return test.PASS() - def do_error_test(self, test, command_handle, command_json, expected_status=None, is12_error=True): - """Execute command with expected error status.""" - # when expected_status = None checking of the status code is skipped - # check the syntax of the error message according to is12_error - - try: - self.create_ncp_socket(test) - - self.send_command(test, command_handle, command_json) - - return test.FAIL("Error expected") - - except NMOSTestException as e: - error_msg = e.args[0].detail - - # Expecting an error status dictionary - if not isinstance(error_msg, dict): - # It must be some other type of error so re-throw - raise e - - # 'protocolVersion' key is found in IS-12 protocol errors, but not in MS-05-02 errors - if is12_error != ('protocolVersion' in error_msg): - spec = "IS-12 protocol" if is12_error else "MS-05-02" - return test.FAIL(spec + " error expected") - - if not error_msg.get('status'): - return test.FAIL("Command error: " + str(error_msg)) - - if error_msg['status'] == NcMethodStatus.OK: - return test.FAIL("Error not handled. Expected: " + expected_status.name - + " (" + str(expected_status) + ")" - + ", actual: " + NcMethodStatus(error_msg['status']).name - + " (" + str(error_msg['status']) + ")") - - if expected_status and error_msg['status'] != expected_status: - return test.WARNING("Unexpected status. Expected: " + expected_status.name - + " (" + str(expected_status) + ")" - + ", actual: " + NcMethodStatus(error_msg['status']).name - + " (" + str(error_msg['status']) + ")") - - return test.PASS() - - def test_05(self, test): - """IS-12 Protocol Error: Node handles incorrect IS-12 protocol version""" - - command_handle = self.get_command_handle() - # Use incorrect protocol version - version = 'DOES.NOT.EXIST' - command_json = \ - self.is12_utils.create_generic_get_command_JSON(version, - command_handle, - self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) - - return self.do_error_test(test, - command_handle, - command_json, - expected_status=NcMethodStatus.ProtocolVersionError) - - def test_06(self, test): - """IS-12 Protocol Error: Node handles invalid command handle""" - - # Use invalid handle - invalid_command_handle = "NOT A HANDLE" - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) - command_json = \ - self.is12_utils.create_generic_get_command_JSON(version, - invalid_command_handle, - self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) - - return self.do_error_test(test, - invalid_command_handle, - command_json) - - def test_07(self, test): - """IS-12 Protocol Error: Node handles invalid command type""" - command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) - command_json = \ - self.is12_utils.create_generic_get_command_JSON(version, - command_handle, - self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) - # Use invalid message type - command_json['messageType'] = 7 - - return self.do_error_test(test, - command_handle, - command_json) - - def test_08(self, test): - """IS-12 Protocol Error: Node handles invalid JSON""" - command_handle = self.get_command_handle() - # Use invalid JSON - command_json = {'not_a': 'valid_command'} - - return self.do_error_test(test, - command_handle, - command_json) - - def test_09(self, test): - """MS-05-02 Error: Node handles invalid oid""" - - command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) - # Use invalid oid - invalid_oid = 999999999 - command_json = \ - self.is12_utils.create_generic_get_command_JSON(version, - command_handle, - invalid_oid, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) - - return self.do_error_test(test, - command_handle, - command_json, - expected_status=NcMethodStatus.BadOid, - is12_error=False) - - def test_10(self, test): - """MS-05-02 Error: Node handles invalid property identifier""" - command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) - # Use invalid property id - invalid_property_identifier = {'level': 1, 'index': 999} - command_json = \ - self.is12_utils.create_generic_get_command_JSON(version, - command_handle, - self.is12_utils.ROOT_BLOCK_OID, - invalid_property_identifier) - - return self.do_error_test(test, - command_handle, - command_json, - expected_status=NcMethodStatus.PropertyNotImplemented, - is12_error=False) - - def test_11(self, test): - """MS-05-02 Error: Node handles invalid method identifier""" - command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) - command_json = \ - self.is12_utils.create_generic_get_command_JSON(version, - command_handle, - self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) - # Use invalid method id - invalid_method_id = {'level': 1, 'index': 999} - command_json['commands'][0]['methodId'] = invalid_method_id - - return self.do_error_test(test, - command_handle, - command_json, - expected_status=NcMethodStatus.MethodNotImplemented, - is12_error=False) - - def test_12(self, test): - """MS-05-02 Error: Node handles read only error""" - command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) - # Try to set a read only property - command_json = \ - self.is12_utils.create_generic_set_command_JSON(version, - command_handle, - self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['ROLE'], - "ROLE IS READ ONLY") - - return self.do_error_test(test, - command_handle, - command_json, - expected_status=NcMethodStatus.Readonly, - is12_error=False) - def validate_property_type(self, test, value, type, is_nullable, datatype_schemas, context=""): if value is None: if is_nullable: @@ -957,20 +791,14 @@ def check_touchpoints(self, test, oid, datatype_schemas, context): self.touchpoints_metadata["error_msg"] = context + str(e.args[0].detail) def validate_block(self, test, block_id, class_descriptors, datatype_schemas, block, context=""): - command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) - - get_member_descriptors_command = \ - self.is12_utils.create_get_member_descriptors_JSON(version, command_handle, block_id) - - response = self.send_command(test, command_handle, get_member_descriptors_command) + response = self._get_property(test, block_id, self.is12_utils.PROPERTY_IDS['NCBLOCK']['MEMBERS']) role_cache = [] manager_cache = [] - for child_object in response["result"]["value"]: - child_block = NcObject(child_object['classId'], child_object['oid'], child_object['role']) + block.add_member_descriptors(response) + for child_object in response: self._validate_schema(test, child_object, datatype_schemas["NcBlockMemberDescriptor"], @@ -1004,6 +832,8 @@ def validate_block(self, test, block_id, class_descriptors, datatype_schemas, bl + "Non-standard class id does not contain authority key: " \ + str(child_object['classId']) + ". " + child_block = NcObject(child_object['classId'], child_object['oid'], child_object['role']) + # If this child object is a Block, recurse if self.is12_utils.is_block(child_object['classId']): self.validate_block(test, @@ -1045,7 +875,7 @@ def validate_device_model(self, test): self.device_model_validated = True return - def test_13(self, test): + def test_05(self, test): """Validate device model properties against discovered classes and datatypes""" # Referencing the Google sheet # MS-05-02 (34) All workers MUST inherit from NcWorker @@ -1054,7 +884,7 @@ def test_13(self, test): return test.PASS() - def test_14(self, test): + def test_06(self, test): """Device model roles are unique within a containing Block""" # Referencing the Google sheet # MS-05-02 (59) The role of an object MUST be unique within its containing Block. @@ -1074,7 +904,7 @@ def test_14(self, test): return test.PASS() - def test_15(self, test): + def test_07(self, test): """Device model oids are globally unique""" # Referencing the Google sheet # MS-05-02 (60) Object ids (oid property) MUST uniquely identity objects in the device model. @@ -1094,7 +924,7 @@ def test_15(self, test): return test.PASS() - def test_16(self, test): + def test_08(self, test): """Managers must be members of the Root Block""" # Referencing the Google sheet # MS-05-02 (36) All managers MUST always exist as members in the Root Block and have a fixed role. @@ -1114,7 +944,7 @@ def test_16(self, test): return test.PASS() - def test_17(self, test): + def test_09(self, test): """Managers are singletons""" # Referencing the Google sheet # MS-05-02 (63) Managers are singleton (MUST only be instantiated once) classes. @@ -1134,7 +964,7 @@ def test_17(self, test): return test.PASS() - def test_18(self, test): + def test_10(self, test): """Device Manager exists in Root Block""" # Referencing the Google sheet # MS-05-02 (37) A minimal device implementation MUST have a device manager in the Root Block. @@ -1155,7 +985,7 @@ def test_18(self, test): return test.PASS() - def test_19(self, test): + def test_11(self, test): """Non-standard classes contain an authority key""" # Referencing the Google sheet # MS-05-02 (72) Non-standard Classes NcClassId @@ -1182,8 +1012,31 @@ def test_19(self, test): return test.PASS() - def test_20(self, test): - """Get/Set properties on Root Block""" + def test_12(self, test): + """Validate touchpoints""" + # Referencing the Google sheet + # MS-05-02 (39) For general NMOS contexts (IS-04, IS-05 and IS-07) the NcTouchpointNmos datatype MUST be used + # which has a resource of type NcTouchpointResourceNmos. + # For IS-08 Audio Channel Mapping the NcTouchpointResourceNmosChannelMapping datatype MUST be used + # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#touchpoints + try: + self.validate_device_model(test) + except NMOSTestException as e: + # Couldn't validate model so can't perform test + return test.UNCLEAR(e.args[0].detail, e.args[0].link) + + if self.touchpoints_metadata["error"]: + return test.FAIL(self.touchpoints_metadata["error_msg"], + "https://specs.amwa.tv/ms-05-02/branches/{}" + "/docs/NcObject.html#touchpoints" + .format(self.apis[MS05_API_KEY]["spec_branch"])) + + if not self.touchpoints_metadata["checked"]: + return test.UNCLEAR("No Touchpoints found.") + return test.PASS() + + def test_13(self, test): + """NcObject method: Get/Set""" # Referencing the Google sheet # MS-05-02 (39) Generic getter and setter # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#generic-getter-and-setter @@ -1224,31 +1077,8 @@ def test_20(self, test): return test.PASS() - def test_21(self, test): - """Validate touchpoints""" - # Referencing the Google sheet - # MS-05-02 (39) For general NMOS contexts (IS-04, IS-05 and IS-07) the NcTouchpointNmos datatype MUST be used - # which has a resource of type NcTouchpointResourceNmos. - # For IS-08 Audio Channel Mapping the NcTouchpointResourceNmosChannelMapping datatype MUST be used - # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#touchpoints - try: - self.validate_device_model(test) - except NMOSTestException as e: - # Couldn't validate model so can't perform test - return test.UNCLEAR(e.args[0].detail, e.args[0].link) - - if self.touchpoints_metadata["error"]: - return test.FAIL(self.touchpoints_metadata["error_msg"], - "https://specs.amwa.tv/ms-05-02/branches/{}" - "/docs/NcObject.html#touchpoints" - .format(self.apis[MS05_API_KEY]["spec_branch"])) - - if not self.touchpoints_metadata["checked"]: - return test.UNCLEAR("No Touchpoints found.") - return test.PASS() - - def test_22(self, test): - """Get Sequence Item""" + def test_14(self, test): + """NcObject method: GetSequenceItem""" try: self.validate_device_model(test) except NMOSTestException as e: @@ -1263,8 +1093,8 @@ def test_22(self, test): return test.PASS() - def test_23(self, test): - """Set Sequence Item""" + def test_15(self, test): + """NcObject method: SetSequenceItem""" try: self.validate_device_model(test) except NMOSTestException as e: @@ -1279,8 +1109,8 @@ def test_23(self, test): return test.PASS() - def test_24(self, test): - """Add Sequence Item""" + def test_16(self, test): + """NcObject method: AddSequenceItem""" try: self.validate_device_model(test) except NMOSTestException as e: @@ -1295,8 +1125,8 @@ def test_24(self, test): return test.PASS() - def test_25(self, test): - """Remove Sequence Item""" + def test_17(self, test): + """NcObject method: RemoveSequenceItem""" try: self.validate_device_model(test) except NMOSTestException as e: @@ -1311,11 +1141,58 @@ def test_25(self, test): return test.PASS() - def do_find_member_by_path_test(self, test, block): + def do_get_member_descriptors_test(self, test, block, context=""): # Recurse through the child blocks for child_object in block.child_objects: if self.is12_utils.is_block(child_object.class_id): - self.do_find_member_by_path_test(test, child_object) + self.do_get_member_descriptors_test(test, child_object, context + block.role + ": ") + + search_conditions = [{"recurse": True}, {"recurse": False}] + + for search_condition in search_conditions: + expected_members = block.get_member_descriptors(search_condition["recurse"]) + + queried_members = self._get_member_descriptors(test, block.oid, search_condition["recurse"]) + + if len(queried_members) != len(expected_members): + raise NMOSTestException(test.FAIL(context + + block.role + + ": Unexpected number of block members found. Expected: " + + str(len(expected_members)) + ", Actual: " + + str(len(queried_members)))) + + expected_members_oids = [m["oid"] for m in expected_members] + + for queried_member in queried_members: + self._validate_schema(test, + queried_member, + self.datatype_schemas["NcBlockMemberDescriptor"], + context=context + + block.role + + ": NcBlockMemberDescriptor: ") + + if queried_member["oid"] not in expected_members_oids: + raise NMOSTestException(test.FAIL(context + + block.role + + ": Unsuccessful attempt to get member descriptors.")) + + def test_18(self, test): + """NcBlock method: GetMemberDescriptors""" + try: + self.validate_device_model(test) + except NMOSTestException as e: + # Couldn't validate model so can't perform test + return test.UNCLEAR(e.args[0].detail, e.args[0].link) + + self.do_get_member_descriptors_test(test, self.root_block) + + return test.PASS() + + def do_find_member_by_path_test(self, test, block, context=""): + # Recurse through the child blocks + for child_object in block.child_objects: + if self.is12_utils.is_block(child_object.class_id): + self.do_find_member_by_path_test(test, child_object, context + block.role + ": ") # Get ground truth role paths role_paths = block.get_role_paths() @@ -1330,19 +1207,25 @@ def do_find_member_by_path_test(self, test, block): self._validate_schema(test, queried_member, self.datatype_schemas["NcBlockMemberDescriptor"], - context="NcBlockMemberDescriptor: ") + context=context + + block.role + + ": NcBlockMemberDescriptor: ") if len(queried_members) != 1: - raise NMOSTestException(test.FAIL("Incorrect member found by role path: " + str(role_path))) + raise NMOSTestException(test.FAIL(context + + block.role + + ": Incorrect member found by role path: " + str(role_path))) queried_member_oids = [m['oid'] for m in queried_members] if expected_member.oid not in queried_member_oids: - raise NMOSTestException(test.FAIL("Unsuccessful attempt to find member by role path: " + raise NMOSTestException(test.FAIL(context + + block.role + + ": Unsuccessful attempt to find member by role path: " + str(role_path))) - def test_26(self, test): - """Find member by path""" + def test_19(self, test): + """NcBlock method: FindMemberByPath""" try: self.validate_device_model(test) except NMOSTestException as e: @@ -1354,11 +1237,11 @@ def test_26(self, test): return test.PASS() - def do_find_member_by_role_test(self, test, block): + def do_find_member_by_role_test(self, test, block, context=""): # Recurse through the child blocks for child_object in block.child_objects: if self.is12_utils.is_block(child_object.class_id): - self.do_find_member_by_role_test(test, child_object) + self.do_find_member_by_role_test(test, child_object, context + block.role + ": ") role_paths = IS12Utils.sampled_list(block.get_role_paths()) # Generate every combination of case_sensitive, match_whole_string and recurse @@ -1390,17 +1273,22 @@ def do_find_member_by_role_test(self, test, block): expected_results_oids = [m.oid for m in expected_results] if len(actual_results) != len(expected_results): - raise NMOSTestException(test.FAIL("Expected " + raise NMOSTestException(test.FAIL(context + + block.role + + ": Expected " + str(len(expected_results)) + ", but got " + str(len(actual_results)))) for actual_result in actual_results: if actual_result["oid"] not in expected_results_oids: - raise NMOSTestException(test.FAIL("Unexpected search result. " + str(actual_result))) + raise NMOSTestException(test.FAIL(context + + block.role + + ": Unexpected search result. " + + str(actual_result))) - def test_27(self, test): - """Find members by role""" + def test_20(self, test): + """NcBlock method: FindMembersByRole""" try: self.validate_device_model(test) except NMOSTestException as e: @@ -1412,11 +1300,11 @@ def test_27(self, test): return test.PASS() - def do_find_members_by_class_id_test(self, test, block): + def do_find_members_by_class_id_test(self, test, block, context=""): # Recurse through the child blocks for child_object in block.child_objects: if self.is12_utils.is_block(child_object.class_id): - self.do_find_members_by_class_id_test(test, child_object) + self.do_find_members_by_class_id_test(test, child_object, context + block.role + ": ") class_ids = [class_id for _, class_id in self.is12_utils.CLASS_IDS.items()] @@ -1441,17 +1329,21 @@ def do_find_members_by_class_id_test(self, test, block): expected_results_oids = [m.oid for m in expected_results] if len(actual_results) != len(expected_results): - raise NMOSTestException(test.FAIL("Expected " + raise NMOSTestException(test.FAIL(context + + block.role + + ": Expected " + str(len(expected_results)) + ", but got " + str(len(actual_results)))) for actual_result in actual_results: if actual_result["oid"] not in expected_results_oids: - raise NMOSTestException(test.FAIL("Unexpected search result. " + str(actual_result))) + raise NMOSTestException(test.FAIL(context + + block.role + + ": Unexpected search result. " + str(actual_result))) - def test_28(self, test): - """Find members by class id""" + def test_21(self, test): + """NcBlock method: FindMembersByClassId""" try: self.validate_device_model(test) except NMOSTestException as e: @@ -1461,3 +1353,178 @@ def test_28(self, test): self.do_find_members_by_class_id_test(test, self.root_block) return test.PASS() + + def do_error_test(self, test, command_handle, command_json, expected_status=None, is12_error=True): + """Execute command with expected error status.""" + # when expected_status = None checking of the status code is skipped + # check the syntax of the error message according to is12_error + + try: + self.create_ncp_socket(test) + + self.send_command(test, command_handle, command_json) + + return test.FAIL("Error expected") + + except NMOSTestException as e: + error_msg = e.args[0].detail + + # Expecting an error status dictionary + if not isinstance(error_msg, dict): + # It must be some other type of error so re-throw + raise e + + # 'protocolVersion' key is found in IS-12 protocol errors, but not in MS-05-02 errors + if is12_error != ('protocolVersion' in error_msg): + spec = "IS-12 protocol" if is12_error else "MS-05-02" + return test.FAIL(spec + " error expected") + + if not error_msg.get('status'): + return test.FAIL("Command error: " + str(error_msg)) + + if error_msg['status'] == NcMethodStatus.OK: + return test.FAIL("Error not handled. Expected: " + expected_status.name + + " (" + str(expected_status) + ")" + + ", actual: " + NcMethodStatus(error_msg['status']).name + + " (" + str(error_msg['status']) + ")") + + if expected_status and error_msg['status'] != expected_status: + return test.WARNING("Unexpected status. Expected: " + expected_status.name + + " (" + str(expected_status) + ")" + + ", actual: " + NcMethodStatus(error_msg['status']).name + + " (" + str(error_msg['status']) + ")") + + return test.PASS() + + def test_22(self, test): + """IS-12 Protocol Error: Node handles incorrect IS-12 protocol version""" + + command_handle = self.get_command_handle() + # Use incorrect protocol version + version = 'DOES.NOT.EXIST' + command_json = \ + self.is12_utils.create_generic_get_command_JSON(version, + command_handle, + self.is12_utils.ROOT_BLOCK_OID, + self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) + + return self.do_error_test(test, + command_handle, + command_json, + expected_status=NcMethodStatus.ProtocolVersionError) + + def test_23(self, test): + """IS-12 Protocol Error: Node handles invalid command handle""" + + # Use invalid handle + invalid_command_handle = "NOT A HANDLE" + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + command_json = \ + self.is12_utils.create_generic_get_command_JSON(version, + invalid_command_handle, + self.is12_utils.ROOT_BLOCK_OID, + self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) + + return self.do_error_test(test, + invalid_command_handle, + command_json) + + def test_24(self, test): + """IS-12 Protocol Error: Node handles invalid command type""" + command_handle = self.get_command_handle() + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + command_json = \ + self.is12_utils.create_generic_get_command_JSON(version, + command_handle, + self.is12_utils.ROOT_BLOCK_OID, + self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) + # Use invalid message type + command_json['messageType'] = 7 + + return self.do_error_test(test, + command_handle, + command_json) + + def test_25(self, test): + """IS-12 Protocol Error: Node handles invalid JSON""" + command_handle = self.get_command_handle() + # Use invalid JSON + command_json = {'not_a': 'valid_command'} + + return self.do_error_test(test, + command_handle, + command_json) + + def test_26(self, test): + """MS-05-02 Error: Node handles invalid oid""" + + command_handle = self.get_command_handle() + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + # Use invalid oid + invalid_oid = 999999999 + command_json = \ + self.is12_utils.create_generic_get_command_JSON(version, + command_handle, + invalid_oid, + self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) + + return self.do_error_test(test, + command_handle, + command_json, + expected_status=NcMethodStatus.BadOid, + is12_error=False) + + def test_27(self, test): + """MS-05-02 Error: Node handles invalid property identifier""" + command_handle = self.get_command_handle() + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + # Use invalid property id + invalid_property_identifier = {'level': 1, 'index': 999} + command_json = \ + self.is12_utils.create_generic_get_command_JSON(version, + command_handle, + self.is12_utils.ROOT_BLOCK_OID, + invalid_property_identifier) + + return self.do_error_test(test, + command_handle, + command_json, + expected_status=NcMethodStatus.PropertyNotImplemented, + is12_error=False) + + def test_28(self, test): + """MS-05-02 Error: Node handles invalid method identifier""" + command_handle = self.get_command_handle() + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + command_json = \ + self.is12_utils.create_generic_get_command_JSON(version, + command_handle, + self.is12_utils.ROOT_BLOCK_OID, + self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) + # Use invalid method id + invalid_method_id = {'level': 1, 'index': 999} + command_json['commands'][0]['methodId'] = invalid_method_id + + return self.do_error_test(test, + command_handle, + command_json, + expected_status=NcMethodStatus.MethodNotImplemented, + is12_error=False) + + def test_29(self, test): + """MS-05-02 Error: Node handles read only error""" + command_handle = self.get_command_handle() + version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) + # Try to set a read only property + command_json = \ + self.is12_utils.create_generic_set_command_JSON(version, + command_handle, + self.is12_utils.ROOT_BLOCK_OID, + self.is12_utils.PROPERTY_IDS['NCOBJECT']['ROLE'], + "ROLE IS READ ONLY") + + return self.do_error_test(test, + command_handle, + command_json, + expected_status=NcMethodStatus.Readonly, + is12_error=False) From d1890c66390e6d1a7acc878a561f5a6dbc452dcc Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Thu, 27 Jul 2023 10:39:07 +0100 Subject: [PATCH 14/45] removed protocolVersion changed id and identity to classId --- nmostesting/IS12Utils.py | 64 ++++++----------- nmostesting/suites/IS1201Test.py | 120 +++++++------------------------ 2 files changed, 50 insertions(+), 134 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index 4be3f73c..52882841 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -36,6 +36,7 @@ class NcMethodStatus(IntEnum): InvalidRequest = 406 Conflict = 409 BufferOverflow = 413 + IndexOutOfBounds = 414 ParameterError = 417 Locked = 423 DeviceError = 500 @@ -43,7 +44,6 @@ class NcMethodStatus(IntEnum): PropertyNotImplemented = 502 NotReady = 503 Timeout = 504 - ProtocolVersionError = 505 class NcDatatypeType(IntEnum): @@ -114,10 +114,9 @@ def protocol_definitions(self): 'NCCLASSMANAGER': [1, 3, 2] } - def create_command_JSON(self, version, handle, oid, method_id, arguments): + def create_command_JSON(self, handle, oid, method_id, arguments): """Create command JSON for generic get of a property""" return { - 'protocolVersion': version, 'messageType': MessageTypes.Command, 'commands': [ { @@ -129,84 +128,75 @@ def create_command_JSON(self, version, handle, oid, method_id, arguments): ], } - def create_generic_get_command_JSON(self, version, handle, oid, property_id): + def create_generic_get_command_JSON(self, handle, oid, property_id): """Create command JSON for generic get of a property""" - return self.create_command_JSON(version, - handle, + return self.create_command_JSON(handle, oid, self.METHOD_IDS["NCOBJECT"]["GENERIC_GET"], {'id': property_id}) - def create_generic_set_command_JSON(self, version, handle, oid, property_id, value): + def create_generic_set_command_JSON(self, handle, oid, property_id, value): """Create command JSON for generic get of a property""" - return self.create_command_JSON(version, - handle, + return self.create_command_JSON(handle, oid, self.METHOD_IDS["NCOBJECT"]["GENERIC_SET"], {'id': property_id, 'value': value}) - def create_get_member_descriptors_JSON(self, version, handle, oid, recurse): + def create_get_member_descriptors_JSON(self, handle, oid, recurse): """Create message that will request the member descriptors of the object with the given oid""" - return self.create_command_JSON(version, - handle, + return self.create_command_JSON(handle, oid, self.METHOD_IDS["NCBLOCK"]["GET_MEMBERS_DESCRIPTOR"], {'recurse': recurse}) - def create_get_sequence_item_command_JSON(self, version, handle, oid, property_id, index): + def create_get_sequence_item_command_JSON(self, handle, oid, property_id, index): """Create message that will request the sequence item value given an oid and index""" - return self.create_command_JSON(version, - handle, + return self.create_command_JSON(handle, oid, self.METHOD_IDS["NCOBJECT"]["GET_SEQUENCE_ITEM"], {'id': property_id, 'index': index}) - def create_set_sequence_item_command_JSON(self, version, handle, oid, property_id, index, value): + def create_set_sequence_item_command_JSON(self, handle, oid, property_id, index, value): """Create message that will add a sequence item value""" - return self.create_command_JSON(version, - handle, + return self.create_command_JSON(handle, oid, self.METHOD_IDS["NCOBJECT"]["SET_SEQUENCE_ITEM"], {'id': property_id, 'index': index, 'value': value}) - def create_add_sequence_item_command_JSON(self, version, handle, oid, property_id, value): + def create_add_sequence_item_command_JSON(self, handle, oid, property_id, value): """Create message that will add a sequence item value""" - return self.create_command_JSON(version, - handle, + return self.create_command_JSON(handle, oid, self.METHOD_IDS["NCOBJECT"]["ADD_SEQUENCE_ITEM"], {'id': property_id, 'value': value}) - def create_remove_sequence_item_command_JSON(self, version, handle, oid, property_id, index): + def create_remove_sequence_item_command_JSON(self, handle, oid, property_id, index): """Create message that will request the sequence item value given an oid and index""" - return self.create_command_JSON(version, - handle, + return self.create_command_JSON(handle, oid, self.METHOD_IDS["NCOBJECT"]["REMOVE_SEQUENCE_ITEM"], {'id': property_id, 'index': index}) - def create_find_members_by_path_command_JSON(self, version, handle, oid, role_path): + def create_find_members_by_path_command_JSON(self, handle, oid, role_path): """Create JSON message for FindMembersByPath method from NcBlock""" - return self.create_command_JSON(version, - handle, + return self.create_command_JSON(handle, oid, self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_PATH"], {'path': role_path}) - def create_find_members_by_role_command_JSON(self, version, handle, oid, role, + def create_find_members_by_role_command_JSON(self, handle, oid, role, case_sensitive, match_whole_string, recurse): """Create JSON message for FindMembersByPath method from NcBlock""" - return self.create_command_JSON(version, - handle, + return self.create_command_JSON(handle, oid, self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_ROLE"], {'role': role, @@ -214,14 +204,13 @@ def create_find_members_by_role_command_JSON(self, version, handle, oid, role, 'matchWholeString': match_whole_string, 'recurse': recurse}) - def create_find_members_by_class_id_command_JSON(self, version, handle, oid, class_id, include_derived, recurse): + def create_find_members_by_class_id_command_JSON(self, handle, oid, class_id, include_derived, recurse): """Create JSON message for FindMembersByClassId method from NcBlock""" - return self.create_command_JSON(version, - handle, + return self.create_command_JSON(handle, oid, self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_CLASS_ID"], - {'id': class_id, + {'classId': class_id, 'includeDerived': include_derived, 'recurse': recurse}) @@ -328,13 +317,6 @@ def descriptor_to_schema(self, descriptor): return json_schema - def format_version(self, version): - """ Formats the spec version to create IS-12 protocol version""" - # Currently IS-12 version format is inconsistant with other IS specs - # this helper converts from spec version to protocol version - # e.g. v1.0 ==> 1.0.0 - return version.strip('v') + ".0" - def get_base_class_id(self, class_id): """ Given a class_id returns the standard base class id as a string""" return '.'.join([str(v) for v in takewhile(lambda x: x > 0, class_id)]) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index acaa30a7..ac8386ff 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -177,7 +177,7 @@ def get_command_handle(self): def send_command(self, test, command_handle, command_json): """Send command to Node under test. Returns [command response]. Raises NMOSTestException on error""" # Referencing the Google sheet - # IS-12 (9) Check protocol version and message type + # IS-12 (9) Check message type # IS-12 (10) Check handle numeric identifier # https://specs.amwa.tv/is-12/branches/v1.0-dev/docs/Protocol_messaging.html # IS-12 (11) Check Command message type @@ -209,15 +209,6 @@ def send_command(self, test, command_handle, command_json): parsed_message, self.schemas["command-response-message"], context="command-response-message: ") - if IS12Utils.compare_api_version(parsed_message["protocolVersion"], - self.apis[CONTROL_API_KEY]["version"]): - raise NMOSTestException(test.FAIL("Incorrect protocol version. Expected " - + self.apis[CONTROL_API_KEY]["version"] - + ", received " + parsed_message["protocolVersion"], - "https://specs.amwa.tv/is-12/branches/{}" - "/docs/Protocol_messaging.html" - .format(self.apis[CONTROL_API_KEY]["spec_branch"]))) - responses = parsed_message["responses"] for response in responses: if response["handle"] == command_handle: @@ -260,7 +251,7 @@ def get_manager(self, test, class_id_str): self.datatype_schemas["NcBlockMemberDescriptor"], context="NcBlockMemberDescriptor: ") - if value["classId"] == class_descriptor["identity"]: + if value["classId"] == class_descriptor["classId"]: manager_found = True manager = value @@ -308,7 +299,7 @@ def validate_descriptor(self, test, reference, descriptor, context=""): if key in non_normative_keys: continue # Check for class ID - if key == 'identity' and isinstance(reference[key], list): + if key == 'classId' and isinstance(reference[key], list): if reference[key] != descriptor[key]: raise NMOSTestException(test.FAIL(context + "Unexpected ClassId. Expected: " + str(reference[key]) @@ -344,11 +335,9 @@ def _validate_schema(self, test, payload, schema, context=""): def _get_property(self, test, oid, property_id): """Get property from object. Raises NMOSTestException on error""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) get_property_command = \ - self.is12_utils.create_generic_get_command_JSON(version, - command_handle, + self.is12_utils.create_generic_get_command_JSON(command_handle, oid, property_id) response = self.send_command(test, command_handle, get_property_command) @@ -358,11 +347,9 @@ def _get_property(self, test, oid, property_id): def _set_property(self, test, oid, property_id, argument): """Get property from object. Raises NMOSTestException on error""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) set_property_command = \ - self.is12_utils.create_generic_set_command_JSON(version, - command_handle, + self.is12_utils.create_generic_set_command_JSON(command_handle, oid, property_id, argument) @@ -373,11 +360,9 @@ def _set_property(self, test, oid, property_id, argument): def _get_sequence_item(self, test, oid, property_id, index): """Get value from sequence property. Raises NMOSTestException on error""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) get_sequence_item_command = \ - self.is12_utils.create_get_sequence_item_command_JSON(version, - command_handle, + self.is12_utils.create_get_sequence_item_command_JSON(command_handle, oid, property_id, index) @@ -388,11 +373,9 @@ def _get_sequence_item(self, test, oid, property_id, index): def _set_sequence_item(self, test, oid, property_id, index, value): """Add value to a sequence property. Raises NMOSTestException on error""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) add_sequence_item_command = \ - self.is12_utils.create_set_sequence_item_command_JSON(version, - command_handle, + self.is12_utils.create_set_sequence_item_command_JSON(command_handle, oid, property_id, index, @@ -404,11 +387,9 @@ def _set_sequence_item(self, test, oid, property_id, index, value): def _add_sequence_item(self, test, oid, property_id, value): """Add value to a sequence property. Raises NMOSTestException on error""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) add_sequence_item_command = \ - self.is12_utils.create_add_sequence_item_command_JSON(version, - command_handle, + self.is12_utils.create_add_sequence_item_command_JSON(command_handle, oid, property_id, value) @@ -419,11 +400,9 @@ def _add_sequence_item(self, test, oid, property_id, value): def _remove_sequence_item(self, test, oid, property_id, index): """Get value from sequence property. Raises NMOSTestException on error""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) get_sequence_item_command = \ - self.is12_utils.create_remove_sequence_item_command_JSON(version, - command_handle, + self.is12_utils.create_remove_sequence_item_command_JSON(command_handle, oid, property_id, index) @@ -434,10 +413,9 @@ def _remove_sequence_item(self, test, oid, property_id, index): def _get_member_descriptors(self, test, oid, recurse): """Get BlockMemberDescritors for this block. Raises NMOSTestException on error""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) get_member_descriptors_command = \ - self.is12_utils.create_get_member_descriptors_JSON(version, command_handle, oid, recurse) + self.is12_utils.create_get_member_descriptors_JSON(command_handle, oid, recurse) response = self.send_command(test, command_handle, get_member_descriptors_command) @@ -446,11 +424,9 @@ def _get_member_descriptors(self, test, oid, recurse): def _find_members_by_path(self, test, oid, role_path): """Query members based on role path. Raises NMOSTestException on error""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) find_members_by_path_command = \ - self.is12_utils.create_find_members_by_path_command_JSON(version, - command_handle, + self.is12_utils.create_find_members_by_path_command_JSON(command_handle, oid, role_path) response = self.send_command(test, command_handle, find_members_by_path_command) @@ -460,11 +436,9 @@ def _find_members_by_path(self, test, oid, role_path): def _find_members_by_role(self, test, oid, role, case_sensitive, match_whole_string, recurse): """Query members based on role. Raises NMOSTestException on error""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) find_members_by_role_command = \ - self.is12_utils.create_find_members_by_role_command_JSON(version, - command_handle, + self.is12_utils.create_find_members_by_role_command_JSON(command_handle, oid, role, case_sensitive, @@ -477,11 +451,9 @@ def _find_members_by_role(self, test, oid, role, case_sensitive, match_whole_str def _find_members_by_class_id(self, test, oid, class_id, include_derived, recurse): """Query members based on class id. Raises NMOSTestException on error""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) find_members_by_class_id_command = \ - self.is12_utils.create_find_members_by_class_id_command_JSON(version, - command_handle, + self.is12_utils.create_find_members_by_class_id_command_JSON(command_handle, oid, class_id, include_derived, @@ -494,9 +466,9 @@ def get_class_manager_descriptors(self, test, class_manager_oid, property_id): response = self._get_property(test, class_manager_oid, property_id) # Create descriptor dictionary from response array - # Use identity as key if present, otherwise use name - def key_lambda(identity, name): return ".".join(map(str, identity)) if identity else name - descriptors = {key_lambda(r.get('identity'), r['name']): r for r in response} + # Use classId as key if present, otherwise use name + def key_lambda(classId, name): return ".".join(map(str, classId)) if classId else name + descriptors = {key_lambda(r.get('classId'), r['name']): r for r in response} return descriptors @@ -1354,7 +1326,7 @@ def test_21(self, test): return test.PASS() - def do_error_test(self, test, command_handle, command_json, expected_status=None, is12_error=True): + def do_error_test(self, test, command_handle, command_json, expected_status=None): """Execute command with expected error status.""" # when expected_status = None checking of the status code is skipped # check the syntax of the error message according to is12_error @@ -1374,11 +1346,6 @@ def do_error_test(self, test, command_handle, command_json, expected_status=None # It must be some other type of error so re-throw raise e - # 'protocolVersion' key is found in IS-12 protocol errors, but not in MS-05-02 errors - if is12_error != ('protocolVersion' in error_msg): - spec = "IS-12 protocol" if is12_error else "MS-05-02" - return test.FAIL(spec + " error expected") - if not error_msg.get('status'): return test.FAIL("Command error: " + str(error_msg)) @@ -1396,32 +1363,13 @@ def do_error_test(self, test, command_handle, command_json, expected_status=None return test.PASS() - def test_22(self, test): - """IS-12 Protocol Error: Node handles incorrect IS-12 protocol version""" - - command_handle = self.get_command_handle() - # Use incorrect protocol version - version = 'DOES.NOT.EXIST' - command_json = \ - self.is12_utils.create_generic_get_command_JSON(version, - command_handle, - self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) - - return self.do_error_test(test, - command_handle, - command_json, - expected_status=NcMethodStatus.ProtocolVersionError) - def test_23(self, test): """IS-12 Protocol Error: Node handles invalid command handle""" # Use invalid handle invalid_command_handle = "NOT A HANDLE" - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) command_json = \ - self.is12_utils.create_generic_get_command_JSON(version, - invalid_command_handle, + self.is12_utils.create_generic_get_command_JSON(invalid_command_handle, self.is12_utils.ROOT_BLOCK_OID, self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) @@ -1432,10 +1380,8 @@ def test_23(self, test): def test_24(self, test): """IS-12 Protocol Error: Node handles invalid command type""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) command_json = \ - self.is12_utils.create_generic_get_command_JSON(version, - command_handle, + self.is12_utils.create_generic_get_command_JSON(command_handle, self.is12_utils.ROOT_BLOCK_OID, self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) # Use invalid message type @@ -1459,46 +1405,38 @@ def test_26(self, test): """MS-05-02 Error: Node handles invalid oid""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) # Use invalid oid invalid_oid = 999999999 command_json = \ - self.is12_utils.create_generic_get_command_JSON(version, - command_handle, + self.is12_utils.create_generic_get_command_JSON(command_handle, invalid_oid, self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) return self.do_error_test(test, command_handle, command_json, - expected_status=NcMethodStatus.BadOid, - is12_error=False) + expected_status=NcMethodStatus.BadOid) def test_27(self, test): """MS-05-02 Error: Node handles invalid property identifier""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) # Use invalid property id invalid_property_identifier = {'level': 1, 'index': 999} command_json = \ - self.is12_utils.create_generic_get_command_JSON(version, - command_handle, + self.is12_utils.create_generic_get_command_JSON(command_handle, self.is12_utils.ROOT_BLOCK_OID, invalid_property_identifier) return self.do_error_test(test, command_handle, command_json, - expected_status=NcMethodStatus.PropertyNotImplemented, - is12_error=False) + expected_status=NcMethodStatus.PropertyNotImplemented) def test_28(self, test): """MS-05-02 Error: Node handles invalid method identifier""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) command_json = \ - self.is12_utils.create_generic_get_command_JSON(version, - command_handle, + self.is12_utils.create_generic_get_command_JSON(command_handle, self.is12_utils.ROOT_BLOCK_OID, self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) # Use invalid method id @@ -1508,17 +1446,14 @@ def test_28(self, test): return self.do_error_test(test, command_handle, command_json, - expected_status=NcMethodStatus.MethodNotImplemented, - is12_error=False) + expected_status=NcMethodStatus.MethodNotImplemented) def test_29(self, test): """MS-05-02 Error: Node handles read only error""" command_handle = self.get_command_handle() - version = self.is12_utils.format_version(self.apis[CONTROL_API_KEY]["version"]) # Try to set a read only property command_json = \ - self.is12_utils.create_generic_set_command_JSON(version, - command_handle, + self.is12_utils.create_generic_set_command_JSON(command_handle, self.is12_utils.ROOT_BLOCK_OID, self.is12_utils.PROPERTY_IDS['NCOBJECT']['ROLE'], "ROLE IS READ ONLY") @@ -1526,5 +1461,4 @@ def test_29(self, test): return self.do_error_test(test, command_handle, command_json, - expected_status=NcMethodStatus.Readonly, - is12_error=False) + expected_status=NcMethodStatus.Readonly) From b872ad2260804e2db868d0040744a0d1aec14c23 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Thu, 27 Jul 2023 11:22:34 +0100 Subject: [PATCH 15/45] GetSequenceLength test Disabled Add/Set/Remove sequence tests --- nmostesting/IS12Utils.py | 11 ++- nmostesting/suites/IS1201Test.py | 128 +++++++++---------------------- 2 files changed, 46 insertions(+), 93 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index 52882841..93c47a61 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -68,7 +68,8 @@ def protocol_definitions(self): 'GET_SEQUENCE_ITEM': {'level': 1, 'index': 3}, 'SET_SEQUENCE_ITEM': {'level': 1, 'index': 4}, 'ADD_SEQUENCE_ITEM': {'level': 1, 'index': 5}, - 'REMOVE_SEQUENCE_ITEM': {'level': 1, 'index': 6} + 'REMOVE_SEQUENCE_ITEM': {'level': 1, 'index': 6}, + 'GET_SEQUENCE_LENGTH': {'level': 1, 'index': 7} }, 'NCBLOCK': { 'GET_MEMBERS_DESCRIPTOR': {'level': 2, 'index': 1}, @@ -184,6 +185,14 @@ def create_remove_sequence_item_command_JSON(self, handle, oid, property_id, ind self.METHOD_IDS["NCOBJECT"]["REMOVE_SEQUENCE_ITEM"], {'id': property_id, 'index': index}) + def create_get_sequence_length_command_JSON(self, handle, oid, property_id): + """Create message that will request the sequence length value given an oid""" + + return self.create_command_JSON(handle, + oid, + self.METHOD_IDS["NCOBJECT"]["GET_SEQUENCE_LENGTH"], + {'id': property_id}) + def create_find_members_by_path_command_JSON(self, handle, oid, role_path): """Create JSON message for FindMembersByPath method from NcBlock""" diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index ac8386ff..4c3d4736 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -57,9 +57,7 @@ def set_up_tests(self): self.organization_metadata = {"checked": False, "error": False, "error_msg": ""} self.touchpoints_metadata = {"checked": False, "error": False, "error_msg": ""} self.get_sequence_item_metadata = {"checked": False, "error": False, "error_msg": ""} - self.set_sequence_item_metadata = {"checked": False, "error": False, "error_msg": ""} - self.add_sequence_item_metadata = {"checked": False, "error": False, "error_msg": ""} - self.remove_sequence_item_metadata = {"checked": False, "error": False, "error_msg": ""} + self.get_sequence_length_metadata = {"checked": False, "error": False, "error_msg": ""} self.oid_cache = [] @@ -410,6 +408,18 @@ def _remove_sequence_item(self, test, oid, property_id, index): return response["result"] + def _get_sequence_length(self, test, oid, property_id): + """Get value from sequence property. Raises NMOSTestException on error""" + command_handle = self.get_command_handle() + + get_sequence_length_command = \ + self.is12_utils.create_get_sequence_length_command_JSON(command_handle, + oid, + property_id) + response = self.send_command(test, command_handle, get_sequence_length_command) + + return response["result"]["value"] + def _get_member_descriptors(self, test, oid, recurse): """Get BlockMemberDescritors for this block. Raises NMOSTestException on error""" command_handle = self.get_command_handle() @@ -620,75 +630,26 @@ def check_get_sequence_item(self, test, oid, sequence_values, property_metadata, context + property_metadata["name"] + ": " + str(e.args[0].detail) + ", " return False - def check_add_sequence_item(self, test, oid, property_metadata, sequence_length, context=""): - try: - self.add_sequence_item_metadata["checked"] = True - # Add a value to the end of the sequence - new_item = self._get_sequence_item(test, oid, property_metadata['id'], index=0) - - self._add_sequence_item(test, oid, property_metadata['id'], new_item) - - # check the value - value = self._get_sequence_item(test, oid, property_metadata['id'], index=sequence_length) - if value != new_item: - self.add_sequence_item_metadata["error"] = True - self.add_sequence_item_metadata["error_msg"] += \ - context + property_metadata["name"] \ - + ": Expected: " + str(new_item) + ", Actual: " + str(value) + ", " - return True - except NMOSTestException as e: - self.add_sequence_item_metadata["error"] = True - self.add_sequence_item_metadata["error_msg"] += \ - context + property_metadata["name"] + ": " + str(e.args[0].detail) + ", " - return False - - def check_set_sequence_item(self, test, oid, property_metadata, sequence_length, context=""): + def check_get_sequence_length(self, test, oid, sequence_values, property_metadata, context=""): try: - self.set_sequence_item_metadata["checked"] = True - new_value = self._get_sequence_item(test, oid, property_metadata['id'], index=sequence_length - 1) - - # set to another value - self._set_sequence_item(test, oid, property_metadata['id'], index=sequence_length, value=new_value) - - # check the value - value = self._get_sequence_item(test, oid, property_metadata['id'], index=sequence_length) - if value != new_value: - self.set_sequence_item_metadata["error"] = True - self.set_sequence_item_metadata["error_msg"] += \ - context + property_metadata["name"] \ - + ": Expected: " + str(new_value) + ", Actual: " + str(value) + ", " - return True + length = self._get_sequence_length(test, oid, property_metadata['id']) + + if length == len(sequence_values): + return True + self.get_sequence_length_metadata["error_msg"] += \ + context + property_metadata["name"] \ + + ": GetSequenceLength error. Expected: " \ + + str(len(sequence_values)) + ", Actual: " + str(length) + ", " except NMOSTestException as e: - self.set_sequence_item_metadata["error"] = True - self.set_sequence_item_metadata["error_msg"] += \ - context + property_metadata["name"] + ": " + str(e.args[0].detail) + ", " - return False - - def check_remove_sequence_item(self, test, oid, property_metadata, sequence_length, context=""): - try: - # remove item - self.remove_sequence_item_metadata["checked"] = True - self._remove_sequence_item(test, oid, property_metadata['id'], index=sequence_length) - return True - except NMOSTestException as e: - self.remove_sequence_item_metadata["error"] = True - self.remove_sequence_item_metadata["error_msg"] += \ + self.get_sequence_length_metadata["error_msg"] += \ context + property_metadata["name"] + ": " + str(e.args[0].detail) + ", " + self.get_sequence_length_metadata["error"] = True return False def check_sequence_methods(self, test, oid, sequence_values, property_metadata, context=""): """Check that sequence manipulation methods work correctly""" self.check_get_sequence_item(test, oid, sequence_values, property_metadata, context) - - if not property_metadata['isReadOnly']: - sequence_length = len(sequence_values) - - if not self.check_add_sequence_item(test, oid, property_metadata, sequence_length, context=context): - return - - self.check_set_sequence_item(test, oid, property_metadata, sequence_length, context=context) - - self.check_remove_sequence_item(test, oid, property_metadata, sequence_length, context) + self.check_get_sequence_length(test, oid, sequence_values, property_metadata, context) def validate_object_properties(self, test, reference_class_descriptor, oid, datatype_schemas, context): for class_property in reference_class_descriptor['properties']: @@ -1067,49 +1028,32 @@ def test_14(self, test): def test_15(self, test): """NcObject method: SetSequenceItem""" - try: - self.validate_device_model(test) - except NMOSTestException as e: - # Couldn't validate model so can't perform test - return test.UNCLEAR(e.args[0].detail, e.args[0].link) - - if self.set_sequence_item_metadata["error"]: - return test.FAIL(self.set_sequence_item_metadata["error_msg"]) - if not self.set_sequence_item_metadata["checked"]: - return test.UNCLEAR("SetSequenceItem not tested.") - - return test.PASS() + return test.DISABLED() def test_16(self, test): """NcObject method: AddSequenceItem""" - try: - self.validate_device_model(test) - except NMOSTestException as e: - # Couldn't validate model so can't perform test - return test.UNCLEAR(e.args[0].detail, e.args[0].link) - - if self.add_sequence_item_metadata["error"]: - return test.FAIL(self.add_sequence_item_metadata["error_msg"]) - if not self.add_sequence_item_metadata["checked"]: - return test.UNCLEAR("AddSequenceItem not tested.") - - return test.PASS() + return test.DISABLED() def test_17(self, test): """NcObject method: RemoveSequenceItem""" + + return test.DISABLED() + + def test_17_1(self, test): + """NcObject method: GetSequenceLength""" try: self.validate_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) - if self.remove_sequence_item_metadata["error"]: - return test.FAIL(self.remove_sequence_item_metadata["error_msg"]) + if self.get_sequence_length_metadata["error"]: + return test.FAIL(self.get_sequence_length_metadata["error_msg"]) - if not self.remove_sequence_item_metadata["checked"]: - return test.UNCLEAR("RemoveSequenceItem not tested.") + if not self.get_sequence_length_metadata["checked"]: + return test.UNCLEAR("GetSequenceItem not tested.") return test.PASS() From a64beefe9aed6042df5c75fa373b47271a8fc71b Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Thu, 27 Jul 2023 13:18:41 +0100 Subject: [PATCH 16/45] Moved IS-12 command support into IS12Utils --- nmostesting/IS12Utils.py | 120 ++++++++++++++++++++++++++++++- nmostesting/suites/IS1201Test.py | 118 +++++------------------------- 2 files changed, 138 insertions(+), 100 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index 93c47a61..22249ccf 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -14,8 +14,15 @@ from .NMOSUtils import NMOSUtils +import json +import time + from enum import IntEnum from itertools import takewhile, dropwhile +from jsonschema import FormatChecker, SchemaError, validate, ValidationError +from .Config import WS_MESSAGE_TIMEOUT +from .GenericTest import NMOSTestException +from .TestHelper import WebsocketWorker, load_resolved_schema class MessageTypes(IntEnum): @@ -54,9 +61,12 @@ class NcDatatypeType(IntEnum): class IS12Utils(NMOSUtils): - def __init__(self, url): + def __init__(self, url, spec_path, spec_branch): NMOSUtils.__init__(self, url) + self.spec_branch = spec_branch self.protocol_definitions() + self.load_is12_schemas(spec_path) + self.ncp_websocket = None def protocol_definitions(self): self.ROOT_BLOCK_OID = 1 @@ -115,6 +125,114 @@ def protocol_definitions(self): 'NCCLASSMANAGER': [1, 3, 2] } + def load_is12_schemas(self, spec_path): + """Load datatype and control class decriptors and create datatype JSON schemas""" + # Load IS-12 schemas + self.schemas = {} + schema_names = ['error-message', 'command-response-message'] + for schema_name in schema_names: + self.schemas[schema_name] = load_resolved_schema(spec_path, schema_name + ".json") + + def open_ncp_websocket(self, test, url): + """Create a WebSocket client connection to Node under test. Raises NMOSTestException on error""" + # Reuse socket if connection already established + if self.ncp_websocket and self.ncp_websocket.is_open(): + return + + # Create a WebSocket connection to NMOS Control Protocol endpoint + self.ncp_websocket = WebsocketWorker(url) + self.ncp_websocket.start() + + # Give WebSocket client a chance to start and open its connection + start_time = time.time() + while time.time() < start_time + WS_MESSAGE_TIMEOUT: + if self.ncp_websocket.is_open(): + break + time.sleep(0.2) + + if self.ncp_websocket.did_error_occur() or not self.ncp_websocket.is_open(): + raise NMOSTestException(test.FAIL("Failed to open WebSocket successfully" + + (": " + str(self.ncp_websocket.get_error_message()) + if self.ncp_websocket.did_error_occur() else "."))) + + def close_ncp_websocket(self): + # Clean up Websocket resources + if self.ncp_websocket: + self.ncp_websocket.close() + + def validate_is12_schema(self, test, payload, schema_name, context=""): + """Delegates to validate_schema. Raises NMOSTestExceptions on error""" + try: + # Validate the JSON schema is correct + checker = FormatChecker(["ipv4", "ipv6", "uri"]) + validate(payload, self.schemas[schema_name], format_checker=checker) + except ValidationError as e: + raise NMOSTestException(test.FAIL(context + "Schema validation error: " + e.message)) + except SchemaError as e: + raise NMOSTestException(test.FAIL(context + "Schema error: " + e.message)) + + return + + def send_command(self, test, command_handle, command_json): + """Send command to Node under test. Returns [command response]. Raises NMOSTestException on error""" + # Referencing the Google sheet + # IS-12 (9) Check message type + # IS-12 (10) Check handle numeric identifier + # https://specs.amwa.tv/is-12/branches/v1.0-dev/docs/Protocol_messaging.html + # IS-12 (11) Check Command message type + # https://specs.amwa.tv/is-12/branches/v1.0-dev/docs/Protocol_messaging.html#command-message-type + # MS-05-02 (74) All methods MUST return a datatype which inherits from NcMethodResult. + # When a method call encounters an error the return MUST be NcMethodResultError + # or a derived datatype. + # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Framework.html#ncmethodresult + + results = [] + + self.ncp_websocket.send(json.dumps(command_json)) + + # Wait for server to respond + start_time = time.time() + while time.time() < start_time + WS_MESSAGE_TIMEOUT: + if self.ncp_websocket.is_messages_received(): + break + time.sleep(0.2) + + messages = self.ncp_websocket.get_messages() + + # find the response to our request + for message in messages: + parsed_message = json.loads(message) + + if parsed_message["messageType"] == MessageTypes.CommandResponse: + self.validate_is12_schema(test, + parsed_message, + "command-response-message", + context="command-response-message: ") + responses = parsed_message["responses"] + for response in responses: + if response["handle"] == command_handle: + if response["result"]["status"] != NcMethodStatus.OK: + raise NMOSTestException(test.FAIL(response["result"])) + results.append(response) + if parsed_message["messageType"] == MessageTypes.Error: + self.validate_is12_schema(test, + parsed_message, + "error-message", + context="error-message: ") + raise NMOSTestException(test.FAIL(parsed_message, "https://specs.amwa.tv/is-12/branches/{}" + "/docs/Protocol_messaging.html#error-messages" + .format(self.spec_branch))) + if len(results) == 0: + raise NMOSTestException(test.FAIL("No Command Message Response received.", + "https://specs.amwa.tv/is-12/branches/{}" + "/docs/Protocol_messaging.html#command-message-type" + .format(self.spec_branch))) + + if len(results) > 1: + raise NMOSTestException(test.FAIL("Received multiple responses : " + len(responses))) + + return results[0] + def create_command_JSON(self, handle, oid, method_id, arguments): """Create command JSON for generic get of a property""" return { diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 4c3d4736..d3a58d50 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -14,15 +14,13 @@ import json import os -import time from itertools import product from jsonschema import ValidationError, SchemaError -from ..Config import WS_MESSAGE_TIMEOUT from ..GenericTest import GenericTest, NMOSTestException -from ..IS12Utils import IS12Utils, NcMethodStatus, MessageTypes, NcObject -from ..TestHelper import WebsocketWorker, load_resolved_schema +from ..IS12Utils import IS12Utils, NcMethodStatus, NcObject +from ..TestHelper import load_resolved_schema from ..TestResult import Test NODE_API_KEY = "node" @@ -42,8 +40,9 @@ def __init__(self, apis, **kwargs): GenericTest.__init__(self, apis, **kwargs) self.node_url = self.apis[NODE_API_KEY]["url"] self.ncp_url = self.apis[CONTROL_API_KEY]["url"] - self.is12_utils = IS12Utils(self.node_url) - self.ncp_websocket = None + self.is12_utils = IS12Utils(self.node_url, + self.apis[CONTROL_API_KEY]["spec_path"], + self.apis[CONTROL_API_KEY]["spec_branch"]) self.load_reference_resources() self.command_handle = 0 self.root_block = None @@ -63,8 +62,7 @@ def set_up_tests(self): def tear_down_tests(self): # Clean up Websocket resources - if self.ncp_websocket: - self.ncp_websocket.close() + self.is12_utils.close_ncp_websocket() def execute_tests(self, test_names): """Perform tests defined within this class""" @@ -147,91 +145,13 @@ def load_reference_resources(self): def create_ncp_socket(self, test): """Create a WebSocket client connection to Node under test. Raises NMOSTestException on error""" - # Reuse socket if connection already established - if self.ncp_websocket and self.ncp_websocket.is_open(): - return - - # Create a WebSocket connection to NMOS Control Protocol endpoint - self.ncp_websocket = WebsocketWorker(self.apis[CONTROL_API_KEY]["url"]) - self.ncp_websocket.start() - - # Give WebSocket client a chance to start and open its connection - start_time = time.time() - while time.time() < start_time + WS_MESSAGE_TIMEOUT: - if self.ncp_websocket.is_open(): - break - time.sleep(0.2) - - if self.ncp_websocket.did_error_occur() or not self.ncp_websocket.is_open(): - raise NMOSTestException(test.FAIL("Failed to open WebSocket successfully" - + (": " + str(self.ncp_websocket.get_error_message()) - if self.ncp_websocket.did_error_occur() else "."))) + self.is12_utils.open_ncp_websocket(test, self.apis[CONTROL_API_KEY]["url"]) def get_command_handle(self): """Get unique command handle""" self.command_handle += 1 return self.command_handle - def send_command(self, test, command_handle, command_json): - """Send command to Node under test. Returns [command response]. Raises NMOSTestException on error""" - # Referencing the Google sheet - # IS-12 (9) Check message type - # IS-12 (10) Check handle numeric identifier - # https://specs.amwa.tv/is-12/branches/v1.0-dev/docs/Protocol_messaging.html - # IS-12 (11) Check Command message type - # https://specs.amwa.tv/is-12/branches/v1.0-dev/docs/Protocol_messaging.html#command-message-type - # MS-05-02 (74) All methods MUST return a datatype which inherits from NcMethodResult. - # When a method call encounters an error the return MUST be NcMethodResultError - # or a derived datatype. - # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Framework.html#ncmethodresult - - results = [] - - self.ncp_websocket.send(json.dumps(command_json)) - - # Wait for server to respond - start_time = time.time() - while time.time() < start_time + WS_MESSAGE_TIMEOUT: - if self.ncp_websocket.is_messages_received(): - break - time.sleep(0.2) - - messages = self.ncp_websocket.get_messages() - - # find the response to our request - for message in messages: - parsed_message = json.loads(message) - - if parsed_message["messageType"] == MessageTypes.CommandResponse: - self._validate_schema(test, - parsed_message, - self.schemas["command-response-message"], - context="command-response-message: ") - responses = parsed_message["responses"] - for response in responses: - if response["handle"] == command_handle: - if response["result"]["status"] != NcMethodStatus.OK: - raise NMOSTestException(test.FAIL(response["result"])) - results.append(response) - if parsed_message["messageType"] == MessageTypes.Error: - self._validate_schema(test, - parsed_message, - self.schemas["error-message"], - context="error-message: ") - raise NMOSTestException(test.FAIL(parsed_message, "https://specs.amwa.tv/is-12/branches/{}" - "/docs/Protocol_messaging.html#error-messages" - .format(self.apis[CONTROL_API_KEY]["spec_branch"]))) - if len(results) == 0: - raise NMOSTestException(test.FAIL("No Command Message Response received.", - "https://specs.amwa.tv/is-12/branches/{}" - "/docs/Protocol_messaging.html#command-message-type" - .format(self.apis[CONTROL_API_KEY]["spec_branch"]))) - - if len(results) > 1: - raise NMOSTestException(test.FAIL("Received multiple responses : " + len(responses))) - - return results[0] - def get_manager(self, test, class_id_str): """Get Manager from Root Block. Returns [Manager]. Raises NMOSTestException on error""" response = self._get_property(test, @@ -338,7 +258,7 @@ def _get_property(self, test, oid, property_id): self.is12_utils.create_generic_get_command_JSON(command_handle, oid, property_id) - response = self.send_command(test, command_handle, get_property_command) + response = self.is12_utils.send_command(test, command_handle, get_property_command) return response["result"]["value"] @@ -351,7 +271,7 @@ def _set_property(self, test, oid, property_id, argument): oid, property_id, argument) - response = self.send_command(test, command_handle, set_property_command) + response = self.is12_utils.send_command(test, command_handle, set_property_command) return response["result"] @@ -364,7 +284,7 @@ def _get_sequence_item(self, test, oid, property_id, index): oid, property_id, index) - response = self.send_command(test, command_handle, get_sequence_item_command) + response = self.is12_utils.send_command(test, command_handle, get_sequence_item_command) return response["result"]["value"] @@ -378,7 +298,7 @@ def _set_sequence_item(self, test, oid, property_id, index, value): property_id, index, value) - response = self.send_command(test, command_handle, add_sequence_item_command) + response = self.is12_utils.send_command(test, command_handle, add_sequence_item_command) return response["result"] @@ -391,7 +311,7 @@ def _add_sequence_item(self, test, oid, property_id, value): oid, property_id, value) - response = self.send_command(test, command_handle, add_sequence_item_command) + response = self.is12_utils.send_command(test, command_handle, add_sequence_item_command) return response["result"] @@ -404,7 +324,7 @@ def _remove_sequence_item(self, test, oid, property_id, index): oid, property_id, index) - response = self.send_command(test, command_handle, get_sequence_item_command) + response = self.is12_utils.send_command(test, command_handle, get_sequence_item_command) return response["result"] @@ -416,7 +336,7 @@ def _get_sequence_length(self, test, oid, property_id): self.is12_utils.create_get_sequence_length_command_JSON(command_handle, oid, property_id) - response = self.send_command(test, command_handle, get_sequence_length_command) + response = self.is12_utils.send_command(test, command_handle, get_sequence_length_command) return response["result"]["value"] @@ -427,7 +347,7 @@ def _get_member_descriptors(self, test, oid, recurse): get_member_descriptors_command = \ self.is12_utils.create_get_member_descriptors_JSON(command_handle, oid, recurse) - response = self.send_command(test, command_handle, get_member_descriptors_command) + response = self.is12_utils.send_command(test, command_handle, get_member_descriptors_command) return response["result"]["value"] @@ -439,7 +359,7 @@ def _find_members_by_path(self, test, oid, role_path): self.is12_utils.create_find_members_by_path_command_JSON(command_handle, oid, role_path) - response = self.send_command(test, command_handle, find_members_by_path_command) + response = self.is12_utils.send_command(test, command_handle, find_members_by_path_command) return response["result"]["value"] @@ -454,7 +374,7 @@ def _find_members_by_role(self, test, oid, role, case_sensitive, match_whole_str case_sensitive, match_whole_string, recurse) - response = self.send_command(test, command_handle, find_members_by_role_command) + response = self.is12_utils.send_command(test, command_handle, find_members_by_role_command) return response["result"]["value"] @@ -468,7 +388,7 @@ def _find_members_by_class_id(self, test, oid, class_id, include_derived, recurs class_id, include_derived, recurse) - response = self.send_command(test, command_handle, find_members_by_class_id_command) + response = self.is12_utils.send_command(test, command_handle, find_members_by_class_id_command) return response["result"]["value"] @@ -1278,7 +1198,7 @@ def do_error_test(self, test, command_handle, command_json, expected_status=None try: self.create_ncp_socket(test) - self.send_command(test, command_handle, command_json) + self.is12_utils.send_command(test, command_handle, command_json) return test.FAIL("Error expected") From 3f17d6bb67e9011353b1508c2276fb75021fd8d8 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Thu, 27 Jul 2023 14:42:06 +0100 Subject: [PATCH 17/45] Refactored command execution --- nmostesting/IS12Utils.py | 180 +++++++++---------- nmostesting/suites/IS1201Test.py | 290 +++++++------------------------ 2 files changed, 144 insertions(+), 326 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index 22249ccf..752ff1ed 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -67,6 +67,7 @@ def __init__(self, url, spec_path, spec_branch): self.protocol_definitions() self.load_is12_schemas(spec_path) self.ncp_websocket = None + self.command_handle = 0 def protocol_definitions(self): self.ROOT_BLOCK_OID = 1 @@ -173,7 +174,7 @@ def validate_is12_schema(self, test, payload, schema_name, context=""): return - def send_command(self, test, command_handle, command_json): + def send_command(self, test, command_json): """Send command to Node under test. Returns [command response]. Raises NMOSTestException on error""" # Referencing the Google sheet # IS-12 (9) Check message type @@ -186,7 +187,8 @@ def send_command(self, test, command_handle, command_json): # or a derived datatype. # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Framework.html#ncmethodresult - results = [] + # Assume single command + command_handle = command_json['commands'][0]['handle'] if command_json.get('commands') else 0 self.ncp_websocket.send(json.dumps(command_json)) @@ -199,6 +201,7 @@ def send_command(self, test, command_handle, command_json): messages = self.ncp_websocket.get_messages() + results = [] # find the response to our request for message in messages: parsed_message = json.loads(message) @@ -233,13 +236,14 @@ def send_command(self, test, command_handle, command_json): return results[0] - def create_command_JSON(self, handle, oid, method_id, arguments): + def create_command_JSON(self, oid, method_id, arguments): """Create command JSON for generic get of a property""" + self.command_handle += 1 return { 'messageType': MessageTypes.Command, 'commands': [ { - 'handle': handle, + 'handle': self.command_handle, 'oid': oid, 'methodId': method_id, 'arguments': arguments @@ -247,99 +251,81 @@ def create_command_JSON(self, handle, oid, method_id, arguments): ], } - def create_generic_get_command_JSON(self, handle, oid, property_id): - """Create command JSON for generic get of a property""" - - return self.create_command_JSON(handle, - oid, - self.METHOD_IDS["NCOBJECT"]["GENERIC_GET"], - {'id': property_id}) - - def create_generic_set_command_JSON(self, handle, oid, property_id, value): - """Create command JSON for generic get of a property""" - - return self.create_command_JSON(handle, - oid, - self.METHOD_IDS["NCOBJECT"]["GENERIC_SET"], - {'id': property_id, 'value': value}) - - def create_get_member_descriptors_JSON(self, handle, oid, recurse): - """Create message that will request the member descriptors of the object with the given oid""" - - return self.create_command_JSON(handle, - oid, - self.METHOD_IDS["NCBLOCK"]["GET_MEMBERS_DESCRIPTOR"], - {'recurse': recurse}) - - def create_get_sequence_item_command_JSON(self, handle, oid, property_id, index): - """Create message that will request the sequence item value given an oid and index""" - - return self.create_command_JSON(handle, - oid, - self.METHOD_IDS["NCOBJECT"]["GET_SEQUENCE_ITEM"], - {'id': property_id, 'index': index}) - - def create_set_sequence_item_command_JSON(self, handle, oid, property_id, index, value): - """Create message that will add a sequence item value""" - - return self.create_command_JSON(handle, - oid, - self.METHOD_IDS["NCOBJECT"]["SET_SEQUENCE_ITEM"], - {'id': property_id, 'index': index, 'value': value}) - - def create_add_sequence_item_command_JSON(self, handle, oid, property_id, value): - """Create message that will add a sequence item value""" - - return self.create_command_JSON(handle, - oid, - self.METHOD_IDS["NCOBJECT"]["ADD_SEQUENCE_ITEM"], - {'id': property_id, 'value': value}) - - def create_remove_sequence_item_command_JSON(self, handle, oid, property_id, index): - """Create message that will request the sequence item value given an oid and index""" - - return self.create_command_JSON(handle, - oid, - self.METHOD_IDS["NCOBJECT"]["REMOVE_SEQUENCE_ITEM"], - {'id': property_id, 'index': index}) - - def create_get_sequence_length_command_JSON(self, handle, oid, property_id): - """Create message that will request the sequence length value given an oid""" - - return self.create_command_JSON(handle, - oid, - self.METHOD_IDS["NCOBJECT"]["GET_SEQUENCE_LENGTH"], - {'id': property_id}) - - def create_find_members_by_path_command_JSON(self, handle, oid, role_path): - """Create JSON message for FindMembersByPath method from NcBlock""" - - return self.create_command_JSON(handle, - oid, - self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_PATH"], - {'path': role_path}) - - def create_find_members_by_role_command_JSON(self, handle, oid, role, - case_sensitive, match_whole_string, recurse): - """Create JSON message for FindMembersByPath method from NcBlock""" - - return self.create_command_JSON(handle, - oid, - self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_ROLE"], - {'role': role, - 'caseSensitive': case_sensitive, - 'matchWholeString': match_whole_string, - 'recurse': recurse}) - - def create_find_members_by_class_id_command_JSON(self, handle, oid, class_id, include_derived, recurse): - """Create JSON message for FindMembersByClassId method from NcBlock""" - - return self.create_command_JSON(handle, - oid, - self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_CLASS_ID"], - {'classId': class_id, - 'includeDerived': include_derived, - 'recurse': recurse}) + def _execute_command(self, test, oid, method_id, arguments): + command_JSON = self.create_command_JSON(oid, method_id, arguments) + response = self.send_command(test, command_JSON) + return response["result"] + + def get_property(self, test, oid, property_id): + """Get property from object. Raises NMOSTestException on error""" + return self._execute_command(test, oid, + self.METHOD_IDS["NCOBJECT"]["GENERIC_GET"], + {'id': property_id})["value"] + + def set_property(self, test, oid, property_id, argument): + """Get property from object. Raises NMOSTestException on error""" + return self._execute_command(test, oid, + self.METHOD_IDS["NCOBJECT"]["GENERIC_SET"], + {'id': property_id, 'value': argument}) + + def get_sequence_item(self, test, oid, property_id, index): + """Get value from sequence property. Raises NMOSTestException on error""" + return self._execute_command(test, oid, + self.METHOD_IDS["NCOBJECT"]["GET_SEQUENCE_ITEM"], + {'id': property_id, 'index': index})["value"] + + def set_sequence_item(self, test, oid, property_id, index, value): + """Add value to a sequence property. Raises NMOSTestException on error""" + return self._execute_command(test, oid, + self.METHOD_IDS["NCOBJECT"]["SET_SEQUENCE_ITEM"], + {'id': property_id, 'index': index, 'value': value}) + + def add_sequence_item(self, test, oid, property_id, value): + """Add value to a sequence property. Raises NMOSTestException on error""" + return self._execute_command(test, oid, + self.METHOD_IDS["NCOBJECT"]["ADD_SEQUENCE_ITEM"], + {'id': property_id, 'value': value}) + + def remove_sequence_item(self, test, oid, property_id, index): + """Get value from sequence property. Raises NMOSTestException on error""" + return self._execute_command(test, oid, + self.METHOD_IDS["NCOBJECT"]["REMOVE_SEQUENCE_ITEM"], + {'id': property_id, 'index': index}) + + def get_sequence_length(self, test, oid, property_id): + """Get value from sequence property. Raises NMOSTestException on error""" + return self._execute_command(test, oid, + self.METHOD_IDS["NCOBJECT"]["GET_SEQUENCE_LENGTH"], + {'id': property_id})["value"] + + def get_member_descriptors(self, test, oid, recurse): + """Get BlockMemberDescritors for this block. Raises NMOSTestException on error""" + return self._execute_command(test, oid, + self.METHOD_IDS["NCBLOCK"]["GET_MEMBERS_DESCRIPTOR"], + {'recurse': recurse})["value"] + + def find_members_by_path(self, test, oid, role_path): + """Query members based on role path. Raises NMOSTestException on error""" + return self._execute_command(test, oid, + self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_PATH"], + {'path': role_path})["value"] + + def find_members_by_role(self, test, oid, role, case_sensitive, match_whole_string, recurse): + """Query members based on role. Raises NMOSTestException on error""" + return self._execute_command(test, oid, + self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_ROLE"], + {'role': role, + 'caseSensitive': case_sensitive, + 'matchWholeString': match_whole_string, + 'recurse': recurse})["value"] + + def find_members_by_class_id(self, test, oid, class_id, include_derived, recurse): + """Query members based on class id. Raises NMOSTestException on error""" + return self._execute_command(test, oid, + self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_CLASS_ID"], + {'classId': class_id, + 'includeDerived': include_derived, + 'recurse': recurse})["value"] def model_primitive_to_JSON(self, type): """Convert MS-05 primitive type to corresponding JSON type""" diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index d3a58d50..01f481e7 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -44,7 +44,6 @@ def __init__(self, apis, **kwargs): self.apis[CONTROL_API_KEY]["spec_path"], self.apis[CONTROL_API_KEY]["spec_branch"]) self.load_reference_resources() - self.command_handle = 0 self.root_block = None def set_up_tests(self): @@ -110,12 +109,6 @@ def generate_json_schemas(self, datatype_descriptors, schema_path): def load_reference_resources(self): """Load datatype and control class decriptors and create datatype JSON schemas""" - # Load IS-12 schemas - self.schemas = {} - schema_names = ['error-message', 'command-response-message'] - for schema_name in schema_names: - self.schemas[schema_name] = load_resolved_schema(self.apis[CONTROL_API_KEY]["spec_path"], - schema_name + ".json") # Calculate paths to MS-05 descriptors # including Feature Sets specified as additional_paths in test definition spec_paths = [os.path.join(self.apis[FEATURE_SETS_KEY]["spec_path"], path) @@ -147,16 +140,11 @@ def create_ncp_socket(self, test): """Create a WebSocket client connection to Node under test. Raises NMOSTestException on error""" self.is12_utils.open_ncp_websocket(test, self.apis[CONTROL_API_KEY]["url"]) - def get_command_handle(self): - """Get unique command handle""" - self.command_handle += 1 - return self.command_handle - def get_manager(self, test, class_id_str): """Get Manager from Root Block. Returns [Manager]. Raises NMOSTestException on error""" - response = self._get_property(test, - self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCBLOCK']['MEMBERS']) + response = self.is12_utils.get_property(test, + self.is12_utils.ROOT_BLOCK_OID, + self.is12_utils.PROPERTY_IDS['NCBLOCK']['MEMBERS']) manager_found = False manager = None @@ -250,150 +238,8 @@ def _validate_schema(self, test, payload, schema, context=""): return - def _get_property(self, test, oid, property_id): - """Get property from object. Raises NMOSTestException on error""" - command_handle = self.get_command_handle() - - get_property_command = \ - self.is12_utils.create_generic_get_command_JSON(command_handle, - oid, - property_id) - response = self.is12_utils.send_command(test, command_handle, get_property_command) - - return response["result"]["value"] - - def _set_property(self, test, oid, property_id, argument): - """Get property from object. Raises NMOSTestException on error""" - command_handle = self.get_command_handle() - - set_property_command = \ - self.is12_utils.create_generic_set_command_JSON(command_handle, - oid, - property_id, - argument) - response = self.is12_utils.send_command(test, command_handle, set_property_command) - - return response["result"] - - def _get_sequence_item(self, test, oid, property_id, index): - """Get value from sequence property. Raises NMOSTestException on error""" - command_handle = self.get_command_handle() - - get_sequence_item_command = \ - self.is12_utils.create_get_sequence_item_command_JSON(command_handle, - oid, - property_id, - index) - response = self.is12_utils.send_command(test, command_handle, get_sequence_item_command) - - return response["result"]["value"] - - def _set_sequence_item(self, test, oid, property_id, index, value): - """Add value to a sequence property. Raises NMOSTestException on error""" - command_handle = self.get_command_handle() - - add_sequence_item_command = \ - self.is12_utils.create_set_sequence_item_command_JSON(command_handle, - oid, - property_id, - index, - value) - response = self.is12_utils.send_command(test, command_handle, add_sequence_item_command) - - return response["result"] - - def _add_sequence_item(self, test, oid, property_id, value): - """Add value to a sequence property. Raises NMOSTestException on error""" - command_handle = self.get_command_handle() - - add_sequence_item_command = \ - self.is12_utils.create_add_sequence_item_command_JSON(command_handle, - oid, - property_id, - value) - response = self.is12_utils.send_command(test, command_handle, add_sequence_item_command) - - return response["result"] - - def _remove_sequence_item(self, test, oid, property_id, index): - """Get value from sequence property. Raises NMOSTestException on error""" - command_handle = self.get_command_handle() - - get_sequence_item_command = \ - self.is12_utils.create_remove_sequence_item_command_JSON(command_handle, - oid, - property_id, - index) - response = self.is12_utils.send_command(test, command_handle, get_sequence_item_command) - - return response["result"] - - def _get_sequence_length(self, test, oid, property_id): - """Get value from sequence property. Raises NMOSTestException on error""" - command_handle = self.get_command_handle() - - get_sequence_length_command = \ - self.is12_utils.create_get_sequence_length_command_JSON(command_handle, - oid, - property_id) - response = self.is12_utils.send_command(test, command_handle, get_sequence_length_command) - - return response["result"]["value"] - - def _get_member_descriptors(self, test, oid, recurse): - """Get BlockMemberDescritors for this block. Raises NMOSTestException on error""" - command_handle = self.get_command_handle() - - get_member_descriptors_command = \ - self.is12_utils.create_get_member_descriptors_JSON(command_handle, oid, recurse) - - response = self.is12_utils.send_command(test, command_handle, get_member_descriptors_command) - - return response["result"]["value"] - - def _find_members_by_path(self, test, oid, role_path): - """Query members based on role path. Raises NMOSTestException on error""" - command_handle = self.get_command_handle() - - find_members_by_path_command = \ - self.is12_utils.create_find_members_by_path_command_JSON(command_handle, - oid, - role_path) - response = self.is12_utils.send_command(test, command_handle, find_members_by_path_command) - - return response["result"]["value"] - - def _find_members_by_role(self, test, oid, role, case_sensitive, match_whole_string, recurse): - """Query members based on role. Raises NMOSTestException on error""" - command_handle = self.get_command_handle() - - find_members_by_role_command = \ - self.is12_utils.create_find_members_by_role_command_JSON(command_handle, - oid, - role, - case_sensitive, - match_whole_string, - recurse) - response = self.is12_utils.send_command(test, command_handle, find_members_by_role_command) - - return response["result"]["value"] - - def _find_members_by_class_id(self, test, oid, class_id, include_derived, recurse): - """Query members based on class id. Raises NMOSTestException on error""" - command_handle = self.get_command_handle() - - find_members_by_class_id_command = \ - self.is12_utils.create_find_members_by_class_id_command_JSON(command_handle, - oid, - class_id, - include_derived, - recurse) - response = self.is12_utils.send_command(test, command_handle, find_members_by_class_id_command) - - return response["result"]["value"] - def get_class_manager_descriptors(self, test, class_manager_oid, property_id): - response = self._get_property(test, class_manager_oid, property_id) + response = self.is12_utils.get_property(test, class_manager_oid, property_id) # Create descriptor dictionary from response array # Use classId as key if present, otherwise use name @@ -485,9 +331,9 @@ def test_03(self, test): self.create_ncp_socket(test) - role = self._get_property(test, - self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['ROLE']) + role = self.is12_utils.get_property(test, + self.is12_utils.ROOT_BLOCK_OID, + self.is12_utils.PROPERTY_IDS['NCOBJECT']['ROLE']) if role != "root": return test.FAIL("Unexpected role in Root Block: " + role, @@ -535,7 +381,7 @@ def check_get_sequence_item(self, test, oid, sequence_values, property_metadata, self.get_sequence_item_metadata["checked"] = True sequence_index = 0 for property_value in sequence_values: - value = self._get_sequence_item(test, oid, property_metadata['id'], sequence_index) + value = self.is12_utils.get_sequence_item(test, oid, property_metadata['id'], sequence_index) if property_value != value: self.get_sequence_item_metadata["error"] = True self.get_sequence_item_metadata["error_msg"] += \ @@ -552,7 +398,7 @@ def check_get_sequence_item(self, test, oid, sequence_values, property_metadata, def check_get_sequence_length(self, test, oid, sequence_values, property_metadata, context=""): try: - length = self._get_sequence_length(test, oid, property_metadata['id']) + length = self.is12_utils.get_sequence_length(test, oid, property_metadata['id']) if length == len(sequence_values): return True @@ -573,7 +419,7 @@ def check_sequence_methods(self, test, oid, sequence_values, property_metadata, def validate_object_properties(self, test, reference_class_descriptor, oid, datatype_schemas, context): for class_property in reference_class_descriptor['properties']: - response = self._get_property(test, oid, class_property['id']) + response = self.is12_utils.get_property(test, oid, class_property['id']) # validate property type if class_property['isSequence']: @@ -625,9 +471,9 @@ def check_manager(self, class_id, owner, class_descriptors, manager_cache): def check_touchpoints(self, test, oid, datatype_schemas, context): """Touchpoint checks""" - touchpoints = self._get_property(test, - oid, - self.is12_utils.PROPERTY_IDS["NCOBJECT"]["TOUCHPOINTS"]) + touchpoints = self.is12_utils.get_property(test, + oid, + self.is12_utils.PROPERTY_IDS["NCOBJECT"]["TOUCHPOINTS"]) if touchpoints is not None: self.touchpoints_metadata["checked"] = True try: @@ -644,7 +490,7 @@ def check_touchpoints(self, test, oid, datatype_schemas, context): self.touchpoints_metadata["error_msg"] = context + str(e.args[0].detail) def validate_block(self, test, block_id, class_descriptors, datatype_schemas, block, context=""): - response = self._get_property(test, block_id, self.is12_utils.PROPERTY_IDS['NCBLOCK']['MEMBERS']) + response = self.is12_utils.get_property(test, block_id, self.is12_utils.PROPERTY_IDS['NCBLOCK']['MEMBERS']) role_cache = [] manager_cache = [] @@ -829,7 +675,7 @@ def test_10(self, test): # Check MS-05-02 Version property_id = self.is12_utils.PROPERTY_IDS['NCDEVICEMANAGER']['NCVERSION'] - version = self._get_property(test, device_manager['oid'], property_id) + version = self.is12_utils.get_property(test, device_manager['oid'], property_id) if self.is12_utils.compare_api_version(version, self.apis[MS05_API_KEY]["version"]): return test.FAIL("Unexpected version. Expected: " @@ -903,14 +749,14 @@ def test_13(self, test): property_id = self.is12_utils.PROPERTY_IDS['NCOBJECT']['USER_LABEL'] - old_user_label = self._get_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id) + old_user_label = self.is12_utils.get_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id) # Set user label new_user_label = "NMOS Testing Tool" - self._set_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id, new_user_label) + self.is12_utils.set_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id, new_user_label) # Check user label - label = self._get_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id) + label = self.is12_utils.get_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id) if label != new_user_label: if label == old_user_label: return test.FAIL("Unable to set user label", link) @@ -918,10 +764,10 @@ def test_13(self, test): return test.FAIL("Unexpected user label: " + str(label), link) # Reset user label - self._set_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id, old_user_label) + self.is12_utils.set_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id, old_user_label) # Check user label - label = self._get_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id) + label = self.is12_utils.get_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id) if label != old_user_label: if label == new_user_label: return test.FAIL("Unable to set user label", link) @@ -988,7 +834,7 @@ def do_get_member_descriptors_test(self, test, block, context=""): for search_condition in search_conditions: expected_members = block.get_member_descriptors(search_condition["recurse"]) - queried_members = self._get_member_descriptors(test, block.oid, search_condition["recurse"]) + queried_members = self.is12_utils.get_member_descriptors(test, block.oid, search_condition["recurse"]) if len(queried_members) != len(expected_members): raise NMOSTestException(test.FAIL(context @@ -1037,7 +883,7 @@ def do_find_member_by_path_test(self, test, block, context=""): # Get ground truth data from local device model object tree expected_member = block.find_members_by_path(role_path) - queried_members = self._find_members_by_path(test, block.oid, role_path) + queried_members = self.is12_utils.find_members_by_path(test, block.oid, role_path) for queried_member in queried_members: self._validate_schema(test, @@ -1099,12 +945,13 @@ def do_find_member_by_role_test(self, test, block, context=""): case_sensitive=condition["case_sensitive"], match_whole_string=condition["match_whole_string"], recurse=condition["recurse"]) - actual_results = self._find_members_by_role(test, - block.oid, - query_string, - case_sensitive=condition["case_sensitive"], - match_whole_string=condition["match_whole_string"], - recurse=condition["recurse"]) + actual_results = \ + self.is12_utils.find_members_by_role(test, + block.oid, + query_string, + case_sensitive=condition["case_sensitive"], + match_whole_string=condition["match_whole_string"], + recurse=condition["recurse"]) expected_results_oids = [m.oid for m in expected_results] @@ -1156,11 +1003,11 @@ def do_find_members_by_class_id_test(self, test, block, context=""): condition["include_derived"], condition["recurse"]) - actual_results = self._find_members_by_class_id(test, - block.oid, - class_id, - condition["include_derived"], - condition["recurse"]) + actual_results = self.is12_utils.find_members_by_class_id(test, + block.oid, + class_id, + condition["include_derived"], + condition["recurse"]) expected_results_oids = [m.oid for m in expected_results] @@ -1190,7 +1037,7 @@ def test_21(self, test): return test.PASS() - def do_error_test(self, test, command_handle, command_json, expected_status=None): + def do_error_test(self, test, command_json, expected_status=None): """Execute command with expected error status.""" # when expected_status = None checking of the status code is skipped # check the syntax of the error message according to is12_error @@ -1198,7 +1045,7 @@ def do_error_test(self, test, command_handle, command_json, expected_status=None try: self.create_ncp_socket(test) - self.is12_utils.send_command(test, command_handle, command_json) + self.is12_utils.send_command(test, command_json) return test.FAIL("Error expected") @@ -1230,99 +1077,84 @@ def do_error_test(self, test, command_handle, command_json, expected_status=None def test_23(self, test): """IS-12 Protocol Error: Node handles invalid command handle""" + command_json = self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, + self.is12_utils.METHOD_IDS["NCOBJECT"]["GENERIC_GET"], + {'id': self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']}) + # Use invalid handle invalid_command_handle = "NOT A HANDLE" - command_json = \ - self.is12_utils.create_generic_get_command_JSON(invalid_command_handle, - self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) + command_json['commands'][0]['handle'] = invalid_command_handle - return self.do_error_test(test, - invalid_command_handle, - command_json) + return self.do_error_test(test, command_json) def test_24(self, test): """IS-12 Protocol Error: Node handles invalid command type""" - command_handle = self.get_command_handle() command_json = \ - self.is12_utils.create_generic_get_command_JSON(command_handle, - self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) + self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, + self.is12_utils.METHOD_IDS["NCOBJECT"]["GENERIC_GET"], + {'id': self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']}) # Use invalid message type command_json['messageType'] = 7 - return self.do_error_test(test, - command_handle, - command_json) + return self.do_error_test(test, command_json) def test_25(self, test): """IS-12 Protocol Error: Node handles invalid JSON""" - command_handle = self.get_command_handle() # Use invalid JSON command_json = {'not_a': 'valid_command'} - return self.do_error_test(test, - command_handle, - command_json) + return self.do_error_test(test, command_json) def test_26(self, test): """MS-05-02 Error: Node handles invalid oid""" - command_handle = self.get_command_handle() # Use invalid oid invalid_oid = 999999999 command_json = \ - self.is12_utils.create_generic_get_command_JSON(command_handle, - invalid_oid, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) + self.is12_utils.create_command_JSON(invalid_oid, + self.is12_utils.METHOD_IDS["NCOBJECT"]["GENERIC_GET"], + {'id': self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']}) return self.do_error_test(test, - command_handle, command_json, expected_status=NcMethodStatus.BadOid) def test_27(self, test): """MS-05-02 Error: Node handles invalid property identifier""" - command_handle = self.get_command_handle() # Use invalid property id invalid_property_identifier = {'level': 1, 'index': 999} command_json = \ - self.is12_utils.create_generic_get_command_JSON(command_handle, - self.is12_utils.ROOT_BLOCK_OID, - invalid_property_identifier) - + self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, + self.is12_utils.METHOD_IDS["NCOBJECT"]["GENERIC_GET"], + {'id': invalid_property_identifier}) return self.do_error_test(test, - command_handle, command_json, expected_status=NcMethodStatus.PropertyNotImplemented) def test_28(self, test): """MS-05-02 Error: Node handles invalid method identifier""" - command_handle = self.get_command_handle() command_json = \ - self.is12_utils.create_generic_get_command_JSON(command_handle, - self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']) + self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, + self.is12_utils.METHOD_IDS["NCOBJECT"]["GENERIC_GET"], + {'id': self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']}) + # Use invalid method id invalid_method_id = {'level': 1, 'index': 999} command_json['commands'][0]['methodId'] = invalid_method_id return self.do_error_test(test, - command_handle, command_json, expected_status=NcMethodStatus.MethodNotImplemented) def test_29(self, test): """MS-05-02 Error: Node handles read only error""" - command_handle = self.get_command_handle() # Try to set a read only property command_json = \ - self.is12_utils.create_generic_set_command_JSON(command_handle, - self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['ROLE'], - "ROLE IS READ ONLY") + self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, + self.is12_utils.METHOD_IDS["NCOBJECT"]["GENERIC_SET"], + {'id': self.is12_utils.PROPERTY_IDS['NCOBJECT']['ROLE'], + 'value': "ROLE IS READ ONLY"}) return self.do_error_test(test, - command_handle, command_json, expected_status=NcMethodStatus.Readonly) From d185e9c19fe4da30fd211afde5f430ac49d1dd67 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Thu, 27 Jul 2023 17:20:16 +0100 Subject: [PATCH 18/45] replaced protocol def dict with enumerations --- nmostesting/IS12Utils.py | 138 +++++++++++++++---------------- nmostesting/suites/IS1201Test.py | 63 +++++++------- 2 files changed, 99 insertions(+), 102 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index 752ff1ed..bb180633 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -17,7 +17,7 @@ import json import time -from enum import IntEnum +from enum import IntEnum, Enum from itertools import takewhile, dropwhile from jsonschema import FormatChecker, SchemaError, validate, ValidationError from .Config import WS_MESSAGE_TIMEOUT @@ -60,72 +60,70 @@ class NcDatatypeType(IntEnum): Enum = 3 # Enum datatype +class NcObjectProperties(Enum): + CLASS_ID = {'level': 1, 'index': 1} + OID = {'level': 1, 'index': 2} + CONSTANT_OID = {'level': 1, 'index': 3} + OWNER = {'level': 1, 'index': 4} + ROLE = {'level': 1, 'index': 5} + USER_LABEL = {'level': 1, 'index': 6} + TOUCHPOINTS = {'level': 1, 'index': 7} + RUNTIME_PROPERTY_CONSTRAINTS = {'level': 1, 'index': 8} + + +class NcObjectMethods(Enum): + GENERIC_GET = {'level': 1, 'index': 1} + GENERIC_SET = {'level': 1, 'index': 2} + GET_SEQUENCE_ITEM = {'level': 1, 'index': 3} + SET_SEQUENCE_ITEM = {'level': 1, 'index': 4} + ADD_SEQUENCE_ITEM = {'level': 1, 'index': 5} + REMOVE_SEQUENCE_ITEM = {'level': 1, 'index': 6} + GET_SEQUENCE_LENGTH = {'level': 1, 'index': 7} + + +class NcBlockProperties(Enum): + ENABLED = {'level': 2, 'index': 1} + MEMBERS = {'level': 2, 'index': 2} + + +class NcBlockMethods(Enum): + GET_MEMBERS_DESCRIPTOR = {'level': 2, 'index': 1} + FIND_MEMBERS_BY_PATH = {'level': 2, 'index': 2} + FIND_MEMBERS_BY_ROLE = {'level': 2, 'index': 3} + FIND_MEMBERS_BY_CLASS_ID = {'level': 2, 'index': 4} + + +class NcClassManagerProperties(Enum): + CONTROL_CLASSES = {'level': 3, 'index': 1} + DATATYPES = {'level': 3, 'index': 2} + + +class NcClassManagerMethods(Enum): + GET_CONTROL_CLASS = {'level': 3, 'index': 1} + + +class NcDeviceManagerProperties(Enum): + NCVERSION = {'level': 3, 'index': 1} + + +class StandardClassIds(Enum): + NCOBJECT = [1] + NCBLOCK = [1, 1] + NCWORKER = [1, 2] + NCMANAGER = [1, 3] + NCDEVICEMANAGER = [1, 3, 1] + NCCLASSMANAGER = [1, 3, 2] + + class IS12Utils(NMOSUtils): def __init__(self, url, spec_path, spec_branch): NMOSUtils.__init__(self, url) self.spec_branch = spec_branch - self.protocol_definitions() self.load_is12_schemas(spec_path) + self.ROOT_BLOCK_OID = 1 self.ncp_websocket = None self.command_handle = 0 - def protocol_definitions(self): - self.ROOT_BLOCK_OID = 1 - - self.METHOD_IDS = { - 'NCOBJECT': { - 'GENERIC_GET': {'level': 1, 'index': 1}, - 'GENERIC_SET': {'level': 1, 'index': 2}, - 'GET_SEQUENCE_ITEM': {'level': 1, 'index': 3}, - 'SET_SEQUENCE_ITEM': {'level': 1, 'index': 4}, - 'ADD_SEQUENCE_ITEM': {'level': 1, 'index': 5}, - 'REMOVE_SEQUENCE_ITEM': {'level': 1, 'index': 6}, - 'GET_SEQUENCE_LENGTH': {'level': 1, 'index': 7} - }, - 'NCBLOCK': { - 'GET_MEMBERS_DESCRIPTOR': {'level': 2, 'index': 1}, - 'FIND_MEMBERS_BY_PATH': {'level': 2, 'index': 2}, - 'FIND_MEMBERS_BY_ROLE': {'level': 2, 'index': 3}, - 'FIND_MEMBERS_BY_CLASS_ID': {'level': 2, 'index': 4} - }, - 'NCCLASSMANAGER': { - 'GET_CONTROL_CLASS': {'level': 3, 'index': 1} - }, - } - - self.PROPERTY_IDS = { - 'NCOBJECT': { - 'CLASS_ID': {'level': 1, 'index': 1}, - 'OID': {'level': 1, 'index': 2}, - 'CONSTANT_OID': {'level': 1, 'index': 3}, - 'OWNER': {'level': 1, 'index': 4}, - 'ROLE': {'level': 1, 'index': 5}, - 'USER_LABEL': {'level': 1, 'index': 6}, - 'TOUCHPOINTS': {'level': 1, 'index': 7}, - 'RUNTIME_PROPERTY_CONSTRAINTS': {'level': 1, 'index': 8} - }, - 'NCBLOCK': { - 'ENABLED': {'level': 2, 'index': 1}, - 'MEMBERS': {'level': 2, 'index': 2} - }, - 'NCCLASSMANAGER': { - 'CONTROL_CLASSES': {'level': 3, 'index': 1}, - 'DATATYPES': {'level': 3, 'index': 2} - }, - 'NCDEVICEMANAGER': { - 'NCVERSION': {'level': 3, 'index': 1} - } - } - - self.CLASS_IDS = { - 'NCOBJECT': [1], - 'NCBLOCK': [1, 1], - 'NCWORKER': [1, 2], - 'NCMANAGER': [1, 3], - 'NCDEVICEMANAGER': [1, 3, 1], - 'NCCLASSMANAGER': [1, 3, 2] - } - def load_is12_schemas(self, spec_path): """Load datatype and control class decriptors and create datatype JSON schemas""" # Load IS-12 schemas @@ -259,61 +257,61 @@ def _execute_command(self, test, oid, method_id, arguments): def get_property(self, test, oid, property_id): """Get property from object. Raises NMOSTestException on error""" return self._execute_command(test, oid, - self.METHOD_IDS["NCOBJECT"]["GENERIC_GET"], + NcObjectMethods.GENERIC_GET.value, {'id': property_id})["value"] def set_property(self, test, oid, property_id, argument): """Get property from object. Raises NMOSTestException on error""" return self._execute_command(test, oid, - self.METHOD_IDS["NCOBJECT"]["GENERIC_SET"], + NcObjectMethods.GENERIC_SET.value, {'id': property_id, 'value': argument}) def get_sequence_item(self, test, oid, property_id, index): """Get value from sequence property. Raises NMOSTestException on error""" return self._execute_command(test, oid, - self.METHOD_IDS["NCOBJECT"]["GET_SEQUENCE_ITEM"], + NcObjectMethods.GET_SEQUENCE_ITEM.value, {'id': property_id, 'index': index})["value"] def set_sequence_item(self, test, oid, property_id, index, value): """Add value to a sequence property. Raises NMOSTestException on error""" return self._execute_command(test, oid, - self.METHOD_IDS["NCOBJECT"]["SET_SEQUENCE_ITEM"], + NcObjectMethods.SET_SEQUENCE_ITEM.value, {'id': property_id, 'index': index, 'value': value}) def add_sequence_item(self, test, oid, property_id, value): """Add value to a sequence property. Raises NMOSTestException on error""" return self._execute_command(test, oid, - self.METHOD_IDS["NCOBJECT"]["ADD_SEQUENCE_ITEM"], + NcObjectMethods.ADD_SEQUENCE_ITEM.value, {'id': property_id, 'value': value}) def remove_sequence_item(self, test, oid, property_id, index): """Get value from sequence property. Raises NMOSTestException on error""" return self._execute_command(test, oid, - self.METHOD_IDS["NCOBJECT"]["REMOVE_SEQUENCE_ITEM"], + NcObjectMethods.REMOVE_SEQUENCE_ITEM.value, {'id': property_id, 'index': index}) def get_sequence_length(self, test, oid, property_id): """Get value from sequence property. Raises NMOSTestException on error""" return self._execute_command(test, oid, - self.METHOD_IDS["NCOBJECT"]["GET_SEQUENCE_LENGTH"], + NcObjectMethods.GET_SEQUENCE_LENGTH.value, {'id': property_id})["value"] def get_member_descriptors(self, test, oid, recurse): """Get BlockMemberDescritors for this block. Raises NMOSTestException on error""" return self._execute_command(test, oid, - self.METHOD_IDS["NCBLOCK"]["GET_MEMBERS_DESCRIPTOR"], + NcBlockMethods.GET_MEMBERS_DESCRIPTOR.value, {'recurse': recurse})["value"] def find_members_by_path(self, test, oid, role_path): """Query members based on role path. Raises NMOSTestException on error""" return self._execute_command(test, oid, - self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_PATH"], + NcBlockMethods.FIND_MEMBERS_BY_PATH.value, {'path': role_path})["value"] def find_members_by_role(self, test, oid, role, case_sensitive, match_whole_string, recurse): """Query members based on role. Raises NMOSTestException on error""" return self._execute_command(test, oid, - self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_ROLE"], + NcBlockMethods.FIND_MEMBERS_BY_ROLE.value, {'role': role, 'caseSensitive': case_sensitive, 'matchWholeString': match_whole_string, @@ -322,7 +320,7 @@ def find_members_by_role(self, test, oid, role, case_sensitive, match_whole_stri def find_members_by_class_id(self, test, oid, class_id, include_derived, recurse): """Query members based on class id. Raises NMOSTestException on error""" return self._execute_command(test, oid, - self.METHOD_IDS["NCBLOCK"]["FIND_MEMBERS_BY_CLASS_ID"], + NcBlockMethods.FIND_MEMBERS_BY_CLASS_ID.value, {'classId': class_id, 'includeDerived': include_derived, 'recurse': recurse})["value"] diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 01f481e7..4410c6d5 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -19,7 +19,8 @@ from jsonschema import ValidationError, SchemaError from ..GenericTest import GenericTest, NMOSTestException -from ..IS12Utils import IS12Utils, NcMethodStatus, NcObject +from ..IS12Utils import IS12Utils, NcObject, NcMethodStatus, NcBlockProperties, NcObjectMethods, NcObjectProperties, \ + NcClassManagerProperties, NcDeviceManagerProperties, StandardClassIds from ..TestHelper import load_resolved_schema from ..TestResult import Test @@ -28,9 +29,6 @@ MS05_API_KEY = "controlframework" FEATURE_SETS_KEY = "featuresets" -CLASS_MANAGER_CLS_ID = "1.3.2" -DEVICE_MANAGER_CLS_ID = "1.3.1" - class IS1201Test(GenericTest): @@ -140,11 +138,12 @@ def create_ncp_socket(self, test): """Create a WebSocket client connection to Node under test. Raises NMOSTestException on error""" self.is12_utils.open_ncp_websocket(test, self.apis[CONTROL_API_KEY]["url"]) - def get_manager(self, test, class_id_str): + def get_manager(self, test, class_id): """Get Manager from Root Block. Returns [Manager]. Raises NMOSTestException on error""" + class_id_str = ".".join(map(str, class_id)) response = self.is12_utils.get_property(test, self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCBLOCK']['MEMBERS']) + NcBlockProperties.MEMBERS.value) manager_found = False manager = None @@ -286,15 +285,15 @@ def auto_tests(self): self.create_ncp_socket(test) - class_manager = self.get_manager(test, CLASS_MANAGER_CLS_ID) + class_manager = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) results += self.validate_model_definitions(class_manager['oid'], - self.is12_utils.PROPERTY_IDS['NCCLASSMANAGER']['CONTROL_CLASSES'], + NcClassManagerProperties.CONTROL_CLASSES.value, 'NcClassDescriptor', self.classes_descriptors) results += self.validate_model_definitions(class_manager['oid'], - self.is12_utils.PROPERTY_IDS['NCCLASSMANAGER']['DATATYPES'], + NcClassManagerProperties.DATATYPES.value, 'NcDatatypeDescriptor', self.datatype_descriptors) return results @@ -333,7 +332,7 @@ def test_03(self, test): role = self.is12_utils.get_property(test, self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.PROPERTY_IDS['NCOBJECT']['ROLE']) + NcObjectProperties.ROLE.value) if role != "root": return test.FAIL("Unexpected role in Root Block: " + role, @@ -350,7 +349,7 @@ def test_04(self, test): self.create_ncp_socket(test) - self.get_manager(test, CLASS_MANAGER_CLS_ID) + self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) return test.PASS() @@ -473,7 +472,7 @@ def check_touchpoints(self, test, oid, datatype_schemas, context): """Touchpoint checks""" touchpoints = self.is12_utils.get_property(test, oid, - self.is12_utils.PROPERTY_IDS["NCOBJECT"]["TOUCHPOINTS"]) + NcObjectProperties.TOUCHPOINTS.value) if touchpoints is not None: self.touchpoints_metadata["checked"] = True try: @@ -490,7 +489,7 @@ def check_touchpoints(self, test, oid, datatype_schemas, context): self.touchpoints_metadata["error_msg"] = context + str(e.args[0].detail) def validate_block(self, test, block_id, class_descriptors, datatype_schemas, block, context=""): - response = self.is12_utils.get_property(test, block_id, self.is12_utils.PROPERTY_IDS['NCBLOCK']['MEMBERS']) + response = self.is12_utils.get_property(test, block_id, NcBlockProperties.MEMBERS.value) role_cache = [] manager_cache = [] @@ -549,21 +548,21 @@ def validate_device_model(self, test): if not self.device_model_validated: self.create_ncp_socket(test) - class_manager = self.get_manager(test, CLASS_MANAGER_CLS_ID) + class_manager = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) class_descriptors = \ self.get_class_manager_descriptors(test, class_manager['oid'], - self.is12_utils.PROPERTY_IDS['NCCLASSMANAGER']['CONTROL_CLASSES']) + NcClassManagerProperties.CONTROL_CLASSES.value) datatype_descriptors = \ self.get_class_manager_descriptors(test, class_manager['oid'], - self.is12_utils.PROPERTY_IDS['NCCLASSMANAGER']['DATATYPES']) + NcClassManagerProperties.DATATYPES.value) # Create JSON schemas for the queried datatypes datatype_schemas = self.generate_json_schemas( datatype_descriptors=datatype_descriptors, schema_path=os.path.join(self.apis[CONTROL_API_KEY]["spec_path"], 'APIs/tmp_schemas/')) - self.root_block = NcObject(self.is12_utils.CLASS_IDS["NCBLOCK"], self.is12_utils.ROOT_BLOCK_OID, "root") + self.root_block = NcObject(StandardClassIds.NCBLOCK.value, self.is12_utils.ROOT_BLOCK_OID, "root") self.validate_block(test, self.is12_utils.ROOT_BLOCK_OID, @@ -670,10 +669,10 @@ def test_10(self, test): self.create_ncp_socket(test) - device_manager = self.get_manager(test, DEVICE_MANAGER_CLS_ID) + device_manager = self.get_manager(test, StandardClassIds.NCDEVICEMANAGER.value) # Check MS-05-02 Version - property_id = self.is12_utils.PROPERTY_IDS['NCDEVICEMANAGER']['NCVERSION'] + property_id = NcDeviceManagerProperties.NCVERSION.value version = self.is12_utils.get_property(test, device_manager['oid'], property_id) @@ -747,7 +746,7 @@ def test_13(self, test): # Attempt to set labels self.create_ncp_socket(test) - property_id = self.is12_utils.PROPERTY_IDS['NCOBJECT']['USER_LABEL'] + property_id = NcObjectProperties.USER_LABEL.value old_user_label = self.is12_utils.get_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id) @@ -989,7 +988,7 @@ def do_find_members_by_class_id_test(self, test, block, context=""): if self.is12_utils.is_block(child_object.class_id): self.do_find_members_by_class_id_test(test, child_object, context + block.role + ": ") - class_ids = [class_id for _, class_id in self.is12_utils.CLASS_IDS.items()] + class_ids = [e.value for e in StandardClassIds] truth_table = IS12Utils.sampled_list(list(product([False, True], repeat=2))) search_conditions = [] @@ -1078,8 +1077,8 @@ def test_23(self, test): """IS-12 Protocol Error: Node handles invalid command handle""" command_json = self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.METHOD_IDS["NCOBJECT"]["GENERIC_GET"], - {'id': self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']}) + NcObjectMethods.GENERIC_GET.value, + {'id': NcObjectProperties.OID.value}) # Use invalid handle invalid_command_handle = "NOT A HANDLE" @@ -1091,8 +1090,8 @@ def test_24(self, test): """IS-12 Protocol Error: Node handles invalid command type""" command_json = \ self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.METHOD_IDS["NCOBJECT"]["GENERIC_GET"], - {'id': self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']}) + NcObjectMethods.GENERIC_GET.value, + {'id': NcObjectProperties.OID.value}) # Use invalid message type command_json['messageType'] = 7 @@ -1112,8 +1111,8 @@ def test_26(self, test): invalid_oid = 999999999 command_json = \ self.is12_utils.create_command_JSON(invalid_oid, - self.is12_utils.METHOD_IDS["NCOBJECT"]["GENERIC_GET"], - {'id': self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']}) + NcObjectMethods.GENERIC_GET.value, + {'id': NcObjectProperties.OID.value}) return self.do_error_test(test, command_json, @@ -1125,7 +1124,7 @@ def test_27(self, test): invalid_property_identifier = {'level': 1, 'index': 999} command_json = \ self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.METHOD_IDS["NCOBJECT"]["GENERIC_GET"], + NcObjectMethods.GENERIC_GET.value, {'id': invalid_property_identifier}) return self.do_error_test(test, command_json, @@ -1135,8 +1134,8 @@ def test_28(self, test): """MS-05-02 Error: Node handles invalid method identifier""" command_json = \ self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.METHOD_IDS["NCOBJECT"]["GENERIC_GET"], - {'id': self.is12_utils.PROPERTY_IDS['NCOBJECT']['OID']}) + NcObjectMethods.GENERIC_GET.value, + {'id': NcObjectProperties.OID.value}) # Use invalid method id invalid_method_id = {'level': 1, 'index': 999} @@ -1151,8 +1150,8 @@ def test_29(self, test): # Try to set a read only property command_json = \ self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, - self.is12_utils.METHOD_IDS["NCOBJECT"]["GENERIC_SET"], - {'id': self.is12_utils.PROPERTY_IDS['NCOBJECT']['ROLE'], + NcObjectMethods.GENERIC_SET.value, + {'id': NcObjectProperties.ROLE.value, 'value': "ROLE IS READ ONLY"}) return self.do_error_test(test, From 8c7e2958d41a6dad842f8ea15852bbea0fd5c4f2 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Fri, 28 Jul 2023 12:07:43 +0100 Subject: [PATCH 19/45] Renamed and reordered tests --- nmostesting/suites/IS1201Test.py | 180 +++++++++++++++---------------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 4410c6d5..c0420236 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -299,7 +299,7 @@ def auto_tests(self): return results def test_01(self, test): - """At least one Device is showing an IS-12 control advertisement matching the API under test""" + """Control Endpoint: Node under test advertising IS-12 control endpoint matching API under test""" # Referencing the Google sheet # IS-12 (1) Control endpoint advertised in Node endpoint's Device controls array @@ -313,7 +313,7 @@ def test_01(self, test): ) def test_02(self, test): - """WebSocket successfully opened on advertised urn:x-nmos:control:ncp endpoint""" + """WebSocket: successfully opened endpoint""" # Referencing the Google sheet # IS-12 (2) WebSocket successfully opened on advertised urn:x-nmos:control:ncp endpoint @@ -322,7 +322,7 @@ def test_02(self, test): return test.PASS() def test_03(self, test): - """Root Block Exists with correct oid and Role""" + """Device Model: Root Block exists with correct oid and role""" # Referencing the Google sheet # MS-05-02 (44) Root Block must exist # MS-05-02 (45) Verify oID and role of Root Block @@ -342,17 +342,6 @@ def test_03(self, test): return test.PASS() - def test_04(self, test): - """Class Manager exists in Root Block""" - # Referencing the Google sheet - # MS-05-02 (40) Class manager exists in root - - self.create_ncp_socket(test) - - self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) - - return test.PASS() - def validate_property_type(self, test, value, type, is_nullable, datatype_schemas, context=""): if value is None: if is_nullable: @@ -573,8 +562,8 @@ def validate_device_model(self, test): self.device_model_validated = True return - def test_05(self, test): - """Validate device model properties against discovered classes and datatypes""" + def test_04(self, test): + """Device Model: check Device Model against classes and datatypes discovered from Class Manager""" # Referencing the Google sheet # MS-05-02 (34) All workers MUST inherit from NcWorker # MS-05-02 (35) All managers MUST inherit from NcManager @@ -582,8 +571,8 @@ def test_05(self, test): return test.PASS() - def test_06(self, test): - """Device model roles are unique within a containing Block""" + def test_05(self, test): + """Device Model: roles are unique within a containing Block""" # Referencing the Google sheet # MS-05-02 (59) The role of an object MUST be unique within its containing Block. # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html @@ -602,8 +591,8 @@ def test_06(self, test): return test.PASS() - def test_07(self, test): - """Device model oids are globally unique""" + def test_06(self, test): + """Device Model: oids are globally unique""" # Referencing the Google sheet # MS-05-02 (60) Object ids (oid property) MUST uniquely identity objects in the device model. # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html @@ -622,8 +611,58 @@ def test_07(self, test): return test.PASS() + def test_07(self, test): + """Device Model: non-standard classes contain an authority key""" + # Referencing the Google sheet + # MS-05-02 (72) Non-standard Classes NcClassId + # MS-05-02 (73) Organization Identifier + # For organizations which own a unique CID or OUI the authority key MUST be the organization + # identifier as an integer which MUST be negated. + # For organizations which do not own a unique CID or OUI the authority key MUST be 0 + # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html + + try: + self.validate_device_model(test) + except NMOSTestException as e: + # Couldn't validate model so can't perform test + return test.UNCLEAR(e.args[0].detail, e.args[0].link) + + if self.organization_metadata["error"]: + return test.FAIL(self.organization_metadata["error_msg"], + "https://specs.amwa.tv/ms-05-02/branches/{}" + "/docs/Framework.html#ncclassid" + .format(self.apis[MS05_API_KEY]["spec_branch"])) + + if not self.organization_metadata["checked"]: + return test.UNCLEAR("No non-standard classes found.") + + return test.PASS() + def test_08(self, test): - """Managers must be members of the Root Block""" + """Device Model: check touchpoint datatypes""" + # Referencing the Google sheet + # MS-05-02 (39) For general NMOS contexts (IS-04, IS-05 and IS-07) the NcTouchpointNmos datatype MUST be used + # which has a resource of type NcTouchpointResourceNmos. + # For IS-08 Audio Channel Mapping the NcTouchpointResourceNmosChannelMapping datatype MUST be used + # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#touchpoints + try: + self.validate_device_model(test) + except NMOSTestException as e: + # Couldn't validate model so can't perform test + return test.UNCLEAR(e.args[0].detail, e.args[0].link) + + if self.touchpoints_metadata["error"]: + return test.FAIL(self.touchpoints_metadata["error_msg"], + "https://specs.amwa.tv/ms-05-02/branches/{}" + "/docs/NcObject.html#touchpoints" + .format(self.apis[MS05_API_KEY]["spec_branch"])) + + if not self.touchpoints_metadata["checked"]: + return test.UNCLEAR("No Touchpoints found.") + return test.PASS() + + def test_09(self, test): + """Managers: managers are members of the Root Block""" # Referencing the Google sheet # MS-05-02 (36) All managers MUST always exist as members in the Root Block and have a fixed role. # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html @@ -642,8 +681,8 @@ def test_08(self, test): return test.PASS() - def test_09(self, test): - """Managers are singletons""" + def test_10(self, test): + """Managers: managers are singletons""" # Referencing the Google sheet # MS-05-02 (63) Managers are singleton (MUST only be instantiated once) classes. # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html @@ -662,8 +701,19 @@ def test_09(self, test): return test.PASS() - def test_10(self, test): - """Device Manager exists in Root Block""" + def test_11(self, test): + """Managers: Class Manager exists with correct role""" + # Referencing the Google sheet + # MS-05-02 (40) Class manager exists in root + + self.create_ncp_socket(test) + + self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) + + return test.PASS() + + def test_12(self, test): + """Managers: Device Manager exists with correct Role""" # Referencing the Google sheet # MS-05-02 (37) A minimal device implementation MUST have a device manager in the Root Block. @@ -683,58 +733,8 @@ def test_10(self, test): return test.PASS() - def test_11(self, test): - """Non-standard classes contain an authority key""" - # Referencing the Google sheet - # MS-05-02 (72) Non-standard Classes NcClassId - # MS-05-02 (73) Organization Identifier - # For organizations which own a unique CID or OUI the authority key MUST be the organization - # identifier as an integer which MUST be negated. - # For organizations which do not own a unique CID or OUI the authority key MUST be 0 - # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html - - try: - self.validate_device_model(test) - except NMOSTestException as e: - # Couldn't validate model so can't perform test - return test.UNCLEAR(e.args[0].detail, e.args[0].link) - - if self.organization_metadata["error"]: - return test.FAIL(self.organization_metadata["error_msg"], - "https://specs.amwa.tv/ms-05-02/branches/{}" - "/docs/Framework.html#ncclassid" - .format(self.apis[MS05_API_KEY]["spec_branch"])) - - if not self.organization_metadata["checked"]: - return test.UNCLEAR("No non-standard classes found.") - - return test.PASS() - - def test_12(self, test): - """Validate touchpoints""" - # Referencing the Google sheet - # MS-05-02 (39) For general NMOS contexts (IS-04, IS-05 and IS-07) the NcTouchpointNmos datatype MUST be used - # which has a resource of type NcTouchpointResourceNmos. - # For IS-08 Audio Channel Mapping the NcTouchpointResourceNmosChannelMapping datatype MUST be used - # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#touchpoints - try: - self.validate_device_model(test) - except NMOSTestException as e: - # Couldn't validate model so can't perform test - return test.UNCLEAR(e.args[0].detail, e.args[0].link) - - if self.touchpoints_metadata["error"]: - return test.FAIL(self.touchpoints_metadata["error_msg"], - "https://specs.amwa.tv/ms-05-02/branches/{}" - "/docs/NcObject.html#touchpoints" - .format(self.apis[MS05_API_KEY]["spec_branch"])) - - if not self.touchpoints_metadata["checked"]: - return test.UNCLEAR("No Touchpoints found.") - return test.PASS() - def test_13(self, test): - """NcObject method: Get/Set""" + """NcObject: check Get/Set method""" # Referencing the Google sheet # MS-05-02 (39) Generic getter and setter # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#generic-getter-and-setter @@ -776,7 +776,7 @@ def test_13(self, test): return test.PASS() def test_14(self, test): - """NcObject method: GetSequenceItem""" + """NcObject: check GetSequenceItem method""" try: self.validate_device_model(test) except NMOSTestException as e: @@ -792,22 +792,22 @@ def test_14(self, test): return test.PASS() def test_15(self, test): - """NcObject method: SetSequenceItem""" + """NcObject: check SetSequenceItem method""" return test.DISABLED() def test_16(self, test): - """NcObject method: AddSequenceItem""" + """NcObject: check AddSequenceItem method""" return test.DISABLED() def test_17(self, test): - """NcObject method: RemoveSequenceItem""" + """NcObject: check RemoveSequenceItem method""" return test.DISABLED() - def test_17_1(self, test): - """NcObject method: GetSequenceLength""" + def test_18(self, test): + """NcObject: check GetSequenceLength method""" try: self.validate_device_model(test) except NMOSTestException as e: @@ -857,8 +857,8 @@ def do_get_member_descriptors_test(self, test, block, context=""): + block.role + ": Unsuccessful attempt to get member descriptors.")) - def test_18(self, test): - """NcBlock method: GetMemberDescriptors""" + def test_19(self, test): + """NcBlock: check GetMemberDescriptors method""" try: self.validate_device_model(test) except NMOSTestException as e: @@ -905,8 +905,8 @@ def do_find_member_by_path_test(self, test, block, context=""): + ": Unsuccessful attempt to find member by role path: " + str(role_path))) - def test_19(self, test): - """NcBlock method: FindMemberByPath""" + def test_20(self, test): + """NcBlock: check FindMemberByPath method""" try: self.validate_device_model(test) except NMOSTestException as e: @@ -969,8 +969,8 @@ def do_find_member_by_role_test(self, test, block, context=""): + ": Unexpected search result. " + str(actual_result))) - def test_20(self, test): - """NcBlock method: FindMembersByRole""" + def test_21(self, test): + """NcBlock: check FindMembersByRole method""" try: self.validate_device_model(test) except NMOSTestException as e: @@ -1024,8 +1024,8 @@ def do_find_members_by_class_id_test(self, test, block, context=""): + block.role + ": Unexpected search result. " + str(actual_result))) - def test_21(self, test): - """NcBlock method: FindMembersByClassId""" + def test_22(self, test): + """NcBlock: check FindMembersByClassId method""" try: self.validate_device_model(test) except NMOSTestException as e: From debcbcb343937516db45e54429149d2199f4fb6a Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Fri, 28 Jul 2023 14:46:22 +0100 Subject: [PATCH 20/45] Set GetSequenceLength checked flag --- nmostesting/suites/IS1201Test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index c0420236..162a5c66 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -386,6 +386,7 @@ def check_get_sequence_item(self, test, oid, sequence_values, property_metadata, def check_get_sequence_length(self, test, oid, sequence_values, property_metadata, context=""): try: + self.get_sequence_length_metadata["checked"] = True length = self.is12_utils.get_sequence_length(test, oid, property_metadata['id']) if length == len(sequence_values): From 264453fd1950caf7fef4c3860d33afc71d86f473 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Fri, 28 Jul 2023 14:42:40 +0100 Subject: [PATCH 21/45] Subscription message (cherry picked from commit c963854efde2ba4b3c9fcf034ff1ead212fc7ca1) --- nmostesting/IS12Utils.py | 52 +++++++++++++++++++++++++------- nmostesting/suites/IS1201Test.py | 9 ++++++ 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index bb180633..321113fb 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -128,7 +128,7 @@ def load_is12_schemas(self, spec_path): """Load datatype and control class decriptors and create datatype JSON schemas""" # Load IS-12 schemas self.schemas = {} - schema_names = ['error-message', 'command-response-message'] + schema_names = ["error-message", "command-response-message", "subscription-response-message"] for schema_name in schema_names: self.schemas[schema_name] = load_resolved_schema(spec_path, schema_name + ".json") @@ -172,6 +172,17 @@ def validate_is12_schema(self, test, payload, schema_name, context=""): return + def message_type_to_schema_name(self, type): + """Convert MessageType to corresponding JSON schema name""" + + types = { + MessageTypes.CommandResponse: "command-response-message", + MessageTypes.SubscriptionResponse: "subscription-response-message", + MessageTypes.Error: "error-message", + } + + return types.get(type, False) + def send_command(self, test, command_json): """Send command to Node under test. Returns [command response]. Raises NMOSTestException on error""" # Referencing the Google sheet @@ -204,27 +215,33 @@ def send_command(self, test, command_json): for message in messages: parsed_message = json.loads(message) + if self.message_type_to_schema_name(parsed_message["messageType"]): + self.validate_is12_schema( + test, + parsed_message, + self.message_type_to_schema_name(parsed_message["messageType"]), + context=self.message_type_to_schema_name(parsed_message["messageType"]) + ": ") + else: + raise NMOSTestException(test.FAIL("Unrecognised message type: " + parsed_message["messageType"], + "https://specs.amwa.tv/is-12/branches/{}" + "/docs/Protocol_messaging.html#command-message-type" + .format(self.spec_branch))) + if parsed_message["messageType"] == MessageTypes.CommandResponse: - self.validate_is12_schema(test, - parsed_message, - "command-response-message", - context="command-response-message: ") responses = parsed_message["responses"] for response in responses: if response["handle"] == command_handle: if response["result"]["status"] != NcMethodStatus.OK: raise NMOSTestException(test.FAIL(response["result"])) results.append(response) + if parsed_message["messageType"] == MessageTypes.SubscriptionResponse: + results.append(parsed_message["subscriptions"]) if parsed_message["messageType"] == MessageTypes.Error: - self.validate_is12_schema(test, - parsed_message, - "error-message", - context="error-message: ") raise NMOSTestException(test.FAIL(parsed_message, "https://specs.amwa.tv/is-12/branches/{}" "/docs/Protocol_messaging.html#error-messages" .format(self.spec_branch))) if len(results) == 0: - raise NMOSTestException(test.FAIL("No Command Message Response received.", + raise NMOSTestException(test.FAIL("No Message Response received.", "https://specs.amwa.tv/is-12/branches/{}" "/docs/Protocol_messaging.html#command-message-type" .format(self.spec_branch))) @@ -235,7 +252,7 @@ def send_command(self, test, command_json): return results[0] def create_command_JSON(self, oid, method_id, arguments): - """Create command JSON for generic get of a property""" + """for sending over websocket""" self.command_handle += 1 return { 'messageType': MessageTypes.Command, @@ -325,6 +342,19 @@ def find_members_by_class_id(self, test, oid, class_id, include_derived, recurse 'includeDerived': include_derived, 'recurse': recurse})["value"] + def create_subscription_JSON(self, subscriptions): + """for sending over websocket""" + return { + 'messageType': MessageTypes.Subscription, + 'subscriptions': subscriptions + } + + def update_subscritions(self, test, subscriptions): + """update Nodes subscriptions""" + command_JSON = self.create_subscription_JSON(subscriptions) + response = self.send_command(test, command_JSON) + return response + def model_primitive_to_JSON(self, type): """Convert MS-05 primitive type to corresponding JSON type""" diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 162a5c66..7829ff87 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -1158,3 +1158,12 @@ def test_29(self, test): return self.do_error_test(test, command_json, expected_status=NcMethodStatus.Readonly) + + + def test_30(self, test): + """Subscriptions""" + self.create_ncp_socket(test) + + self.is12_utils.update_subscritions(test, [1]) + + return test.PASS() \ No newline at end of file From 46b2d1762fc6209abda264492f156a94b0cb8e46 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Fri, 28 Jul 2023 16:56:17 +0100 Subject: [PATCH 22/45] Subscriptions and notifications --- nmostesting/IS12Utils.py | 41 ++++++++++++++++++++- nmostesting/suites/IS1201Test.py | 61 ++++++++++++++++++++++++++++---- 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index 321113fb..b7819de5 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -51,6 +51,11 @@ class NcMethodStatus(IntEnum): PropertyNotImplemented = 502 NotReady = 503 Timeout = 504 + UNKNOWN = 9999 + + @classmethod + def _missing_(cls, _): + return cls.UNKNOWN class NcDatatypeType(IntEnum): @@ -60,6 +65,18 @@ class NcDatatypeType(IntEnum): Enum = 3 # Enum datatype +class NcPropertyChangeType(IntEnum): + ValueChanged = 0 # Current value changed + SequenceItemAdded = 1 # Sequence item added + SequenceItemChanged = 2 # Sequence item changed + SequenceItemRemoved = 3 # Sequence item removed + UNKNOWN = 9999 + + @classmethod + def _missing_(cls, _): + return cls.UNKNOWN + + class NcObjectProperties(Enum): CLASS_ID = {'level': 1, 'index': 1} OID = {'level': 1, 'index': 2} @@ -69,6 +86,11 @@ class NcObjectProperties(Enum): USER_LABEL = {'level': 1, 'index': 6} TOUCHPOINTS = {'level': 1, 'index': 7} RUNTIME_PROPERTY_CONSTRAINTS = {'level': 1, 'index': 8} + UNKNOWN = {'level': 9999, 'index': 9999} + + @classmethod + def _missing_(cls, _): + return cls.UNKNOWN class NcObjectMethods(Enum): @@ -81,6 +103,10 @@ class NcObjectMethods(Enum): GET_SEQUENCE_LENGTH = {'level': 1, 'index': 7} +class NcObjectEvents(Enum): + PROPERTY_CHANGED = {'level': 1, 'index': 1} + + class NcBlockProperties(Enum): ENABLED = {'level': 2, 'index': 1} MEMBERS = {'level': 2, 'index': 2} @@ -123,12 +149,16 @@ def __init__(self, url, spec_path, spec_branch): self.ROOT_BLOCK_OID = 1 self.ncp_websocket = None self.command_handle = 0 + self.notifications = [] def load_is12_schemas(self, spec_path): """Load datatype and control class decriptors and create datatype JSON schemas""" # Load IS-12 schemas self.schemas = {} - schema_names = ["error-message", "command-response-message", "subscription-response-message"] + schema_names = ["error-message", + "command-response-message", + "subscription-response-message", + "notification-message"] for schema_name in schema_names: self.schemas[schema_name] = load_resolved_schema(spec_path, schema_name + ".json") @@ -177,6 +207,7 @@ def message_type_to_schema_name(self, type): types = { MessageTypes.CommandResponse: "command-response-message", + MessageTypes.Notification: "notification-message", MessageTypes.SubscriptionResponse: "subscription-response-message", MessageTypes.Error: "error-message", } @@ -236,6 +267,8 @@ def send_command(self, test, command_json): results.append(response) if parsed_message["messageType"] == MessageTypes.SubscriptionResponse: results.append(parsed_message["subscriptions"]) + if parsed_message["messageType"] == MessageTypes.Notification: + self.notifications += parsed_message["notifications"] if parsed_message["messageType"] == MessageTypes.Error: raise NMOSTestException(test.FAIL(parsed_message, "https://specs.amwa.tv/is-12/branches/{}" "/docs/Protocol_messaging.html#error-messages" @@ -251,6 +284,12 @@ def send_command(self, test, command_json): return results[0] + def get_notifications(self): + return self.notifications + + def reset_notifications(self): + self.notifications = [] + def create_command_JSON(self, oid, method_id, arguments): """for sending over websocket""" self.command_handle += 1 diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 7829ff87..38292bbe 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -19,8 +19,9 @@ from jsonschema import ValidationError, SchemaError from ..GenericTest import GenericTest, NMOSTestException -from ..IS12Utils import IS12Utils, NcObject, NcMethodStatus, NcBlockProperties, NcObjectMethods, NcObjectProperties, \ - NcClassManagerProperties, NcDeviceManagerProperties, StandardClassIds +from ..IS12Utils import IS12Utils, NcObject, NcMethodStatus, NcBlockProperties, NcPropertyChangeType,\ + NcObjectMethods, NcObjectProperties, NcObjectEvents, NcClassManagerProperties, NcDeviceManagerProperties,\ + StandardClassIds from ..TestHelper import load_resolved_schema from ..TestResult import Test @@ -1159,11 +1160,59 @@ def test_29(self, test): command_json, expected_status=NcMethodStatus.Readonly) - def test_30(self, test): - """Subscriptions""" + """Subscriptions and notifications""" self.create_ncp_socket(test) - self.is12_utils.update_subscritions(test, [1]) + oid = self.is12_utils.ROOT_BLOCK_OID + + self.is12_utils.update_subscritions(test, [oid]) + + self.is12_utils.reset_notifications() + + new_user_label = "NMOS Testing Tool" + + old_user_label = self.is12_utils.get_property(test, 1, NcObjectProperties.USER_LABEL.value) + + self.is12_utils.set_property(test, 1, NcObjectProperties.USER_LABEL.value, new_user_label) + + error = False + error_message = "" + + if len(self.is12_utils.get_notifications()) == 0: + error = True + error_message = "No notification recieved" - return test.PASS() \ No newline at end of file + for notification in self.is12_utils.get_notifications(): + if notification['oid'] != oid: + error = True + error_message += "Unexpected Oid " + str(notification['oid']) + ", " + + if notification['eventId'] != NcObjectEvents.PROPERTY_CHANGED.value: + error = True + error_message += "Unexpected event type: " + str(notification['eventId']) + ", " + + if notification["eventData"]["propertyId"] != NcObjectProperties.USER_LABEL.value: + error = True + error_message += "Unexpected property id: " \ + + str(NcObjectProperties(notification["eventData"]["propertyId"]).name) + ", " + + if notification["eventData"]["changeType"] != NcPropertyChangeType.ValueChanged.value: + error = True + error_message += "Unexpected change type: " \ + + str(NcPropertyChangeType(notification["eventData"]["changeType"]).name) + ", " + + if notification["eventData"]["value"] != new_user_label: + error = True + error_message += "Unexpected value: " + str(notification["eventData"]["value"]) + ", " + + if notification["eventData"]["sequenceItemIndex"] is not None: + error = True + error_message += "Unexpected sequence item index: " \ + + str(notification["eventData"]["sequenceItemIndex"]) + ", " + + self.is12_utils.set_property(test, 1, NcObjectProperties.USER_LABEL.value, old_user_label) + + if error: + return test.FAIL(error_message) + return test.PASS() From 3956ee2d0949315a7061077420d997ed1e19b2db Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Mon, 31 Jul 2023 15:02:03 +0100 Subject: [PATCH 23/45] Subscribe to all objects and test changing all user labels --- nmostesting/IS12Utils.py | 7 +- nmostesting/suites/IS1201Test.py | 147 +++++++++++++++---------------- 2 files changed, 75 insertions(+), 79 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index b7819de5..ece16a9a 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -516,19 +516,16 @@ def is_manager(self, class_id): class NcObject(): - def __init__(self, class_id, oid, role): + def __init__(self, class_id, oid, role, descriptors): self.class_id = class_id self.oid = oid self.role = role self.child_objects = [] - self.member_descriptors = [] + self.member_descriptors = descriptors def add_child_object(self, nc_object): self.child_objects.append(nc_object) - def add_member_descriptors(self, member_descriptors): - self.member_descriptors = member_descriptors - def get_role_paths(self, root=True): role_paths = [[self.role]] if not root else [] for child_object in self.child_objects: diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 38292bbe..a0ce6c6f 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -43,7 +43,7 @@ def __init__(self, apis, **kwargs): self.apis[CONTROL_API_KEY]["spec_path"], self.apis[CONTROL_API_KEY]["spec_branch"]) self.load_reference_resources() - self.root_block = None + self.device_model = None def set_up_tests(self): self.unique_roles_error = False @@ -479,14 +479,12 @@ def check_touchpoints(self, test, oid, datatype_schemas, context): self.touchpoints_metadata["error"] = True self.touchpoints_metadata["error_msg"] = context + str(e.args[0].detail) - def validate_block(self, test, block_id, class_descriptors, datatype_schemas, block, context=""): + def validate_block(self, test, block_id, class_descriptors, datatype_schemas, context=""): response = self.is12_utils.get_property(test, block_id, NcBlockProperties.MEMBERS.value) role_cache = [] manager_cache = [] - block.add_member_descriptors(response) - for child_object in response: self._validate_schema(test, child_object, @@ -521,20 +519,14 @@ def validate_block(self, test, block_id, class_descriptors, datatype_schemas, bl + "Non-standard class id does not contain authority key: " \ + str(child_object['classId']) + ". " - child_block = NcObject(child_object['classId'], child_object['oid'], child_object['role']) - # If this child object is a Block, recurse if self.is12_utils.is_block(child_object['classId']): self.validate_block(test, child_object['oid'], class_descriptors, datatype_schemas, - child_block, context=context + child_object['role'] + ': ') - block.add_child_object(child_block) - return - def validate_device_model(self, test): if not self.device_model_validated: self.create_ncp_socket(test) @@ -553,17 +545,34 @@ def validate_device_model(self, test): datatype_descriptors=datatype_descriptors, schema_path=os.path.join(self.apis[CONTROL_API_KEY]["spec_path"], 'APIs/tmp_schemas/')) - self.root_block = NcObject(StandardClassIds.NCBLOCK.value, self.is12_utils.ROOT_BLOCK_OID, "root") - self.validate_block(test, self.is12_utils.ROOT_BLOCK_OID, class_descriptors, - datatype_schemas, - self.root_block) + datatype_schemas) self.device_model_validated = True return + def create_nc_object(self, test, class_id, oid, role): + """Create NcObject and child NcObjects""" + + member_descriptors = self.is12_utils.get_property(test, oid, NcBlockProperties.MEMBERS.value) \ + if self.is12_utils.is_block(class_id) else [] + nc_object = NcObject(class_id, oid, role, member_descriptors) + + for m in member_descriptors: + nc_object.add_child_object(self.create_nc_object(test, m["classId"], m["oid"], m["role"])) + + return nc_object + + def query_device_model(self, test): + self.create_ncp_socket(test) + if not self.device_model: + self.device_model = self.create_nc_object(test, + StandardClassIds.NCBLOCK.value, + self.is12_utils.ROOT_BLOCK_OID, + "root") + def test_04(self, test): """Device Model: check Device Model against classes and datatypes discovered from Class Manager""" # Referencing the Google sheet @@ -754,6 +763,7 @@ def test_13(self, test): # Set user label new_user_label = "NMOS Testing Tool" + self.is12_utils.set_property(test, self.is12_utils.ROOT_BLOCK_OID, property_id, new_user_label) # Check user label @@ -861,13 +871,9 @@ def do_get_member_descriptors_test(self, test, block, context=""): def test_19(self, test): """NcBlock: check GetMemberDescriptors method""" - try: - self.validate_device_model(test) - except NMOSTestException as e: - # Couldn't validate model so can't perform test - return test.UNCLEAR(e.args[0].detail, e.args[0].link) + self.query_device_model(test) - self.do_get_member_descriptors_test(test, self.root_block) + self.do_get_member_descriptors_test(test, self.device_model) return test.PASS() @@ -909,14 +915,10 @@ def do_find_member_by_path_test(self, test, block, context=""): def test_20(self, test): """NcBlock: check FindMemberByPath method""" - try: - self.validate_device_model(test) - except NMOSTestException as e: - # Couldn't validate model so can't perform test - return test.UNCLEAR(e.args[0].detail, e.args[0].link) + self.query_device_model(test) # Recursively check each block in Device Model - self.do_find_member_by_path_test(test, self.root_block) + self.do_find_member_by_path_test(test, self.device_model) return test.PASS() @@ -973,14 +975,10 @@ def do_find_member_by_role_test(self, test, block, context=""): def test_21(self, test): """NcBlock: check FindMembersByRole method""" - try: - self.validate_device_model(test) - except NMOSTestException as e: - # Couldn't validate model so can't perform test - return test.UNCLEAR(e.args[0].detail, e.args[0].link) + self.query_device_model(test) # Recursively check each block in Device Model - self.do_find_member_by_role_test(test, self.root_block) + self.do_find_member_by_role_test(test, self.device_model) return test.PASS() @@ -1028,13 +1026,9 @@ def do_find_members_by_class_id_test(self, test, block, context=""): def test_22(self, test): """NcBlock: check FindMembersByClassId method""" - try: - self.validate_device_model(test) - except NMOSTestException as e: - # Couldn't validate model so can't perform test - return test.UNCLEAR(e.args[0].detail, e.args[0].link) + self.query_device_model(test) - self.do_find_members_by_class_id_test(test, self.root_block) + self.do_find_members_by_class_id_test(test, self.device_model) return test.PASS() @@ -1162,56 +1156,61 @@ def test_29(self, test): def test_30(self, test): """Subscriptions and notifications""" - self.create_ncp_socket(test) + self.query_device_model(test) - oid = self.is12_utils.ROOT_BLOCK_OID + # Get all oids for objects in this Device Model + device_model_objects = self.device_model.find_members_by_class_id(class_id=StandardClassIds.NCOBJECT.value, + include_derived=True, + recurse=True) - self.is12_utils.update_subscritions(test, [oid]) + oids = [self.is12_utils.ROOT_BLOCK_OID] + [o.oid for o in device_model_objects] - self.is12_utils.reset_notifications() - - new_user_label = "NMOS Testing Tool" - - old_user_label = self.is12_utils.get_property(test, 1, NcObjectProperties.USER_LABEL.value) - - self.is12_utils.set_property(test, 1, NcObjectProperties.USER_LABEL.value, new_user_label) + self.is12_utils.update_subscritions(test, oids) error = False error_message = "" - if len(self.is12_utils.get_notifications()) == 0: - error = True - error_message = "No notification recieved" + for oid in oids: + new_user_label = "NMOS Testing Tool " + str(oid) + old_user_label = self.is12_utils.get_property(test, oid, NcObjectProperties.USER_LABEL.value) + + context = "oid: " + str(oid) + ", " + + for label in [new_user_label, old_user_label]: + self.is12_utils.reset_notifications() + self.is12_utils.set_property(test, oid, NcObjectProperties.USER_LABEL.value, label) - for notification in self.is12_utils.get_notifications(): - if notification['oid'] != oid: - error = True - error_message += "Unexpected Oid " + str(notification['oid']) + ", " + if len(self.is12_utils.get_notifications()) == 0: + error = True + error_message = context + "No notification recieved" - if notification['eventId'] != NcObjectEvents.PROPERTY_CHANGED.value: - error = True - error_message += "Unexpected event type: " + str(notification['eventId']) + ", " + for notification in self.is12_utils.get_notifications(): + if notification['oid'] != oid: + error = True + error_message += context + "Unexpected Oid " + str(notification['oid']) + ", " - if notification["eventData"]["propertyId"] != NcObjectProperties.USER_LABEL.value: - error = True - error_message += "Unexpected property id: " \ - + str(NcObjectProperties(notification["eventData"]["propertyId"]).name) + ", " + if notification['eventId'] != NcObjectEvents.PROPERTY_CHANGED.value: + error = True + error_message += context + "Unexpected event type: " + str(notification['eventId']) + ", " - if notification["eventData"]["changeType"] != NcPropertyChangeType.ValueChanged.value: - error = True - error_message += "Unexpected change type: " \ - + str(NcPropertyChangeType(notification["eventData"]["changeType"]).name) + ", " + if notification["eventData"]["propertyId"] != NcObjectProperties.USER_LABEL.value: + error = True + error_message += context + "Unexpected property id: " \ + + str(NcObjectProperties(notification["eventData"]["propertyId"]).name) + ", " - if notification["eventData"]["value"] != new_user_label: - error = True - error_message += "Unexpected value: " + str(notification["eventData"]["value"]) + ", " + if notification["eventData"]["changeType"] != NcPropertyChangeType.ValueChanged.value: + error = True + error_message += context + "Unexpected change type: " \ + + str(NcPropertyChangeType(notification["eventData"]["changeType"]).name) + ", " - if notification["eventData"]["sequenceItemIndex"] is not None: - error = True - error_message += "Unexpected sequence item index: " \ - + str(notification["eventData"]["sequenceItemIndex"]) + ", " + if notification["eventData"]["value"] != label: + error = True + error_message += context + "Unexpected value: " + str(notification["eventData"]["value"]) + ", " - self.is12_utils.set_property(test, 1, NcObjectProperties.USER_LABEL.value, old_user_label) + if notification["eventData"]["sequenceItemIndex"] is not None: + error = True + error_message += context + "Unexpected sequence item index: " \ + + str(notification["eventData"]["sequenceItemIndex"]) + ", " if error: return test.FAIL(error_message) From 921190b9a73dfb38df51839aa1b010e8ff8a9559 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Mon, 31 Jul 2023 16:40:14 +0100 Subject: [PATCH 24/45] Improve error reporting --- nmostesting/suites/IS1201Test.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index a0ce6c6f..7d6a8952 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -191,6 +191,8 @@ def validate_descriptor(self, test, reference, descriptor, context=""): descriptor.pop('constraints', None) reference.pop('isConstant', None) descriptor.pop('isConstant', None) + reference.pop('isPersistent', None) + descriptor.pop('isPersistent', None) # JRT: End reference_keys = set(reference.keys()) @@ -199,8 +201,9 @@ def validate_descriptor(self, test, reference, descriptor, context=""): # compare the keys to see if any extra/missing key_diff = (set(reference_keys) | set(descriptor_keys)) - (set(reference_keys) & set(descriptor_keys)) if len(key_diff) > 0: - raise NMOSTestException(test.FAIL(context + 'Missing/additional keys ' + str(key_diff))) - + error_description = "Missing keys " if key_diff in reference_keys else "Additional keys " + type_name = descriptor.get("typeName") + ": " if descriptor.get("typeName") else "" + raise NMOSTestException(test.FAIL(context + type_name + error_description + str(key_diff))) for key in reference_keys: if key in non_normative_keys: continue @@ -336,7 +339,7 @@ def test_03(self, test): NcObjectProperties.ROLE.value) if role != "root": - return test.FAIL("Unexpected role in Root Block: " + role, + return test.FAIL("Unexpected role in Root Block: " + str(role), "https://specs.amwa.tv/ms-05-02/branches/{}" "/docs/Blocks.html" .format(self.apis[CONTROL_API_KEY]["spec_branch"])) @@ -502,7 +505,7 @@ def validate_block(self, test, block_id, class_descriptors, datatype_schemas, co self.check_manager(child_object['classId'], child_object["owner"], class_descriptors, manager_cache) self.check_touchpoints(test, child_object['oid'], datatype_schemas, - context=context + child_object['role'] + ': ') + context=context + str(child_object['role']) + ': ') class_identifier = ".".join(map(str, child_object['classId'])) @@ -511,11 +514,11 @@ def validate_block(self, test, block_id, class_descriptors, datatype_schemas, co class_descriptors[class_identifier], child_object['oid'], datatype_schemas, - context=context + child_object['role'] + ': ') + context=context + str(child_object['role']) + ': ') else: # Not a standard or non-standard class self.organization_metadata["error"] = True - self.organization_metadata["error_msg"] = child_object['role'] + ': ' \ + self.organization_metadata["error_msg"] = str(child_object['role']) + ': ' \ + "Non-standard class id does not contain authority key: " \ + str(child_object['classId']) + ". " @@ -525,7 +528,7 @@ def validate_block(self, test, block_id, class_descriptors, datatype_schemas, co child_object['oid'], class_descriptors, datatype_schemas, - context=context + child_object['role'] + ': ') + context=context + str(child_object['role']) + ': ') def validate_device_model(self, test): if not self.device_model_validated: From ef9006aaf83aeebfc5e80c9bdb327a946c275c38 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Mon, 31 Jul 2023 16:42:01 +0100 Subject: [PATCH 25/45] handle null descriptions --- nmostesting/IS12Utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index ece16a9a..c7cbb070 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -437,7 +437,7 @@ def descriptor_to_schema(self, descriptor): json_schema['$schema'] = 'http://json-schema.org/draft-07/schema#' json_schema['title'] = descriptor['name'] - json_schema['description'] = descriptor['description'] + json_schema['description'] = descriptor['description'] if descriptor['description'] else "" # Inheritance of datatype if descriptor.get('parentType'): @@ -482,7 +482,7 @@ def descriptor_to_schema(self, descriptor): if field.get('isSequence'): property_type = {'type': 'array', 'items': property_type} - property_type['description'] = field['description'] + property_type['description'] = field['description'] if descriptor['description'] else "" properties[field['name']] = property_type json_schema['required'] = required From bc24b1e9b8ea6f0223ef171aeb3cc7d030fab74d Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Mon, 31 Jul 2023 20:52:24 +0100 Subject: [PATCH 26/45] Bug fix - now won't ignore description field in NcDescriptor --- nmostesting/suites/IS1201Test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 7d6a8952..c907a07a 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -205,7 +205,7 @@ def validate_descriptor(self, test, reference, descriptor, context=""): type_name = descriptor.get("typeName") + ": " if descriptor.get("typeName") else "" raise NMOSTestException(test.FAIL(context + type_name + error_description + str(key_diff))) for key in reference_keys: - if key in non_normative_keys: + if key in non_normative_keys and not isinstance(reference[key], dict): continue # Check for class ID if key == 'classId' and isinstance(reference[key], list): From 884da9017326cf525e24332f65c7ad19ebcc2bee Mon Sep 17 00:00:00 2001 From: Gareth Sylvester-Bradley <31761158+garethsb@users.noreply.github.com> Date: Tue, 1 Aug 2023 11:59:46 +0100 Subject: [PATCH 27/45] Fix flake8/pycodestyle E721 do not compare types --- utilities/dut-data-exporter/dutDataExporter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utilities/dut-data-exporter/dutDataExporter.py b/utilities/dut-data-exporter/dutDataExporter.py index e2d7188d..d32a9653 100755 --- a/utilities/dut-data-exporter/dutDataExporter.py +++ b/utilities/dut-data-exporter/dutDataExporter.py @@ -54,9 +54,9 @@ def url_port_number(url): response = requests.get(url) mac_addresses = response.json()["interfaces"] for address in mac_addresses: - if type(address["port_id"]) == str: + if isinstance(address["port_id"], str): address["port_id"] = address["port_id"].replace("-", ":") - if type(address["chassis_id"]) == str: + if isinstance(address["chassis_id"], str): address["chassis_id"] = address["chassis_id"].replace("-", ":") node_id = response.json()["id"] print("Host: {}".format(response.json()["description"])) From c51a42b3c18602f523b137cc434ce428723b1436 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Tue, 1 Aug 2023 17:00:43 +0100 Subject: [PATCH 28/45] Add ClassManager class --- nmostesting/IS12Utils.py | 60 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index c7cbb070..66a17aaf 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -17,6 +17,7 @@ import json import time +from copy import deepcopy from enum import IntEnum, Enum from itertools import takewhile, dropwhile from jsonschema import FormatChecker, SchemaError, validate, ValidationError @@ -126,6 +127,7 @@ class NcClassManagerProperties(Enum): class NcClassManagerMethods(Enum): GET_CONTROL_CLASS = {'level': 3, 'index': 1} + GET_DATATYPE = {'level': 3, 'index': 2} class NcDeviceManagerProperties(Enum): @@ -381,6 +383,20 @@ def find_members_by_class_id(self, test, oid, class_id, include_derived, recurse 'includeDerived': include_derived, 'recurse': recurse})["value"] + def get_control_class(self, test, oid, class_id, include_inherited): + """Query Class Manager for control class. Raises NMOSTestException on error""" + return self._execute_command(test, oid, + NcClassManagerMethods.GET_CONTROL_CLASS.value, + {'classId': class_id, + 'includeInherited': include_inherited})["value"] + + def get_datatype(self, test, oid, name, include_inherited): + """Query Class Manager for datatype. Raises NMOSTestException on error""" + return self._execute_command(test, oid, + NcClassManagerMethods.GET_DATATYPE.value, + {'name': name, + 'includeInherited': include_inherited})["value"] + def create_subscription_JSON(self, subscriptions): """for sending over websocket""" return { @@ -586,3 +602,47 @@ def match(query_class_id, class_id, include_derived): include_derived, recurse) return query_results + + +class NcClassManager(): + def __init__(self, oid, class_descriptors, datatype_descriptors): + self.oid = oid + self.class_descriptors = class_descriptors + self.datatype_descriptors = datatype_descriptors + + def get_control_class(self, class_id, include_inherited): + class_id_str = ".".join(map(str, class_id)) + descriptor = self.class_descriptors[class_id_str] + + if not include_inherited: + return descriptor + + parent_class = class_id[:-1] + inherited_descriptor = deepcopy(descriptor) + + # add inherited classes + while len(parent_class) > 0: + if parent_class[-1] > 0: # Ignore Authority Keys + class_id_str = ".".join(map(str, parent_class)) + parent_descriptor = self.class_descriptors[class_id_str] + inherited_descriptor["properties"] += parent_descriptor["properties"] + inherited_descriptor["methods"] += parent_descriptor["methods"] + inherited_descriptor["events"] += parent_descriptor["events"] + parent_class.pop() + + return inherited_descriptor + + def get_datatype(self, name, include_inherited): + descriptor = self.datatype_descriptors[name] + + if not include_inherited or descriptor["type"] != NcDatatypeType.Struct: + return descriptor + + inherited_descriptor = deepcopy(descriptor) + + while descriptor.get("parentType"): + parent_type = descriptor.get("parentType") + descriptor = self.datatype_descriptors[parent_type] + inherited_descriptor["fields"] += descriptor["fields"] + + return inherited_descriptor From 42bae13db0d4b1c187b93282f54f46144faf289c Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Tue, 1 Aug 2023 17:01:08 +0100 Subject: [PATCH 29/45] Add Class Manager method tests --- nmostesting/suites/IS1201Test.py | 86 +++++++++++++++++++++++++------- 1 file changed, 67 insertions(+), 19 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index c907a07a..d90e9d5c 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -21,7 +21,7 @@ from ..GenericTest import GenericTest, NMOSTestException from ..IS12Utils import IS12Utils, NcObject, NcMethodStatus, NcBlockProperties, NcPropertyChangeType,\ NcObjectMethods, NcObjectProperties, NcObjectEvents, NcClassManagerProperties, NcDeviceManagerProperties,\ - StandardClassIds + StandardClassIds, NcClassManager from ..TestHelper import load_resolved_schema from ..TestResult import Test @@ -182,28 +182,14 @@ def validate_descriptor(self, test, reference, descriptor, context=""): non_normative_keys = ['description'] if isinstance(reference, dict): - # JRT: These two manipulation are to mitigate two issues - # to be resolved regarding the MS-05-02 JSON descriptors. - # Firstly the constraints property is missing from certain descriptors - # Secondly the isConstant flag is missing from - # the NcObject descriptor properties - reference.pop('constraints', None) - descriptor.pop('constraints', None) - reference.pop('isConstant', None) - descriptor.pop('isConstant', None) - reference.pop('isPersistent', None) - descriptor.pop('isPersistent', None) - # JRT: End - reference_keys = set(reference.keys()) descriptor_keys = set(descriptor.keys()) # compare the keys to see if any extra/missing key_diff = (set(reference_keys) | set(descriptor_keys)) - (set(reference_keys) & set(descriptor_keys)) if len(key_diff) > 0: - error_description = "Missing keys " if key_diff in reference_keys else "Additional keys " - type_name = descriptor.get("typeName") + ": " if descriptor.get("typeName") else "" - raise NMOSTestException(test.FAIL(context + type_name + error_description + str(key_diff))) + error_description = "Missing keys " if set(key_diff) <= set(reference_keys) else "Additional keys " + raise NMOSTestException(test.FAIL(context + error_description + str(key_diff))) for key in reference_keys: if key in non_normative_keys and not isinstance(reference[key], dict): continue @@ -422,7 +408,8 @@ def validate_object_properties(self, test, reference_class_descriptor, oid, data class_property['typeName'], class_property['isNullable'], datatype_schemas, - context=context + class_property["name"] + ": ") + context=context + class_property["typeName"] + + ": " + class_property["name"] + ": ") self.check_sequence_methods(test, oid, response, class_property, context=context) else: self.validate_property_type(test, @@ -430,7 +417,8 @@ def validate_object_properties(self, test, reference_class_descriptor, oid, data class_property['typeName'], class_property['isNullable'], datatype_schemas, - context=context + class_property["name"] + ": ") + context=context + class_property["typeName"] + + class_property["name"] + ": ") return def check_unique_roles(self, role, role_cache): @@ -1218,3 +1206,63 @@ def test_30(self, test): if error: return test.FAIL(error_message) return test.PASS() + + def test_31(self, test): + """Class Manager: check GetControlClass""" + self.create_ncp_socket(test) + + class_manager_oid = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value)["oid"] + + class_descriptors = self.get_class_manager_descriptors(test, + class_manager_oid, + NcClassManagerProperties.CONTROL_CLASSES.value) + datatype_descriptors = self.get_class_manager_descriptors(test, + class_manager_oid, + NcClassManagerProperties.DATATYPES.value) + # Construct local class manager to use as source of ground truths + class_manager = NcClassManager(class_manager_oid, class_descriptors, datatype_descriptors) + + for _, class_descriptor in class_descriptors.items(): + for include_inherited in [False, True]: + actual_descriptor = self.is12_utils.get_control_class(test, + class_manager.oid, + class_descriptor["classId"], + include_inherited) + expected_descriptor = class_manager.get_control_class(class_descriptor["classId"], + include_inherited) + self.validate_descriptor(test, + expected_descriptor, + actual_descriptor, + context=str(class_descriptor["classId"]) + ": ") + + return test.PASS() + + def test_32(self, test): + """Class Manager: check GetDatatype""" + self.create_ncp_socket(test) + + class_manager_oid = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value)["oid"] + + class_descriptors = self.get_class_manager_descriptors(test, + class_manager_oid, + NcClassManagerProperties.CONTROL_CLASSES.value) + datatype_descriptors = self.get_class_manager_descriptors(test, + class_manager_oid, + NcClassManagerProperties.DATATYPES.value) + # Construct local class manager to use as source of ground truths + class_manager = NcClassManager(class_manager_oid, class_descriptors, datatype_descriptors) + + for _, datatype_descriptor in datatype_descriptors.items(): + for include_inherited in [False, True]: + actual_descriptor = self.is12_utils.get_datatype(test, + class_manager.oid, + datatype_descriptor["name"], + include_inherited) + expected_descriptor = class_manager.get_datatype(datatype_descriptor["name"], + include_inherited) + self.validate_descriptor(test, + expected_descriptor, + actual_descriptor, + context=datatype_descriptor["name"] + ": ") + + return test.PASS() From 814e6d8d7d5fbeabfc9bddece71d1b7aeea9d681 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Tue, 1 Aug 2023 20:19:13 +0100 Subject: [PATCH 30/45] Factor out querying of class manager --- nmostesting/suites/IS1201Test.py | 84 +++++++++++++------------------- 1 file changed, 33 insertions(+), 51 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index d90e9d5c..dda123c0 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -44,6 +44,7 @@ def __init__(self, apis, **kwargs): self.apis[CONTROL_API_KEY]["spec_branch"]) self.load_reference_resources() self.device_model = None + self.class_manager = None def set_up_tests(self): self.unique_roles_error = False @@ -237,13 +238,10 @@ def key_lambda(classId, name): return ".".join(map(str, classId)) if classId els return descriptors - def validate_model_definitions(self, class_manager_oid, property_id, schema_name, reference_descriptors): + def validate_model_definitions(self, descriptors, schema_name, reference_descriptors): """Validate class manager model definitions against reference model descriptors. Returns [test result array]""" results = list() - test = Test("Validate model definitions", "auto_ValidateModel") - descriptors = self.get_class_manager_descriptors(test, class_manager_oid, property_id) - reference_descriptor_keys = sorted(reference_descriptors.keys()) for key in reference_descriptor_keys: @@ -275,15 +273,13 @@ def auto_tests(self): self.create_ncp_socket(test) - class_manager = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) + self.query_class_manager(test) - results += self.validate_model_definitions(class_manager['oid'], - NcClassManagerProperties.CONTROL_CLASSES.value, + results += self.validate_model_definitions(self.class_manager.class_descriptors, 'NcClassDescriptor', self.classes_descriptors) - results += self.validate_model_definitions(class_manager['oid'], - NcClassManagerProperties.DATATYPES.value, + results += self.validate_model_definitions(self.class_manager.datatype_descriptors, 'NcDatatypeDescriptor', self.datatype_descriptors) return results @@ -522,23 +518,16 @@ def validate_device_model(self, test): if not self.device_model_validated: self.create_ncp_socket(test) - class_manager = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) - - class_descriptors = \ - self.get_class_manager_descriptors(test, class_manager['oid'], - NcClassManagerProperties.CONTROL_CLASSES.value) - datatype_descriptors = \ - self.get_class_manager_descriptors(test, class_manager['oid'], - NcClassManagerProperties.DATATYPES.value) + self.query_class_manager(test) # Create JSON schemas for the queried datatypes datatype_schemas = self.generate_json_schemas( - datatype_descriptors=datatype_descriptors, + datatype_descriptors=self.class_manager.datatype_descriptors, schema_path=os.path.join(self.apis[CONTROL_API_KEY]["spec_path"], 'APIs/tmp_schemas/')) self.validate_block(test, self.is12_utils.ROOT_BLOCK_OID, - class_descriptors, + self.class_manager.class_descriptors, datatype_schemas) self.device_model_validated = True @@ -564,6 +553,21 @@ def query_device_model(self, test): self.is12_utils.ROOT_BLOCK_OID, "root") + def query_class_manager(self, test): + """Query class manager to use as source of ground truths""" + if not self.class_manager: + self.create_ncp_socket(test) + + class_manager_oid = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value)["oid"] + + class_descriptors = self.get_class_manager_descriptors(test, + class_manager_oid, + NcClassManagerProperties.CONTROL_CLASSES.value) + datatype_descriptors = self.get_class_manager_descriptors(test, + class_manager_oid, + NcClassManagerProperties.DATATYPES.value) + self.class_manager = NcClassManager(class_manager_oid, class_descriptors, datatype_descriptors) + def test_04(self, test): """Device Model: check Device Model against classes and datatypes discovered from Class Manager""" # Referencing the Google sheet @@ -1209,27 +1213,16 @@ def test_30(self, test): def test_31(self, test): """Class Manager: check GetControlClass""" - self.create_ncp_socket(test) - - class_manager_oid = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value)["oid"] + self.query_class_manager(test) - class_descriptors = self.get_class_manager_descriptors(test, - class_manager_oid, - NcClassManagerProperties.CONTROL_CLASSES.value) - datatype_descriptors = self.get_class_manager_descriptors(test, - class_manager_oid, - NcClassManagerProperties.DATATYPES.value) - # Construct local class manager to use as source of ground truths - class_manager = NcClassManager(class_manager_oid, class_descriptors, datatype_descriptors) - - for _, class_descriptor in class_descriptors.items(): + for _, class_descriptor in self.class_manager.class_descriptors.items(): for include_inherited in [False, True]: actual_descriptor = self.is12_utils.get_control_class(test, - class_manager.oid, + self.class_manager.oid, class_descriptor["classId"], include_inherited) - expected_descriptor = class_manager.get_control_class(class_descriptor["classId"], - include_inherited) + expected_descriptor = self.class_manager.get_control_class(class_descriptor["classId"], + include_inherited) self.validate_descriptor(test, expected_descriptor, actual_descriptor, @@ -1239,27 +1232,16 @@ def test_31(self, test): def test_32(self, test): """Class Manager: check GetDatatype""" - self.create_ncp_socket(test) - - class_manager_oid = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value)["oid"] + self.query_class_manager(test) - class_descriptors = self.get_class_manager_descriptors(test, - class_manager_oid, - NcClassManagerProperties.CONTROL_CLASSES.value) - datatype_descriptors = self.get_class_manager_descriptors(test, - class_manager_oid, - NcClassManagerProperties.DATATYPES.value) - # Construct local class manager to use as source of ground truths - class_manager = NcClassManager(class_manager_oid, class_descriptors, datatype_descriptors) - - for _, datatype_descriptor in datatype_descriptors.items(): + for _, datatype_descriptor in self.class_manager.datatype_descriptors.items(): for include_inherited in [False, True]: actual_descriptor = self.is12_utils.get_datatype(test, - class_manager.oid, + self.class_manager.oid, datatype_descriptor["name"], include_inherited) - expected_descriptor = class_manager.get_datatype(datatype_descriptor["name"], - include_inherited) + expected_descriptor = self.class_manager.get_datatype(datatype_descriptor["name"], + include_inherited) self.validate_descriptor(test, expected_descriptor, actual_descriptor, From 7aa05b2c7bbc52222be7966ea9f2b516aec5fe3f Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Tue, 1 Aug 2023 20:31:47 +0100 Subject: [PATCH 31/45] Tidy up querying of class manager and device model --- nmostesting/suites/IS1201Test.py | 105 ++++++++++++++++--------------- 1 file changed, 54 insertions(+), 51 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index dda123c0..c84ec26c 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -264,6 +264,32 @@ def validate_model_definitions(self, descriptors, schema_name, reference_descrip return results + def query_device_model(self, test): + self.create_ncp_socket(test) + if not self.device_model: + self.device_model = self.create_nc_object(test, + StandardClassIds.NCBLOCK.value, + self.is12_utils.ROOT_BLOCK_OID, + "root") + return self.device_model + + def query_class_manager(self, test): + """Query class manager to use as source of ground truths""" + if not self.class_manager: + self.create_ncp_socket(test) + + class_manager_oid = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value)["oid"] + + class_descriptors = self.get_class_manager_descriptors(test, + class_manager_oid, + NcClassManagerProperties.CONTROL_CLASSES.value) + datatype_descriptors = self.get_class_manager_descriptors(test, + class_manager_oid, + NcClassManagerProperties.DATATYPES.value) + self.class_manager = NcClassManager(class_manager_oid, class_descriptors, datatype_descriptors) + + return self.class_manager + def auto_tests(self): """Automatically validate all standard datatypes and control classes. Returns [test result array]""" # Referencing the Google sheet @@ -273,13 +299,13 @@ def auto_tests(self): self.create_ncp_socket(test) - self.query_class_manager(test) + class_manager = self.query_class_manager(test) - results += self.validate_model_definitions(self.class_manager.class_descriptors, + results += self.validate_model_definitions(class_manager.class_descriptors, 'NcClassDescriptor', self.classes_descriptors) - results += self.validate_model_definitions(self.class_manager.datatype_descriptors, + results += self.validate_model_definitions(class_manager.datatype_descriptors, 'NcDatatypeDescriptor', self.datatype_descriptors) return results @@ -518,16 +544,16 @@ def validate_device_model(self, test): if not self.device_model_validated: self.create_ncp_socket(test) - self.query_class_manager(test) + class_manager = self.query_class_manager(test) # Create JSON schemas for the queried datatypes datatype_schemas = self.generate_json_schemas( - datatype_descriptors=self.class_manager.datatype_descriptors, + datatype_descriptors=class_manager.datatype_descriptors, schema_path=os.path.join(self.apis[CONTROL_API_KEY]["spec_path"], 'APIs/tmp_schemas/')) self.validate_block(test, self.is12_utils.ROOT_BLOCK_OID, - self.class_manager.class_descriptors, + class_manager.class_descriptors, datatype_schemas) self.device_model_validated = True @@ -545,29 +571,6 @@ def create_nc_object(self, test, class_id, oid, role): return nc_object - def query_device_model(self, test): - self.create_ncp_socket(test) - if not self.device_model: - self.device_model = self.create_nc_object(test, - StandardClassIds.NCBLOCK.value, - self.is12_utils.ROOT_BLOCK_OID, - "root") - - def query_class_manager(self, test): - """Query class manager to use as source of ground truths""" - if not self.class_manager: - self.create_ncp_socket(test) - - class_manager_oid = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value)["oid"] - - class_descriptors = self.get_class_manager_descriptors(test, - class_manager_oid, - NcClassManagerProperties.CONTROL_CLASSES.value) - datatype_descriptors = self.get_class_manager_descriptors(test, - class_manager_oid, - NcClassManagerProperties.DATATYPES.value) - self.class_manager = NcClassManager(class_manager_oid, class_descriptors, datatype_descriptors) - def test_04(self, test): """Device Model: check Device Model against classes and datatypes discovered from Class Manager""" # Referencing the Google sheet @@ -866,9 +869,9 @@ def do_get_member_descriptors_test(self, test, block, context=""): def test_19(self, test): """NcBlock: check GetMemberDescriptors method""" - self.query_device_model(test) + device_model = self.query_device_model(test) - self.do_get_member_descriptors_test(test, self.device_model) + self.do_get_member_descriptors_test(test, device_model) return test.PASS() @@ -910,10 +913,10 @@ def do_find_member_by_path_test(self, test, block, context=""): def test_20(self, test): """NcBlock: check FindMemberByPath method""" - self.query_device_model(test) + device_model = self.query_device_model(test) # Recursively check each block in Device Model - self.do_find_member_by_path_test(test, self.device_model) + self.do_find_member_by_path_test(test, device_model) return test.PASS() @@ -970,10 +973,10 @@ def do_find_member_by_role_test(self, test, block, context=""): def test_21(self, test): """NcBlock: check FindMembersByRole method""" - self.query_device_model(test) + device_model = self.query_device_model(test) # Recursively check each block in Device Model - self.do_find_member_by_role_test(test, self.device_model) + self.do_find_member_by_role_test(test, device_model) return test.PASS() @@ -1021,9 +1024,9 @@ def do_find_members_by_class_id_test(self, test, block, context=""): def test_22(self, test): """NcBlock: check FindMembersByClassId method""" - self.query_device_model(test) + device_model = self.query_device_model(test) - self.do_find_members_by_class_id_test(test, self.device_model) + self.do_find_members_by_class_id_test(test, device_model) return test.PASS() @@ -1151,12 +1154,12 @@ def test_29(self, test): def test_30(self, test): """Subscriptions and notifications""" - self.query_device_model(test) + device_model = self.query_device_model(test) # Get all oids for objects in this Device Model - device_model_objects = self.device_model.find_members_by_class_id(class_id=StandardClassIds.NCOBJECT.value, - include_derived=True, - recurse=True) + device_model_objects = device_model.find_members_by_class_id(class_id=StandardClassIds.NCOBJECT.value, + include_derived=True, + recurse=True) oids = [self.is12_utils.ROOT_BLOCK_OID] + [o.oid for o in device_model_objects] @@ -1213,16 +1216,16 @@ def test_30(self, test): def test_31(self, test): """Class Manager: check GetControlClass""" - self.query_class_manager(test) + class_manager = self.query_class_manager(test) - for _, class_descriptor in self.class_manager.class_descriptors.items(): + for _, class_descriptor in class_manager.class_descriptors.items(): for include_inherited in [False, True]: actual_descriptor = self.is12_utils.get_control_class(test, - self.class_manager.oid, + class_manager.oid, class_descriptor["classId"], include_inherited) - expected_descriptor = self.class_manager.get_control_class(class_descriptor["classId"], - include_inherited) + expected_descriptor = class_manager.get_control_class(class_descriptor["classId"], + include_inherited) self.validate_descriptor(test, expected_descriptor, actual_descriptor, @@ -1232,16 +1235,16 @@ def test_31(self, test): def test_32(self, test): """Class Manager: check GetDatatype""" - self.query_class_manager(test) + class_manager = self.query_class_manager(test) - for _, datatype_descriptor in self.class_manager.datatype_descriptors.items(): + for _, datatype_descriptor in class_manager.datatype_descriptors.items(): for include_inherited in [False, True]: actual_descriptor = self.is12_utils.get_datatype(test, - self.class_manager.oid, + class_manager.oid, datatype_descriptor["name"], include_inherited) - expected_descriptor = self.class_manager.get_datatype(datatype_descriptor["name"], - include_inherited) + expected_descriptor = class_manager.get_datatype(datatype_descriptor["name"], + include_inherited) self.validate_descriptor(test, expected_descriptor, actual_descriptor, From 5531ca88d9960fb2223d112c0bc981182928e4d6 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Tue, 1 Aug 2023 20:53:52 +0100 Subject: [PATCH 32/45] Refactored validate device model --- nmostesting/suites/IS1201Test.py | 56 ++++++++++++++------------------ 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index c84ec26c..5c04e5e2 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -492,53 +492,45 @@ def check_touchpoints(self, test, oid, datatype_schemas, context): self.touchpoints_metadata["error"] = True self.touchpoints_metadata["error_msg"] = context + str(e.args[0].detail) - def validate_block(self, test, block_id, class_descriptors, datatype_schemas, context=""): - response = self.is12_utils.get_property(test, block_id, NcBlockProperties.MEMBERS.value) - + def validate_block(self, test, block, class_descriptors, datatype_schemas, context=""): + for child_object in block.child_objects: + # If this child object is a Block, recurse + if self.is12_utils.is_block(child_object.class_id): + self.validate_block(test, + child_object, + class_descriptors, + datatype_schemas, + context=context + str(child_object.role) + ': ') role_cache = [] manager_cache = [] - - for child_object in response: + for descriptor in block.member_descriptors: self._validate_schema(test, - child_object, + descriptor, datatype_schemas["NcBlockMemberDescriptor"], context="NcBlockMemberDescriptor: ") - self.check_unique_roles(child_object['role'], role_cache) - - self.check_unique_oid(child_object['oid']) - + self.check_unique_roles(descriptor['role'], role_cache) + self.check_unique_oid(descriptor['oid']) # check for non-standard classes - if self.is12_utils.is_non_standard_class(child_object['classId']): + if self.is12_utils.is_non_standard_class(descriptor['classId']): self.organization_metadata["checked"] = True + self.check_manager(descriptor['classId'], descriptor["owner"], class_descriptors, manager_cache) + self.check_touchpoints(test, descriptor['oid'], datatype_schemas, + context=context + str(descriptor['role']) + ': ') - self.check_manager(child_object['classId'], child_object["owner"], class_descriptors, manager_cache) - - self.check_touchpoints(test, child_object['oid'], datatype_schemas, - context=context + str(child_object['role']) + ': ') - - class_identifier = ".".join(map(str, child_object['classId'])) - + class_identifier = ".".join(map(str, descriptor['classId'])) if class_identifier: self.validate_object_properties(test, class_descriptors[class_identifier], - child_object['oid'], + descriptor['oid'], datatype_schemas, - context=context + str(child_object['role']) + ': ') + context=context + str(descriptor['role']) + ': ') else: # Not a standard or non-standard class self.organization_metadata["error"] = True - self.organization_metadata["error_msg"] = str(child_object['role']) + ': ' \ + self.organization_metadata["error_msg"] = str(descriptor['role']) + ': ' \ + "Non-standard class id does not contain authority key: " \ - + str(child_object['classId']) + ". " - - # If this child object is a Block, recurse - if self.is12_utils.is_block(child_object['classId']): - self.validate_block(test, - child_object['oid'], - class_descriptors, - datatype_schemas, - context=context + str(child_object['role']) + ': ') + + str(descriptor['classId']) + ". " def validate_device_model(self, test): if not self.device_model_validated: @@ -546,13 +538,15 @@ def validate_device_model(self, test): class_manager = self.query_class_manager(test) + device_model = self.query_device_model(test) + # Create JSON schemas for the queried datatypes datatype_schemas = self.generate_json_schemas( datatype_descriptors=class_manager.datatype_descriptors, schema_path=os.path.join(self.apis[CONTROL_API_KEY]["spec_path"], 'APIs/tmp_schemas/')) self.validate_block(test, - self.is12_utils.ROOT_BLOCK_OID, + device_model, class_manager.class_descriptors, datatype_schemas) From 4a26c82a6312e9551f3020b9c10f8b4125ecca9e Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Wed, 2 Aug 2023 09:40:54 +0100 Subject: [PATCH 33/45] Handle null sequences. Renamed some parameters --- nmostesting/suites/IS1201Test.py | 77 +++++++++++++++++++------------- 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 5c04e5e2..ec4b6ecc 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -51,7 +51,7 @@ def set_up_tests(self): self.unique_oids_error = False self.managers_are_singletons_error = False self.managers_members_root_block_error = False - self.device_model_validated = False + self.device_model_checked = False self.organization_metadata = {"checked": False, "error": False, "error_msg": ""} self.touchpoints_metadata = {"checked": False, "error": False, "error_msg": ""} self.get_sequence_item_metadata = {"checked": False, "error": False, "error_msg": ""} @@ -126,14 +126,14 @@ def load_reference_resources(self): classes_paths.append(classes_path) # Load class and datatype descriptors - self.classes_descriptors = self.load_model_descriptors(classes_paths) + self.reference_class_descriptors = self.load_model_descriptors(classes_paths) # Load MS-05 datatype descriptors - self.datatype_descriptors = self.load_model_descriptors(datatype_paths) + self.reference_datatype_descriptors = self.load_model_descriptors(datatype_paths) # Generate MS-05 datatype schemas from MS-05 datatype descriptors self.datatype_schemas = self.generate_json_schemas( - datatype_descriptors=self.datatype_descriptors, + datatype_descriptors=self.reference_datatype_descriptors, schema_path=os.path.join(self.apis[CONTROL_API_KEY]["spec_path"], 'APIs/schemas/')) def create_ncp_socket(self, test): @@ -150,7 +150,7 @@ def get_manager(self, test, class_id): manager_found = False manager = None - class_descriptor = self.classes_descriptors[class_id_str] + class_descriptor = self.reference_class_descriptors[class_id_str] for value in response: self._validate_schema(test, @@ -303,11 +303,11 @@ def auto_tests(self): results += self.validate_model_definitions(class_manager.class_descriptors, 'NcClassDescriptor', - self.classes_descriptors) + self.reference_class_descriptors) results += self.validate_model_definitions(class_manager.datatype_descriptors, 'NcDatatypeDescriptor', - self.datatype_descriptors) + self.reference_datatype_descriptors) return results def test_01(self, test): @@ -376,6 +376,11 @@ def validate_property_type(self, test, value, type, is_nullable, datatype_schema return def check_get_sequence_item(self, test, oid, sequence_values, property_metadata, context=""): + if sequence_values is None and not property_metadata["isNullable"]: + self.get_sequence_item_metadata["error"] = True + self.get_sequence_item_metadata["error_msg"] += \ + context + property_metadata["name"] + ": Non-nullable property set to null, " + return try: # GetSequenceItem self.get_sequence_item_metadata["checked"] = True @@ -397,6 +402,12 @@ def check_get_sequence_item(self, test, oid, sequence_values, property_metadata, return False def check_get_sequence_length(self, test, oid, sequence_values, property_metadata, context=""): + if sequence_values is None and not property_metadata["isNullable"]: + self.get_sequence_length_metadata["error"] = True + self.get_sequence_length_metadata["error_msg"] += \ + context + property_metadata["name"] + ": Non-nullable property set to null, " + return + try: self.get_sequence_length_metadata["checked"] = True length = self.is12_utils.get_sequence_length(test, oid, property_metadata['id']) @@ -432,7 +443,11 @@ def validate_object_properties(self, test, reference_class_descriptor, oid, data datatype_schemas, context=context + class_property["typeName"] + ": " + class_property["name"] + ": ") - self.check_sequence_methods(test, oid, response, class_property, context=context) + self.check_sequence_methods(test, + oid, + response, + class_property, + context=context) else: self.validate_property_type(test, response, @@ -492,15 +507,15 @@ def check_touchpoints(self, test, oid, datatype_schemas, context): self.touchpoints_metadata["error"] = True self.touchpoints_metadata["error_msg"] = context + str(e.args[0].detail) - def validate_block(self, test, block, class_descriptors, datatype_schemas, context=""): + def check_block(self, test, block, class_descriptors, datatype_schemas, context=""): for child_object in block.child_objects: # If this child object is a Block, recurse if self.is12_utils.is_block(child_object.class_id): - self.validate_block(test, - child_object, - class_descriptors, - datatype_schemas, - context=context + str(child_object.role) + ': ') + self.check_block(test, + child_object, + class_descriptors, + datatype_schemas, + context=context + str(child_object.role) + ': ') role_cache = [] manager_cache = [] for descriptor in block.member_descriptors: @@ -532,12 +547,10 @@ def validate_block(self, test, block, class_descriptors, datatype_schemas, conte + "Non-standard class id does not contain authority key: " \ + str(descriptor['classId']) + ". " - def validate_device_model(self, test): - if not self.device_model_validated: + def check_device_model(self, test): + if not self.device_model_checked: self.create_ncp_socket(test) - class_manager = self.query_class_manager(test) - device_model = self.query_device_model(test) # Create JSON schemas for the queried datatypes @@ -545,12 +558,12 @@ def validate_device_model(self, test): datatype_descriptors=class_manager.datatype_descriptors, schema_path=os.path.join(self.apis[CONTROL_API_KEY]["spec_path"], 'APIs/tmp_schemas/')) - self.validate_block(test, - device_model, - class_manager.class_descriptors, - datatype_schemas) + self.check_block(test, + device_model, + class_manager.class_descriptors, + datatype_schemas) - self.device_model_validated = True + self.device_model_checked = True return def create_nc_object(self, test, class_id, oid, role): @@ -570,7 +583,7 @@ def test_04(self, test): # Referencing the Google sheet # MS-05-02 (34) All workers MUST inherit from NcWorker # MS-05-02 (35) All managers MUST inherit from NcManager - self.validate_device_model(test) + self.check_device_model(test) return test.PASS() @@ -581,7 +594,7 @@ def test_05(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html try: - self.validate_device_model(test) + self.check_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -601,7 +614,7 @@ def test_06(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html try: - self.validate_device_model(test) + self.check_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -625,7 +638,7 @@ def test_07(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html try: - self.validate_device_model(test) + self.check_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -649,7 +662,7 @@ def test_08(self, test): # For IS-08 Audio Channel Mapping the NcTouchpointResourceNmosChannelMapping datatype MUST be used # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#touchpoints try: - self.validate_device_model(test) + self.check_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -671,7 +684,7 @@ def test_09(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html try: - self.validate_device_model(test) + self.check_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -691,7 +704,7 @@ def test_10(self, test): # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html try: - self.validate_device_model(test) + self.check_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -782,7 +795,7 @@ def test_13(self, test): def test_14(self, test): """NcObject: check GetSequenceItem method""" try: - self.validate_device_model(test) + self.check_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) @@ -813,7 +826,7 @@ def test_17(self, test): def test_18(self, test): """NcObject: check GetSequenceLength method""" try: - self.validate_device_model(test) + self.check_device_model(test) except NMOSTestException as e: # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) From 06742174b9977cec4ae20b51edd53ab22b161a76 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Wed, 2 Aug 2023 12:01:48 +0100 Subject: [PATCH 34/45] Created NcBlock object for queried device model --- nmostesting/IS12Utils.py | 31 +++++++++++++----------- nmostesting/suites/IS1201Test.py | 41 ++++++++++++++++---------------- 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index 66a17aaf..119d165b 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -522,20 +522,21 @@ def is_non_standard_class(self, class_id): # Assumes at least one value follows the authority key return len([v for v in dropwhile(lambda x: x > 0, class_id)]) > 1 - def is_block(self, class_id): - """ Check class id to determine if this is a block """ - return len(class_id) > 1 and class_id[0] == 1 and class_id[1] == 1 - def is_manager(self, class_id): """ Check class id to determine if this is a manager """ return len(class_id) > 1 and class_id[0] == 1 and class_id[1] == 3 class NcObject(): - def __init__(self, class_id, oid, role, descriptors): + def __init__(self, class_id, oid, role): self.class_id = class_id self.oid = oid self.role = role + + +class NcBlock(NcObject): + def __init__(self, class_id, oid, role, descriptors): + NcObject.__init__(self, class_id, oid, role) self.child_objects = [] self.member_descriptors = descriptors @@ -545,11 +546,12 @@ def add_child_object(self, nc_object): def get_role_paths(self, root=True): role_paths = [[self.role]] if not root else [] for child_object in self.child_objects: - child_paths = child_object.get_role_paths(False) - for child_path in child_paths: - role_path = [self.role] if not root else [] - role_path += child_path - role_paths.append(role_path) + if type(child_object) is NcBlock: + child_paths = child_object.get_role_paths(False) + for child_path in child_paths: + role_path = [self.role] if not root else [] + role_path += child_path + role_paths.append(role_path) return role_paths def get_member_descriptors(self, recurse=False): @@ -557,14 +559,15 @@ def get_member_descriptors(self, recurse=False): query_results += self.member_descriptors if recurse: for child_object in self.child_objects: - query_results += child_object.get_member_descriptors(recurse) + if type(child_object) is NcBlock: + query_results += child_object.get_member_descriptors(recurse) return query_results def find_members_by_path(self, role_path): query_role = role_path[0] for child_object in self.child_objects: if child_object.role == query_role: - if len(role_path[1:]): + if len(role_path[1:]) and type(child_object) is NcBlock: return child_object.find_members_by_path(role_path[1:]) else: return child_object @@ -580,7 +583,7 @@ def match(query_role, role, case_sensitive, match_whole_string): for child_object in self.child_objects: if match(role, child_object.role, case_sensitive, match_whole_string): query_results.append(child_object) - if recurse: + if recurse and type(child_object) is NcBlock: query_results += child_object.find_members_by_role(role, case_sensitive, match_whole_string, @@ -597,7 +600,7 @@ def match(query_class_id, class_id, include_derived): for child_object in self.child_objects: if match(class_id, child_object.class_id, include_derived): query_results.append(child_object) - if recurse: + if recurse and type(child_object) is NcBlock: query_results += child_object.find_members_by_class_id(class_id, include_derived, recurse) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index ec4b6ecc..ff1773bf 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -21,7 +21,7 @@ from ..GenericTest import GenericTest, NMOSTestException from ..IS12Utils import IS12Utils, NcObject, NcMethodStatus, NcBlockProperties, NcPropertyChangeType,\ NcObjectMethods, NcObjectProperties, NcObjectEvents, NcClassManagerProperties, NcDeviceManagerProperties,\ - StandardClassIds, NcClassManager + StandardClassIds, NcClassManager, NcBlock from ..TestHelper import load_resolved_schema from ..TestResult import Test @@ -267,10 +267,10 @@ def validate_model_definitions(self, descriptors, schema_name, reference_descrip def query_device_model(self, test): self.create_ncp_socket(test) if not self.device_model: - self.device_model = self.create_nc_object(test, - StandardClassIds.NCBLOCK.value, - self.is12_utils.ROOT_BLOCK_OID, - "root") + self.device_model = self.nc_object_factory(test, + StandardClassIds.NCBLOCK.value, + self.is12_utils.ROOT_BLOCK_OID, + "root") return self.device_model def query_class_manager(self, test): @@ -510,7 +510,7 @@ def check_touchpoints(self, test, oid, datatype_schemas, context): def check_block(self, test, block, class_descriptors, datatype_schemas, context=""): for child_object in block.child_objects: # If this child object is a Block, recurse - if self.is12_utils.is_block(child_object.class_id): + if type(child_object) is NcBlock: self.check_block(test, child_object, class_descriptors, @@ -566,17 +566,18 @@ def check_device_model(self, test): self.device_model_checked = True return - def create_nc_object(self, test, class_id, oid, role): - """Create NcObject and child NcObjects""" + def nc_object_factory(self, test, class_id, oid, role): + """Create NcObject or NcBlock based on class_id""" + # Check class id to determine if this is a block + if len(class_id) > 1 and class_id[0] == 1 and class_id[1] == 1: + member_descriptors = self.is12_utils.get_property(test, oid, NcBlockProperties.MEMBERS.value) + nc_block = NcBlock(class_id, oid, role, member_descriptors) - member_descriptors = self.is12_utils.get_property(test, oid, NcBlockProperties.MEMBERS.value) \ - if self.is12_utils.is_block(class_id) else [] - nc_object = NcObject(class_id, oid, role, member_descriptors) - - for m in member_descriptors: - nc_object.add_child_object(self.create_nc_object(test, m["classId"], m["oid"], m["role"])) - - return nc_object + for m in member_descriptors: + nc_block.add_child_object(self.nc_object_factory(test, m["classId"], m["oid"], m["role"])) + return nc_block + else: + return NcObject(class_id, oid, role) def test_04(self, test): """Device Model: check Device Model against classes and datatypes discovered from Class Manager""" @@ -842,7 +843,7 @@ def test_18(self, test): def do_get_member_descriptors_test(self, test, block, context=""): # Recurse through the child blocks for child_object in block.child_objects: - if self.is12_utils.is_block(child_object.class_id): + if type(child_object) is NcBlock: self.do_get_member_descriptors_test(test, child_object, context + block.role + ": ") search_conditions = [{"recurse": True}, {"recurse": False}] @@ -885,7 +886,7 @@ def test_19(self, test): def do_find_member_by_path_test(self, test, block, context=""): # Recurse through the child blocks for child_object in block.child_objects: - if self.is12_utils.is_block(child_object.class_id): + if type(child_object) is NcBlock: self.do_find_member_by_path_test(test, child_object, context + block.role + ": ") # Get ground truth role paths @@ -930,7 +931,7 @@ def test_20(self, test): def do_find_member_by_role_test(self, test, block, context=""): # Recurse through the child blocks for child_object in block.child_objects: - if self.is12_utils.is_block(child_object.class_id): + if type(child_object) is NcBlock: self.do_find_member_by_role_test(test, child_object, context + block.role + ": ") role_paths = IS12Utils.sampled_list(block.get_role_paths()) @@ -990,7 +991,7 @@ def test_21(self, test): def do_find_members_by_class_id_test(self, test, block, context=""): # Recurse through the child blocks for child_object in block.child_objects: - if self.is12_utils.is_block(child_object.class_id): + if type(child_object) is NcBlock: self.do_find_members_by_class_id_test(test, child_object, context + block.role + ": ") class_ids = [e.value for e in StandardClassIds] From 69a096213d783b862232d0c68efd23716f71f716 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Wed, 2 Aug 2023 15:07:02 +0100 Subject: [PATCH 35/45] Fixed get_role_paths. Added get_oids --- nmostesting/IS12Utils.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index 119d165b..62e424e1 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -543,17 +543,26 @@ def __init__(self, class_id, oid, role, descriptors): def add_child_object(self, nc_object): self.child_objects.append(nc_object) - def get_role_paths(self, root=True): - role_paths = [[self.role]] if not root else [] + def get_role_paths(self): + role_paths = [] for child_object in self.child_objects: + role_paths.append([child_object.role]) if type(child_object) is NcBlock: - child_paths = child_object.get_role_paths(False) + child_paths = child_object.get_role_paths() for child_path in child_paths: - role_path = [self.role] if not root else [] + role_path = [child_object.role] role_path += child_path role_paths.append(role_path) return role_paths + def get_oids(self, root=True): + oids = [self.oid] if root else [] + for child_object in self.child_objects: + oids.append(child_object.oid) + if type(child_object) is NcBlock: + oids += child_object.get_oids(False) + return oids + def get_member_descriptors(self, recurse=False): query_results = [] query_results += self.member_descriptors From c9132a765cdda0c2d45cc81f1611ed0409198324 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Wed, 2 Aug 2023 15:07:30 +0100 Subject: [PATCH 36/45] Added illegal and invalid oid tests --- nmostesting/suites/IS1201Test.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index ff1773bf..9667638e 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -1107,10 +1107,24 @@ def test_25(self, test): return self.do_error_test(test, command_json) def test_26(self, test): - """MS-05-02 Error: Node handles invalid oid""" - - # Use invalid oid + """MS-05-02 Error: Node handles illegal oid""" + # Oid should be between 1 and 65535 invalid_oid = 999999999 + command_json = \ + self.is12_utils.create_command_JSON(invalid_oid, + NcObjectMethods.GENERIC_GET.value, + {'id': NcObjectProperties.OID.value}) + + return self.do_error_test(test, + command_json) + + def test_26_1(self, test): + """MS-05-02 Error: Node handles oid not in Device Model""" + device_model = self.query_device_model(test) + # Calculate invalid oid from the max oid value in device model + oids = device_model.get_oids() + invalid_oid = max(oids) + 1 + command_json = \ self.is12_utils.create_command_JSON(invalid_oid, NcObjectMethods.GENERIC_GET.value, From 7714d4b6b78149eaf6a89e659e6885b61abdc5b2 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Wed, 2 Aug 2023 15:20:17 +0100 Subject: [PATCH 37/45] Added illegal command handle test --- nmostesting/suites/IS1201Test.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 9667638e..301077d3 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -1076,8 +1076,19 @@ def do_error_test(self, test, command_json, expected_status=None): return test.PASS() def test_23(self, test): - """IS-12 Protocol Error: Node handles invalid command handle""" + """IS-12 Protocol Error: Node handles command handle not in range 1 to 65535""" + command_json = self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, + NcObjectMethods.GENERIC_GET.value, + {'id': NcObjectProperties.OID.value}) + + # Handle should be between 1 and 65535 + illegal_command_handle = 999999999 + command_json['commands'][0]['handle'] = illegal_command_handle + return self.do_error_test(test, command_json) + + def test_23_1(self, test): + """IS-12 Protocol Error: Node handles command handle not a number""" command_json = self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, NcObjectMethods.GENERIC_GET.value, {'id': NcObjectProperties.OID.value}) @@ -1107,7 +1118,7 @@ def test_25(self, test): return self.do_error_test(test, command_json) def test_26(self, test): - """MS-05-02 Error: Node handles illegal oid""" + """MS-05-02 Error: Node handles oid not in range 1 to 65535""" # Oid should be between 1 and 65535 invalid_oid = 999999999 command_json = \ @@ -1115,11 +1126,10 @@ def test_26(self, test): NcObjectMethods.GENERIC_GET.value, {'id': NcObjectProperties.OID.value}) - return self.do_error_test(test, - command_json) + return self.do_error_test(test, command_json) def test_26_1(self, test): - """MS-05-02 Error: Node handles oid not in Device Model""" + """MS-05-02 Error: Node handles oid of object not in Device Model""" device_model = self.query_device_model(test) # Calculate invalid oid from the max oid value in device model oids = device_model.get_oids() From 9b9cda630810f9ab15c9cd988971228a45368d8b Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Wed, 2 Aug 2023 17:50:06 +0100 Subject: [PATCH 38/45] IndexOutOfBounds test. WebSocket kept open test --- nmostesting/suites/IS1201Test.py | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 301077d3..b73f9cd7 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -14,10 +14,12 @@ import json import os +import time from itertools import product from jsonschema import ValidationError, SchemaError +from ..Config import WS_MESSAGE_TIMEOUT from ..GenericTest import GenericTest, NMOSTestException from ..IS12Utils import IS12Utils, NcObject, NcMethodStatus, NcBlockProperties, NcPropertyChangeType,\ NcObjectMethods, NcObjectProperties, NcObjectEvents, NcClassManagerProperties, NcDeviceManagerProperties,\ @@ -333,6 +335,22 @@ def test_02(self, test): return test.PASS() + def test_02_1(self, test): + """WebSocket: socket is kept open until client closes""" + # Referencing the Google sheet + # IS-12 (2) WebSocket successfully opened on advertised urn:x-nmos:control:ncp endpoint + + self.create_ncp_socket(test) + + # Ensure WebSocket remains open + start_time = time.time() + while time.time() < start_time + WS_MESSAGE_TIMEOUT: + if not self.is12_utils.ncp_websocket.is_open(): + return test.FAIL("Node failed to keep WebSocket open") + time.sleep(0.2) + + return test.PASS() + def test_03(self, test): """Device Model: Root Block exists with correct oid and role""" # Referencing the Google sheet @@ -1184,6 +1202,24 @@ def test_29(self, test): command_json, expected_status=NcMethodStatus.Readonly) + def test_29_1(self, test): + """MS-05-02 Error: Node handles GetSequence index out of bounds """ + self.create_ncp_socket(test) + + length = self.is12_utils.get_sequence_length(test, + self.is12_utils.ROOT_BLOCK_OID, + NcBlockProperties.MEMBERS.value) + out_of_bounds_index = length + 10 + + command_json = \ + self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, + NcObjectMethods.GET_SEQUENCE_ITEM.value, + {'id': NcBlockProperties.MEMBERS.value, + 'index': out_of_bounds_index}) + return self.do_error_test(test, + command_json, + expected_status=NcMethodStatus.IndexOutOfBounds) + def test_30(self, test): """Subscriptions and notifications""" device_model = self.query_device_model(test) From a7e22fb0ec7a519060efd68c51e90ac01c2224f9 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Wed, 2 Aug 2023 21:00:29 +0100 Subject: [PATCH 39/45] Renumbered tests. Added comments --- nmostesting/suites/IS1201Test.py | 275 ++++++++++++++++++++----------- 1 file changed, 183 insertions(+), 92 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index b73f9cd7..97091995 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -313,7 +313,7 @@ def auto_tests(self): return results def test_01(self, test): - """Control Endpoint: Node under test advertising IS-12 control endpoint matching API under test""" + """Control Endpoint: Node under test advertises IS-12 control endpoint matching API under test""" # Referencing the Google sheet # IS-12 (1) Control endpoint advertised in Node endpoint's Device controls array @@ -327,7 +327,7 @@ def test_01(self, test): ) def test_02(self, test): - """WebSocket: successfully opened endpoint""" + """WebSocket: endpoint successfully opened""" # Referencing the Google sheet # IS-12 (2) WebSocket successfully opened on advertised urn:x-nmos:control:ncp endpoint @@ -335,7 +335,7 @@ def test_02(self, test): return test.PASS() - def test_02_1(self, test): + def test_03(self, test): """WebSocket: socket is kept open until client closes""" # Referencing the Google sheet # IS-12 (2) WebSocket successfully opened on advertised urn:x-nmos:control:ncp endpoint @@ -351,7 +351,7 @@ def test_02_1(self, test): return test.PASS() - def test_03(self, test): + def test_04(self, test): """Device Model: Root Block exists with correct oid and role""" # Referencing the Google sheet # MS-05-02 (44) Root Block must exist @@ -597,16 +597,17 @@ def nc_object_factory(self, test, class_id, oid, role): else: return NcObject(class_id, oid, role) - def test_04(self, test): - """Device Model: check Device Model against classes and datatypes discovered from Class Manager""" + def test_05(self, test): + """Device Model: Device Model is correct according to classes and datatypes advertised by Class Manager""" # Referencing the Google sheet # MS-05-02 (34) All workers MUST inherit from NcWorker # MS-05-02 (35) All managers MUST inherit from NcManager + self.check_device_model(test) return test.PASS() - def test_05(self, test): + def test_06(self, test): """Device Model: roles are unique within a containing Block""" # Referencing the Google sheet # MS-05-02 (59) The role of an object MUST be unique within its containing Block. @@ -626,7 +627,7 @@ def test_05(self, test): return test.PASS() - def test_06(self, test): + def test_07(self, test): """Device Model: oids are globally unique""" # Referencing the Google sheet # MS-05-02 (60) Object ids (oid property) MUST uniquely identity objects in the device model. @@ -646,7 +647,7 @@ def test_06(self, test): return test.PASS() - def test_07(self, test): + def test_08(self, test): """Device Model: non-standard classes contain an authority key""" # Referencing the Google sheet # MS-05-02 (72) Non-standard Classes NcClassId @@ -673,13 +674,14 @@ def test_07(self, test): return test.PASS() - def test_08(self, test): - """Device Model: check touchpoint datatypes""" + def test_09(self, test): + """Device Model: touchpoint datatypes are correct""" # Referencing the Google sheet - # MS-05-02 (39) For general NMOS contexts (IS-04, IS-05 and IS-07) the NcTouchpointNmos datatype MUST be used + # MS-05-02 (56) For general NMOS contexts (IS-04, IS-05 and IS-07) the NcTouchpointNmos datatype MUST be used # which has a resource of type NcTouchpointResourceNmos. # For IS-08 Audio Channel Mapping the NcTouchpointResourceNmosChannelMapping datatype MUST be used # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#touchpoints + try: self.check_device_model(test) except NMOSTestException as e: @@ -696,7 +698,7 @@ def test_08(self, test): return test.UNCLEAR("No Touchpoints found.") return test.PASS() - def test_09(self, test): + def test_10(self, test): """Managers: managers are members of the Root Block""" # Referencing the Google sheet # MS-05-02 (36) All managers MUST always exist as members in the Root Block and have a fixed role. @@ -716,7 +718,7 @@ def test_09(self, test): return test.PASS() - def test_10(self, test): + def test_11(self, test): """Managers: managers are singletons""" # Referencing the Google sheet # MS-05-02 (63) Managers are singleton (MUST only be instantiated once) classes. @@ -736,7 +738,7 @@ def test_10(self, test): return test.PASS() - def test_11(self, test): + def test_12(self, test): """Managers: Class Manager exists with correct role""" # Referencing the Google sheet # MS-05-02 (40) Class manager exists in root @@ -747,7 +749,7 @@ def test_11(self, test): return test.PASS() - def test_12(self, test): + def test_13(self, test): """Managers: Device Manager exists with correct Role""" # Referencing the Google sheet # MS-05-02 (37) A minimal device implementation MUST have a device manager in the Root Block. @@ -768,10 +770,57 @@ def test_12(self, test): return test.PASS() - def test_13(self, test): - """NcObject: check Get/Set method""" + def test_14(self, test): + """Class Manager: GetControlClass method is correct""" + # Referencing the Google sheet + # MS-05-02 (93) Where the functionality of a device uses control classes and datatypes listed in this + # specification it MUST comply with the model definitions published + + class_manager = self.query_class_manager(test) + + for _, class_descriptor in class_manager.class_descriptors.items(): + for include_inherited in [False, True]: + actual_descriptor = self.is12_utils.get_control_class(test, + class_manager.oid, + class_descriptor["classId"], + include_inherited) + expected_descriptor = class_manager.get_control_class(class_descriptor["classId"], + include_inherited) + self.validate_descriptor(test, + expected_descriptor, + actual_descriptor, + context=str(class_descriptor["classId"]) + ": ") + + return test.PASS() + + def test_15(self, test): + """Class Manager: GetDatatype method is correct""" + # Referencing the Google sheet + # MS-05-02 (94) Where the functionality of a device uses control classes and datatypes listed in this + # specification it MUST comply with the model definitions published + + class_manager = self.query_class_manager(test) + + for _, datatype_descriptor in class_manager.datatype_descriptors.items(): + for include_inherited in [False, True]: + actual_descriptor = self.is12_utils.get_datatype(test, + class_manager.oid, + datatype_descriptor["name"], + include_inherited) + expected_descriptor = class_manager.get_datatype(datatype_descriptor["name"], + include_inherited) + self.validate_descriptor(test, + expected_descriptor, + actual_descriptor, + context=datatype_descriptor["name"] + ": ") + + return test.PASS() + + def test_16(self, test): + """NcObject: Get and Set methods are correct""" # Referencing the Google sheet - # MS-05-02 (39) Generic getter and setter + # MS-05-02 (39) Generic getter and setter. The value of any property of a control class MUST be retrievable + # using the Get method. # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#generic-getter-and-setter link = "https://specs.amwa.tv/ms-05-02/branches/{}" \ @@ -811,8 +860,12 @@ def test_13(self, test): return test.PASS() - def test_14(self, test): - """NcObject: check GetSequenceItem method""" + def test_17(self, test): + """NcObject: GetSequenceItem method is correct""" + # Referencing the Google sheet + # MS-05-02 (76) Where the functionality of a device uses control classes and datatypes listed in this + # specification it MUST comply with the model definitions published + try: self.check_device_model(test) except NMOSTestException as e: @@ -827,23 +880,36 @@ def test_14(self, test): return test.PASS() - def test_15(self, test): - """NcObject: check SetSequenceItem method""" + def test_18(self, test): + """NcObject: SetSequenceItem method is correct""" + # Referencing the Google sheet + # MS-05-02 (77) Where the functionality of a device uses control classes and datatypes listed in this + # specification it MUST comply with the model definitions published return test.DISABLED() - def test_16(self, test): - """NcObject: check AddSequenceItem method""" + def test_19(self, test): + """NcObject: AddSequenceItem method is correct""" + # Referencing the Google sheet + # MS-05-02 (78) Where the functionality of a device uses control classes and datatypes listed in this + # specification it MUST comply with the model definitions published return test.DISABLED() - def test_17(self, test): - """NcObject: check RemoveSequenceItem method""" + def test_20(self, test): + """NcObject: RemoveSequenceItem method is correct""" + # Referencing the Google sheet + # MS-05-02 (79) Where the functionality of a device uses control classes and datatypes listed in this + # specification it MUST comply with the model definitions published return test.DISABLED() - def test_18(self, test): - """NcObject: check GetSequenceLength method""" + def test_21(self, test): + """NcObject: GetSequenceLength method is correct""" + # Referencing the Google sheet + # MS-05-02 (80) Where the functionality of a device uses control classes and datatypes listed in this + # specification it MUST comply with the model definitions published + try: self.check_device_model(test) except NMOSTestException as e: @@ -893,8 +959,12 @@ def do_get_member_descriptors_test(self, test, block, context=""): + block.role + ": Unsuccessful attempt to get member descriptors.")) - def test_19(self, test): - """NcBlock: check GetMemberDescriptors method""" + def test_22(self, test): + """NcBlock: GetMemberDescriptors method is correct""" + # Referencing the Google sheet + # MS-05-02 (91) Where the functionality of a device uses control classes and datatypes listed in this + # specification it MUST comply with the model definitions published + device_model = self.query_device_model(test) self.do_get_member_descriptors_test(test, device_model) @@ -937,8 +1007,12 @@ def do_find_member_by_path_test(self, test, block, context=""): + ": Unsuccessful attempt to find member by role path: " + str(role_path))) - def test_20(self, test): - """NcBlock: check FindMemberByPath method""" + def test_23(self, test): + """NcBlock: FindMemberByPath method is correct""" + # Referencing the Google sheet + # MS-05-02 (52) Where the functionality of a device uses control classes and datatypes listed in this + # specification it MUST comply with the model definitions published + device_model = self.query_device_model(test) # Recursively check each block in Device Model @@ -997,8 +1071,12 @@ def do_find_member_by_role_test(self, test, block, context=""): + ": Unexpected search result. " + str(actual_result))) - def test_21(self, test): - """NcBlock: check FindMembersByRole method""" + def test_24(self, test): + """NcBlock: FindMembersByRole method is correct""" + # Referencing the Google sheet + # MS-05-02 (52) Where the functionality of a device uses control classes and datatypes listed in this + # specification it MUST comply with the model definitions published + device_model = self.query_device_model(test) # Recursively check each block in Device Model @@ -1048,8 +1126,12 @@ def do_find_members_by_class_id_test(self, test, block, context=""): + block.role + ": Unexpected search result. " + str(actual_result))) - def test_22(self, test): - """NcBlock: check FindMembersByClassId method""" + def test_25(self, test): + """NcBlock: FindMembersByClassId method is correct""" + # Referencing the Google sheet + # MS-05-02 (52) Where the functionality of a device uses control classes and datatypes listed in this + # specification it MUST comply with the model definitions published + device_model = self.query_device_model(test) self.do_find_members_by_class_id_test(test, device_model) @@ -1093,8 +1175,12 @@ def do_error_test(self, test, command_json, expected_status=None): return test.PASS() - def test_23(self, test): - """IS-12 Protocol Error: Node handles command handle not in range 1 to 65535""" + def test_26(self, test): + """IS-12 Protocol Error: Node handles command handle that is not in range 1 to 65535""" + # Referencing the Google sheet + # IS-12 (5) Error messages MUST be used by devices to return general error messages when more specific + # responses cannot be returned + command_json = self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, NcObjectMethods.GENERIC_GET.value, {'id': NcObjectProperties.OID.value}) @@ -1105,8 +1191,12 @@ def test_23(self, test): return self.do_error_test(test, command_json) - def test_23_1(self, test): - """IS-12 Protocol Error: Node handles command handle not a number""" + def test_27(self, test): + """IS-12 Protocol Error: Node handles command handle that is not a number""" + # Referencing the Google sheet + # IS-12 (5) Error messages MUST be used by devices to return general error messages when more specific + # responses cannot be returned + command_json = self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, NcObjectMethods.GENERIC_GET.value, {'id': NcObjectProperties.OID.value}) @@ -1117,8 +1207,12 @@ def test_23_1(self, test): return self.do_error_test(test, command_json) - def test_24(self, test): + def test_28(self, test): """IS-12 Protocol Error: Node handles invalid command type""" + # Referencing the Google sheet + # IS-12 (5) Error messages MUST be used by devices to return general error messages when more specific + # responses cannot be returned + command_json = \ self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, NcObjectMethods.GENERIC_GET.value, @@ -1128,15 +1222,23 @@ def test_24(self, test): return self.do_error_test(test, command_json) - def test_25(self, test): + def test_29(self, test): """IS-12 Protocol Error: Node handles invalid JSON""" + # Referencing the Google sheet + # IS-12 (5) Error messages MUST be used by devices to return general error messages when more specific + # responses cannot be returned + # Use invalid JSON command_json = {'not_a': 'valid_command'} return self.do_error_test(test, command_json) - def test_26(self, test): - """MS-05-02 Error: Node handles oid not in range 1 to 65535""" + def test_30(self, test): + """IS-12 Protocol Error: Node handles oid not in range 1 to 65535""" + # Referencing the Google sheet + # IS-12 (5) Error messages MUST be used by devices to return general error messages when more specific + # responses cannot be returned + # Oid should be between 1 and 65535 invalid_oid = 999999999 command_json = \ @@ -1146,8 +1248,12 @@ def test_26(self, test): return self.do_error_test(test, command_json) - def test_26_1(self, test): - """MS-05-02 Error: Node handles oid of object not in Device Model""" + def test_31(self, test): + """MS-05-02 Error: Node handles oid of object not found in Device Model""" + # Referencing the Google sheet + # MS-05-02 (15) Devices MUST use the exact status code from NcMethodStatus when errors are encountered + # for the following scenarios... + device_model = self.query_device_model(test) # Calculate invalid oid from the max oid value in device model oids = device_model.get_oids() @@ -1162,8 +1268,12 @@ def test_26_1(self, test): command_json, expected_status=NcMethodStatus.BadOid) - def test_27(self, test): + def test_32(self, test): """MS-05-02 Error: Node handles invalid property identifier""" + # Referencing the Google sheet + # MS-05-02 (15) Devices MUST use the exact status code from NcMethodStatus when errors are encountered + # for the following scenarios... + # Use invalid property id invalid_property_identifier = {'level': 1, 'index': 999} command_json = \ @@ -1174,8 +1284,12 @@ def test_27(self, test): command_json, expected_status=NcMethodStatus.PropertyNotImplemented) - def test_28(self, test): + def test_33(self, test): """MS-05-02 Error: Node handles invalid method identifier""" + # Referencing the Google sheet + # MS-05-02 (15) Devices MUST use the exact status code from NcMethodStatus when errors are encountered + # for the following scenarios... + command_json = \ self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, NcObjectMethods.GENERIC_GET.value, @@ -1189,9 +1303,13 @@ def test_28(self, test): command_json, expected_status=NcMethodStatus.MethodNotImplemented) - def test_29(self, test): + def test_34(self, test): """MS-05-02 Error: Node handles read only error""" # Try to set a read only property + # Referencing the Google sheet + # MS-05-02 (15) Devices MUST use the exact status code from NcMethodStatus when errors are encountered + # for the following scenarios... + command_json = \ self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, NcObjectMethods.GENERIC_SET.value, @@ -1202,8 +1320,12 @@ def test_29(self, test): command_json, expected_status=NcMethodStatus.Readonly) - def test_29_1(self, test): - """MS-05-02 Error: Node handles GetSequence index out of bounds """ + def test_35(self, test): + """MS-05-02 Error: Node handles GetSequence index out of bounds error""" + # Referencing the Google sheet + # MS-05-02 (15) Devices MUST use the exact status code from NcMethodStatus when errors are encountered + # for the following scenarios... + self.create_ncp_socket(test) length = self.is12_utils.get_sequence_length(test, @@ -1220,8 +1342,15 @@ def test_29_1(self, test): command_json, expected_status=NcMethodStatus.IndexOutOfBounds) - def test_30(self, test): - """Subscriptions and notifications""" + def test_36(self, test): + """Node implements subscription and notification mechanism""" + # Referencing the Google sheet + # MS-05-02 (12) Notification message type + # MS-05-02 (13) Subscription message type + # MS-05-02 (14) Subscription response message type + # MS-05-02 (17) Property Changed events + # MS-05-02 (21) Check notification is received + device_model = self.query_device_model(test) # Get all oids for objects in this Device Model @@ -1281,41 +1410,3 @@ def test_30(self, test): if error: return test.FAIL(error_message) return test.PASS() - - def test_31(self, test): - """Class Manager: check GetControlClass""" - class_manager = self.query_class_manager(test) - - for _, class_descriptor in class_manager.class_descriptors.items(): - for include_inherited in [False, True]: - actual_descriptor = self.is12_utils.get_control_class(test, - class_manager.oid, - class_descriptor["classId"], - include_inherited) - expected_descriptor = class_manager.get_control_class(class_descriptor["classId"], - include_inherited) - self.validate_descriptor(test, - expected_descriptor, - actual_descriptor, - context=str(class_descriptor["classId"]) + ": ") - - return test.PASS() - - def test_32(self, test): - """Class Manager: check GetDatatype""" - class_manager = self.query_class_manager(test) - - for _, datatype_descriptor in class_manager.datatype_descriptors.items(): - for include_inherited in [False, True]: - actual_descriptor = self.is12_utils.get_datatype(test, - class_manager.oid, - datatype_descriptor["name"], - include_inherited) - expected_descriptor = class_manager.get_datatype(datatype_descriptor["name"], - include_inherited) - self.validate_descriptor(test, - expected_descriptor, - actual_descriptor, - context=datatype_descriptor["name"] + ": ") - - return test.PASS() From df03b2776e074d1477f193f3483dec02b2f99792 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Wed, 2 Aug 2023 21:11:41 +0100 Subject: [PATCH 40/45] Updated comment --- nmostesting/suites/IS1201Test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 97091995..e428671d 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -338,7 +338,7 @@ def test_02(self, test): def test_03(self, test): """WebSocket: socket is kept open until client closes""" # Referencing the Google sheet - # IS-12 (2) WebSocket successfully opened on advertised urn:x-nmos:control:ncp endpoint + # IS-12 (3) Socket is kept open until client closes self.create_ncp_socket(test) From ac0e39d6e764386cac1934131a9687ebb7e76b91 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Thu, 3 Aug 2023 12:05:17 +0100 Subject: [PATCH 41/45] Simplify querying of managers from device model --- nmostesting/IS12Utils.py | 26 +++++++- nmostesting/suites/IS1201Test.py | 100 +++++++++++++------------------ 2 files changed, 66 insertions(+), 60 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index 62e424e1..ba9eacde 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -540,6 +540,7 @@ def __init__(self, class_id, oid, role, descriptors): self.child_objects = [] self.member_descriptors = descriptors + # Utility Methods def add_child_object(self, nc_object): self.child_objects.append(nc_object) @@ -563,6 +564,20 @@ def get_oids(self, root=True): oids += child_object.get_oids(False) return oids + def get_manager(self, test, spec_branch, class_id): + members = self.find_members_by_class_id(class_id, include_derived=True) + + spec_link = "https://specs.amwa.tv/ms-05-02/branches/{}/docs/Managers.html".format(spec_branch) + + if len(members) == 0: + raise NMOSTestException(test.FAIL("Manager not found in Root Block.", spec_link)) + + if len(members) > 1: + raise NMOSTestException(test.FAIL("Manager MUST be a singleton.", spec_link)) + + return members[0] + + # NcBlock Methods def get_member_descriptors(self, recurse=False): query_results = [] query_results += self.member_descriptors @@ -616,9 +631,14 @@ def match(query_class_id, class_id, include_derived): return query_results -class NcClassManager(): - def __init__(self, oid, class_descriptors, datatype_descriptors): - self.oid = oid +class NcManager(NcObject): + def __init__(self, class_id, oid, role): + NcObject.__init__(self, class_id, oid, role) + + +class NcClassManager(NcManager): + def __init__(self, class_id, oid, role, class_descriptors, datatype_descriptors): + NcObject.__init__(self, class_id, oid, role) self.class_descriptors = class_descriptors self.datatype_descriptors = datatype_descriptors diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index e428671d..4b9519e9 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -46,7 +46,6 @@ def __init__(self, apis, **kwargs): self.apis[CONTROL_API_KEY]["spec_branch"]) self.load_reference_resources() self.device_model = None - self.class_manager = None def set_up_tests(self): self.unique_roles_error = False @@ -142,44 +141,6 @@ def create_ncp_socket(self, test): """Create a WebSocket client connection to Node under test. Raises NMOSTestException on error""" self.is12_utils.open_ncp_websocket(test, self.apis[CONTROL_API_KEY]["url"]) - def get_manager(self, test, class_id): - """Get Manager from Root Block. Returns [Manager]. Raises NMOSTestException on error""" - class_id_str = ".".join(map(str, class_id)) - response = self.is12_utils.get_property(test, - self.is12_utils.ROOT_BLOCK_OID, - NcBlockProperties.MEMBERS.value) - - manager_found = False - manager = None - - class_descriptor = self.reference_class_descriptors[class_id_str] - - for value in response: - self._validate_schema(test, - value, - self.datatype_schemas["NcBlockMemberDescriptor"], - context="NcBlockMemberDescriptor: ") - - if value["classId"] == class_descriptor["classId"]: - manager_found = True - manager = value - - if value["role"] != class_descriptor["fixedRole"]: - raise NMOSTestException(test.FAIL("Incorrect Role for Manager " + class_id_str + ": " - + value["role"], - "https://specs.amwa.tv/ms-05-02/branches/{}" - "/docs/Managers.html" - .format(self.apis[CONTROL_API_KEY]["spec_branch"]))) - - if not manager_found: - raise NMOSTestException(test.FAIL(str(class_id_str) + " Manager " - + class_id_str + " not found in Root Block", - "https://specs.amwa.tv/ms-05-02/branches/{}" - "/docs/Managers.html" - .format(self.apis[CONTROL_API_KEY]["spec_branch"]))) - - return manager - def validate_descriptor(self, test, reference, descriptor, context=""): """Validate descriptor against reference descriptor. Raises NMOSTestException on error""" non_normative_keys = ['description'] @@ -277,20 +238,23 @@ def query_device_model(self, test): def query_class_manager(self, test): """Query class manager to use as source of ground truths""" - if not self.class_manager: - self.create_ncp_socket(test) - class_manager_oid = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value)["oid"] + self.create_ncp_socket(test) + device_model = self.query_device_model(test) + + return device_model.get_manager(test, + self.apis[CONTROL_API_KEY]["spec_branch"], + StandardClassIds.NCCLASSMANAGER.value) - class_descriptors = self.get_class_manager_descriptors(test, - class_manager_oid, - NcClassManagerProperties.CONTROL_CLASSES.value) - datatype_descriptors = self.get_class_manager_descriptors(test, - class_manager_oid, - NcClassManagerProperties.DATATYPES.value) - self.class_manager = NcClassManager(class_manager_oid, class_descriptors, datatype_descriptors) + def query_device_manager(self, test): + """Query class manager to use as source of ground truths""" + + self.create_ncp_socket(test) + device_model = self.query_device_model(test) - return self.class_manager + return device_model.get_manager(test, + self.apis[CONTROL_API_KEY]["spec_branch"], + StandardClassIds.NCDEVICEMANAGER.value) def auto_tests(self): """Automatically validate all standard datatypes and control classes. Returns [test result array]""" @@ -595,6 +559,15 @@ def nc_object_factory(self, test, class_id, oid, role): nc_block.add_child_object(self.nc_object_factory(test, m["classId"], m["oid"], m["role"])) return nc_block else: + # Check to determine if this is a Class Manager + if len(class_id) > 2 and class_id[0] == 1 and class_id[1] == 3 and class_id[2] == 2: + class_descriptors = self.get_class_manager_descriptors(test, + oid, + NcClassManagerProperties.CONTROL_CLASSES.value) + datatype_descriptors = self.get_class_manager_descriptors(test, + oid, + NcClassManagerProperties.DATATYPES.value) + return NcClassManager(class_id, oid, role, class_descriptors, datatype_descriptors) return NcObject(class_id, oid, role) def test_05(self, test): @@ -730,7 +703,7 @@ def test_11(self, test): # Couldn't validate model so can't perform test return test.UNCLEAR(e.args[0].detail, e.args[0].link) - if self.managers_members_root_block_error: + if self.managers_are_singletons_error: return test.FAIL("Managers must be singleton classes. ", "https://specs.amwa.tv/ms-05-02/branches/{}" "/docs/Managers.html" @@ -743,9 +716,16 @@ def test_12(self, test): # Referencing the Google sheet # MS-05-02 (40) Class manager exists in root - self.create_ncp_socket(test) + spec_link = "https://specs.amwa.tv/ms-05-02/branches/{}/docs/Managers.html"\ + .format(self.apis[CONTROL_API_KEY]["spec_branch"]) - self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) + class_manager = self.query_class_manager(test) + + class_id_str = ".".join(map(str, StandardClassIds.NCCLASSMANAGER.value)) + class_descriptor = self.reference_class_descriptors[class_id_str] + + if class_manager.role != class_descriptor["fixedRole"]: + return test.FAIL("Class Manager MUST have a role of ClassManager.", spec_link) return test.PASS() @@ -754,20 +734,26 @@ def test_13(self, test): # Referencing the Google sheet # MS-05-02 (37) A minimal device implementation MUST have a device manager in the Root Block. - self.create_ncp_socket(test) + spec_link = "https://specs.amwa.tv/ms-05-02/branches/{}/docs/Managers.html"\ + .format(self.apis[CONTROL_API_KEY]["spec_branch"]) - device_manager = self.get_manager(test, StandardClassIds.NCDEVICEMANAGER.value) + device_manager = self.query_device_manager(test) + + class_id_str = ".".join(map(str, StandardClassIds.NCDEVICEMANAGER.value)) + class_descriptor = self.reference_class_descriptors[class_id_str] + + if device_manager.role != class_descriptor["fixedRole"]: + return test.FAIL("Device Manager MUST have a role of DeviceManager.", spec_link) # Check MS-05-02 Version property_id = NcDeviceManagerProperties.NCVERSION.value - version = self.is12_utils.get_property(test, device_manager['oid'], property_id) + version = self.is12_utils.get_property(test, device_manager.oid, property_id) if self.is12_utils.compare_api_version(version, self.apis[MS05_API_KEY]["version"]): return test.FAIL("Unexpected version. Expected: " + self.apis[MS05_API_KEY]["version"] + ". Actual: " + str(version)) - return test.PASS() def test_14(self, test): From 4bb4efb2518af8b5db7ee3371c88b98e642d2bd1 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Thu, 3 Aug 2023 20:28:44 +0100 Subject: [PATCH 42/45] Removed redundant oid test --- nmostesting/suites/IS1201Test.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 4b9519e9..de2cdf29 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -1220,21 +1220,6 @@ def test_29(self, test): return self.do_error_test(test, command_json) def test_30(self, test): - """IS-12 Protocol Error: Node handles oid not in range 1 to 65535""" - # Referencing the Google sheet - # IS-12 (5) Error messages MUST be used by devices to return general error messages when more specific - # responses cannot be returned - - # Oid should be between 1 and 65535 - invalid_oid = 999999999 - command_json = \ - self.is12_utils.create_command_JSON(invalid_oid, - NcObjectMethods.GENERIC_GET.value, - {'id': NcObjectProperties.OID.value}) - - return self.do_error_test(test, command_json) - - def test_31(self, test): """MS-05-02 Error: Node handles oid of object not found in Device Model""" # Referencing the Google sheet # MS-05-02 (15) Devices MUST use the exact status code from NcMethodStatus when errors are encountered @@ -1254,7 +1239,7 @@ def test_31(self, test): command_json, expected_status=NcMethodStatus.BadOid) - def test_32(self, test): + def test_31(self, test): """MS-05-02 Error: Node handles invalid property identifier""" # Referencing the Google sheet # MS-05-02 (15) Devices MUST use the exact status code from NcMethodStatus when errors are encountered @@ -1270,7 +1255,7 @@ def test_32(self, test): command_json, expected_status=NcMethodStatus.PropertyNotImplemented) - def test_33(self, test): + def test_32(self, test): """MS-05-02 Error: Node handles invalid method identifier""" # Referencing the Google sheet # MS-05-02 (15) Devices MUST use the exact status code from NcMethodStatus when errors are encountered @@ -1289,7 +1274,7 @@ def test_33(self, test): command_json, expected_status=NcMethodStatus.MethodNotImplemented) - def test_34(self, test): + def test_33(self, test): """MS-05-02 Error: Node handles read only error""" # Try to set a read only property # Referencing the Google sheet @@ -1306,7 +1291,7 @@ def test_34(self, test): command_json, expected_status=NcMethodStatus.Readonly) - def test_35(self, test): + def test_34(self, test): """MS-05-02 Error: Node handles GetSequence index out of bounds error""" # Referencing the Google sheet # MS-05-02 (15) Devices MUST use the exact status code from NcMethodStatus when errors are encountered @@ -1328,7 +1313,7 @@ def test_35(self, test): command_json, expected_status=NcMethodStatus.IndexOutOfBounds) - def test_36(self, test): + def test_35(self, test): """Node implements subscription and notification mechanism""" # Referencing the Google sheet # MS-05-02 (12) Notification message type From 6af4df6642d57a7369e4797350bea228af90dec1 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Fri, 4 Aug 2023 09:37:24 +0100 Subject: [PATCH 43/45] correct linting error --- nmostesting/suites/IS1201Test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index de2cdf29..55bc273b 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -21,8 +21,8 @@ from ..Config import WS_MESSAGE_TIMEOUT from ..GenericTest import GenericTest, NMOSTestException -from ..IS12Utils import IS12Utils, NcObject, NcMethodStatus, NcBlockProperties, NcPropertyChangeType,\ - NcObjectMethods, NcObjectProperties, NcObjectEvents, NcClassManagerProperties, NcDeviceManagerProperties,\ +from ..IS12Utils import IS12Utils, NcObject, NcMethodStatus, NcBlockProperties, NcPropertyChangeType, \ + NcObjectMethods, NcObjectProperties, NcObjectEvents, NcClassManagerProperties, NcDeviceManagerProperties, \ StandardClassIds, NcClassManager, NcBlock from ..TestHelper import load_resolved_schema from ..TestResult import Test From 4dabde7f767277c67aac29f2a99bb90da5ff6188 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Fri, 4 Aug 2023 11:39:05 +0100 Subject: [PATCH 44/45] Move get_manager out of NcBlock --- nmostesting/IS12Utils.py | 13 ------------ nmostesting/suites/IS1201Test.py | 34 ++++++++++++++------------------ 2 files changed, 15 insertions(+), 32 deletions(-) diff --git a/nmostesting/IS12Utils.py b/nmostesting/IS12Utils.py index ba9eacde..66716b55 100644 --- a/nmostesting/IS12Utils.py +++ b/nmostesting/IS12Utils.py @@ -564,19 +564,6 @@ def get_oids(self, root=True): oids += child_object.get_oids(False) return oids - def get_manager(self, test, spec_branch, class_id): - members = self.find_members_by_class_id(class_id, include_derived=True) - - spec_link = "https://specs.amwa.tv/ms-05-02/branches/{}/docs/Managers.html".format(spec_branch) - - if len(members) == 0: - raise NMOSTestException(test.FAIL("Manager not found in Root Block.", spec_link)) - - if len(members) > 1: - raise NMOSTestException(test.FAIL("Manager MUST be a singleton.", spec_link)) - - return members[0] - # NcBlock Methods def get_member_descriptors(self, recurse=False): query_results = [] diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 55bc273b..323a30de 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -236,25 +236,21 @@ def query_device_model(self, test): "root") return self.device_model - def query_class_manager(self, test): - """Query class manager to use as source of ground truths""" - + def get_manager(self, test, class_id): self.create_ncp_socket(test) device_model = self.query_device_model(test) + members = device_model.find_members_by_class_id(class_id, include_derived=True) - return device_model.get_manager(test, - self.apis[CONTROL_API_KEY]["spec_branch"], - StandardClassIds.NCCLASSMANAGER.value) + spec_link = "https://specs.amwa.tv/ms-05-02/branches/{}/docs/Managers.html"\ + .format(self.apis[CONTROL_API_KEY]["spec_branch"]) - def query_device_manager(self, test): - """Query class manager to use as source of ground truths""" + if len(members) == 0: + raise NMOSTestException(test.FAIL("Manager not found in Root Block.", spec_link)) - self.create_ncp_socket(test) - device_model = self.query_device_model(test) + if len(members) > 1: + raise NMOSTestException(test.FAIL("Manager MUST be a singleton.", spec_link)) - return device_model.get_manager(test, - self.apis[CONTROL_API_KEY]["spec_branch"], - StandardClassIds.NCDEVICEMANAGER.value) + return members[0] def auto_tests(self): """Automatically validate all standard datatypes and control classes. Returns [test result array]""" @@ -265,7 +261,7 @@ def auto_tests(self): self.create_ncp_socket(test) - class_manager = self.query_class_manager(test) + class_manager = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) results += self.validate_model_definitions(class_manager.class_descriptors, 'NcClassDescriptor', @@ -532,7 +528,7 @@ def check_block(self, test, block, class_descriptors, datatype_schemas, context= def check_device_model(self, test): if not self.device_model_checked: self.create_ncp_socket(test) - class_manager = self.query_class_manager(test) + class_manager = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) device_model = self.query_device_model(test) # Create JSON schemas for the queried datatypes @@ -719,7 +715,7 @@ def test_12(self, test): spec_link = "https://specs.amwa.tv/ms-05-02/branches/{}/docs/Managers.html"\ .format(self.apis[CONTROL_API_KEY]["spec_branch"]) - class_manager = self.query_class_manager(test) + class_manager = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) class_id_str = ".".join(map(str, StandardClassIds.NCCLASSMANAGER.value)) class_descriptor = self.reference_class_descriptors[class_id_str] @@ -737,7 +733,7 @@ def test_13(self, test): spec_link = "https://specs.amwa.tv/ms-05-02/branches/{}/docs/Managers.html"\ .format(self.apis[CONTROL_API_KEY]["spec_branch"]) - device_manager = self.query_device_manager(test) + device_manager = self.get_manager(test, StandardClassIds.NCDEVICEMANAGER.value) class_id_str = ".".join(map(str, StandardClassIds.NCDEVICEMANAGER.value)) class_descriptor = self.reference_class_descriptors[class_id_str] @@ -762,7 +758,7 @@ def test_14(self, test): # MS-05-02 (93) Where the functionality of a device uses control classes and datatypes listed in this # specification it MUST comply with the model definitions published - class_manager = self.query_class_manager(test) + class_manager = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) for _, class_descriptor in class_manager.class_descriptors.items(): for include_inherited in [False, True]: @@ -785,7 +781,7 @@ def test_15(self, test): # MS-05-02 (94) Where the functionality of a device uses control classes and datatypes listed in this # specification it MUST comply with the model definitions published - class_manager = self.query_class_manager(test) + class_manager = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) for _, datatype_descriptor in class_manager.datatype_descriptors.items(): for include_inherited in [False, True]: From ebe162e6355596c73fddd52aa314014b8701bee9 Mon Sep 17 00:00:00 2001 From: "Jonathan Thorpe (Sony)" Date: Fri, 4 Aug 2023 14:30:41 +0100 Subject: [PATCH 45/45] Updated comments and links to specification --- nmostesting/suites/IS1201Test.py | 166 ++++++++++++++----------------- 1 file changed, 77 insertions(+), 89 deletions(-) diff --git a/nmostesting/suites/IS1201Test.py b/nmostesting/suites/IS1201Test.py index 323a30de..9d1b1add 100644 --- a/nmostesting/suites/IS1201Test.py +++ b/nmostesting/suites/IS1201Test.py @@ -254,8 +254,8 @@ def get_manager(self, test, class_id): def auto_tests(self): """Automatically validate all standard datatypes and control classes. Returns [test result array]""" - # Referencing the Google sheet - # MS-05-02 (75) Model definitions + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html + results = list() test = Test("Initialize auto tests", "auto_init") @@ -274,8 +274,7 @@ def auto_tests(self): def test_01(self, test): """Control Endpoint: Node under test advertises IS-12 control endpoint matching API under test""" - # Referencing the Google sheet - # IS-12 (1) Control endpoint advertised in Node endpoint's Device controls array + # https://specs.amwa.tv/is-12/branches/v1.0/docs/IS-04_interactions.html control_type = "urn:x-nmos:control:ncp/" + self.apis[CONTROL_API_KEY]["version"] return self.is12_utils.do_test_device_control( @@ -288,8 +287,7 @@ def test_01(self, test): def test_02(self, test): """WebSocket: endpoint successfully opened""" - # Referencing the Google sheet - # IS-12 (2) WebSocket successfully opened on advertised urn:x-nmos:control:ncp endpoint + # https://specs.amwa.tv/is-12/branches/v1.0/docs/Transport_and_message_encoding.html self.create_ncp_socket(test) @@ -297,8 +295,7 @@ def test_02(self, test): def test_03(self, test): """WebSocket: socket is kept open until client closes""" - # Referencing the Google sheet - # IS-12 (3) Socket is kept open until client closes + # https://specs.amwa.tv/is-12/branches/v1.0/docs/Protocol_messaging.html#control-session self.create_ncp_socket(test) @@ -313,10 +310,7 @@ def test_03(self, test): def test_04(self, test): """Device Model: Root Block exists with correct oid and role""" - # Referencing the Google sheet - # MS-05-02 (44) Root Block must exist - # MS-05-02 (45) Verify oID and role of Root Block - # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Blocks.html + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Blocks.html self.create_ncp_socket(test) @@ -568,9 +562,8 @@ def nc_object_factory(self, test, class_id, oid, role): def test_05(self, test): """Device Model: Device Model is correct according to classes and datatypes advertised by Class Manager""" - # Referencing the Google sheet - # MS-05-02 (34) All workers MUST inherit from NcWorker - # MS-05-02 (35) All managers MUST inherit from NcManager + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Managers.html + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Workers.html self.check_device_model(test) @@ -578,9 +571,7 @@ def test_05(self, test): def test_06(self, test): """Device Model: roles are unique within a containing Block""" - # Referencing the Google sheet - # MS-05-02 (59) The role of an object MUST be unique within its containing Block. - # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/NcObject.html try: self.check_device_model(test) @@ -598,9 +589,7 @@ def test_06(self, test): def test_07(self, test): """Device Model: oids are globally unique""" - # Referencing the Google sheet - # MS-05-02 (60) Object ids (oid property) MUST uniquely identity objects in the device model. - # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/NcObject.html try: self.check_device_model(test) @@ -618,13 +607,10 @@ def test_07(self, test): def test_08(self, test): """Device Model: non-standard classes contain an authority key""" - # Referencing the Google sheet - # MS-05-02 (72) Non-standard Classes NcClassId - # MS-05-02 (73) Organization Identifier # For organizations which own a unique CID or OUI the authority key MUST be the organization # identifier as an integer which MUST be negated. # For organizations which do not own a unique CID or OUI the authority key MUST be 0 - # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncclassid try: self.check_device_model(test) @@ -645,11 +631,10 @@ def test_08(self, test): def test_09(self, test): """Device Model: touchpoint datatypes are correct""" - # Referencing the Google sheet - # MS-05-02 (56) For general NMOS contexts (IS-04, IS-05 and IS-07) the NcTouchpointNmos datatype MUST be used + # For general NMOS contexts (IS-04, IS-05 and IS-07) the NcTouchpointNmos datatype MUST be used # which has a resource of type NcTouchpointResourceNmos. # For IS-08 Audio Channel Mapping the NcTouchpointResourceNmosChannelMapping datatype MUST be used - # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#touchpoints + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/NcObject.html#touchpoints try: self.check_device_model(test) @@ -669,9 +654,8 @@ def test_09(self, test): def test_10(self, test): """Managers: managers are members of the Root Block""" - # Referencing the Google sheet - # MS-05-02 (36) All managers MUST always exist as members in the Root Block and have a fixed role. - # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html + # All managers MUST always exist as members in the Root Block and have a fixed role. + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Managers.html try: self.check_device_model(test) @@ -689,9 +673,8 @@ def test_10(self, test): def test_11(self, test): """Managers: managers are singletons""" - # Referencing the Google sheet - # MS-05-02 (63) Managers are singleton (MUST only be instantiated once) classes. - # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/Managers.html + # Managers are singleton (MUST only be instantiated once) classes. + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Managers.html try: self.check_device_model(test) @@ -709,8 +692,8 @@ def test_11(self, test): def test_12(self, test): """Managers: Class Manager exists with correct role""" - # Referencing the Google sheet - # MS-05-02 (40) Class manager exists in root + # Class manager exists in root + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Managers.html spec_link = "https://specs.amwa.tv/ms-05-02/branches/{}/docs/Managers.html"\ .format(self.apis[CONTROL_API_KEY]["spec_branch"]) @@ -727,8 +710,8 @@ def test_12(self, test): def test_13(self, test): """Managers: Device Manager exists with correct Role""" - # Referencing the Google sheet - # MS-05-02 (37) A minimal device implementation MUST have a device manager in the Root Block. + # A minimal device implementation MUST have a device manager in the Root Block. + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Managers.html spec_link = "https://specs.amwa.tv/ms-05-02/branches/{}/docs/Managers.html"\ .format(self.apis[CONTROL_API_KEY]["spec_branch"]) @@ -754,9 +737,9 @@ def test_13(self, test): def test_14(self, test): """Class Manager: GetControlClass method is correct""" - # Referencing the Google sheet - # MS-05-02 (93) Where the functionality of a device uses control classes and datatypes listed in this + # Where the functionality of a device uses control classes and datatypes listed in this # specification it MUST comply with the model definitions published + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncclassmanager class_manager = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) @@ -777,9 +760,9 @@ def test_14(self, test): def test_15(self, test): """Class Manager: GetDatatype method is correct""" - # Referencing the Google sheet - # MS-05-02 (94) Where the functionality of a device uses control classes and datatypes listed in this + # Where the functionality of a device uses control classes and datatypes listed in this # specification it MUST comply with the model definitions published + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncclassmanager class_manager = self.get_manager(test, StandardClassIds.NCCLASSMANAGER.value) @@ -800,10 +783,9 @@ def test_15(self, test): def test_16(self, test): """NcObject: Get and Set methods are correct""" - # Referencing the Google sheet - # MS-05-02 (39) Generic getter and setter. The value of any property of a control class MUST be retrievable + # Generic getter and setter. The value of any property of a control class MUST be retrievable # using the Get method. - # https://specs.amwa.tv/ms-05-02/branches/v1.0-dev/docs/NcObject.html#generic-getter-and-setter + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/NcObject.html#generic-getter-and-setter link = "https://specs.amwa.tv/ms-05-02/branches/{}" \ "/docs/NcObject.html#generic-getter-and-setter" \ @@ -844,9 +826,9 @@ def test_16(self, test): def test_17(self, test): """NcObject: GetSequenceItem method is correct""" - # Referencing the Google sheet - # MS-05-02 (76) Where the functionality of a device uses control classes and datatypes listed in this + # Where the functionality of a device uses control classes and datatypes listed in this # specification it MUST comply with the model definitions published + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncobject try: self.check_device_model(test) @@ -864,33 +846,33 @@ def test_17(self, test): def test_18(self, test): """NcObject: SetSequenceItem method is correct""" - # Referencing the Google sheet - # MS-05-02 (77) Where the functionality of a device uses control classes and datatypes listed in this + # Where the functionality of a device uses control classes and datatypes listed in this # specification it MUST comply with the model definitions published + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncobject return test.DISABLED() def test_19(self, test): """NcObject: AddSequenceItem method is correct""" - # Referencing the Google sheet - # MS-05-02 (78) Where the functionality of a device uses control classes and datatypes listed in this + # Where the functionality of a device uses control classes and datatypes listed in this # specification it MUST comply with the model definitions published + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncobject return test.DISABLED() def test_20(self, test): """NcObject: RemoveSequenceItem method is correct""" - # Referencing the Google sheet - # MS-05-02 (79) Where the functionality of a device uses control classes and datatypes listed in this + # Where the functionality of a device uses control classes and datatypes listed in this # specification it MUST comply with the model definitions published + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncobject return test.DISABLED() def test_21(self, test): """NcObject: GetSequenceLength method is correct""" - # Referencing the Google sheet - # MS-05-02 (80) Where the functionality of a device uses control classes and datatypes listed in this + # Where the functionality of a device uses control classes and datatypes listed in this # specification it MUST comply with the model definitions published + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncobject try: self.check_device_model(test) @@ -943,9 +925,9 @@ def do_get_member_descriptors_test(self, test, block, context=""): def test_22(self, test): """NcBlock: GetMemberDescriptors method is correct""" - # Referencing the Google sheet - # MS-05-02 (91) Where the functionality of a device uses control classes and datatypes listed in this + # Where the functionality of a device uses control classes and datatypes listed in this # specification it MUST comply with the model definitions published + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncblock device_model = self.query_device_model(test) @@ -991,9 +973,9 @@ def do_find_member_by_path_test(self, test, block, context=""): def test_23(self, test): """NcBlock: FindMemberByPath method is correct""" - # Referencing the Google sheet - # MS-05-02 (52) Where the functionality of a device uses control classes and datatypes listed in this + # Where the functionality of a device uses control classes and datatypes listed in this # specification it MUST comply with the model definitions published + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncblock device_model = self.query_device_model(test) @@ -1055,9 +1037,9 @@ def do_find_member_by_role_test(self, test, block, context=""): def test_24(self, test): """NcBlock: FindMembersByRole method is correct""" - # Referencing the Google sheet - # MS-05-02 (52) Where the functionality of a device uses control classes and datatypes listed in this + # Where the functionality of a device uses control classes and datatypes listed in this # specification it MUST comply with the model definitions published + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncblock device_model = self.query_device_model(test) @@ -1110,9 +1092,9 @@ def do_find_members_by_class_id_test(self, test, block, context=""): def test_25(self, test): """NcBlock: FindMembersByClassId method is correct""" - # Referencing the Google sheet - # MS-05-02 (52) Where the functionality of a device uses control classes and datatypes listed in this + # Where the functionality of a device uses control classes and datatypes listed in this # specification it MUST comply with the model definitions published + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncblock device_model = self.query_device_model(test) @@ -1130,7 +1112,10 @@ def do_error_test(self, test, command_json, expected_status=None): self.is12_utils.send_command(test, command_json) - return test.FAIL("Error expected") + return test.FAIL("Error not handled.", + "https://specs.amwa.tv/is-12/branches/{}" + "/docs/Protocol_messaging.html#error-messages" + .format(self.apis[CONTROL_API_KEY]["spec_branch"])) except NMOSTestException as e: error_msg = e.args[0].detail @@ -1147,21 +1132,27 @@ def do_error_test(self, test, command_json, expected_status=None): return test.FAIL("Error not handled. Expected: " + expected_status.name + " (" + str(expected_status) + ")" + ", actual: " + NcMethodStatus(error_msg['status']).name - + " (" + str(error_msg['status']) + ")") + + " (" + str(error_msg['status']) + ")", + "https://specs.amwa.tv/is-12/branches/{}" + "/docs/Protocol_messaging.html#error-messages" + .format(self.apis[CONTROL_API_KEY]["spec_branch"])) if expected_status and error_msg['status'] != expected_status: return test.WARNING("Unexpected status. Expected: " + expected_status.name + " (" + str(expected_status) + ")" + ", actual: " + NcMethodStatus(error_msg['status']).name - + " (" + str(error_msg['status']) + ")") + + " (" + str(error_msg['status']) + ")", + "https://specs.amwa.tv/ms-05-02/branches/{}" + "/docs/Framework.html#ncmethodresult" + .format(self.apis[CONTROL_API_KEY]["spec_branch"])) return test.PASS() def test_26(self, test): """IS-12 Protocol Error: Node handles command handle that is not in range 1 to 65535""" - # Referencing the Google sheet - # IS-12 (5) Error messages MUST be used by devices to return general error messages when more specific + # Error messages MUST be used by devices to return general error messages when more specific # responses cannot be returned + # https://specs.amwa.tv/is-12/branches/v1.0/docs/Protocol_messaging.html#error-messages command_json = self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, NcObjectMethods.GENERIC_GET.value, @@ -1175,9 +1166,9 @@ def test_26(self, test): def test_27(self, test): """IS-12 Protocol Error: Node handles command handle that is not a number""" - # Referencing the Google sheet - # IS-12 (5) Error messages MUST be used by devices to return general error messages when more specific + # Error messages MUST be used by devices to return general error messages when more specific # responses cannot be returned + # https://specs.amwa.tv/is-12/branches/v1.0/docs/Protocol_messaging.html#error-messages command_json = self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, NcObjectMethods.GENERIC_GET.value, @@ -1191,9 +1182,9 @@ def test_27(self, test): def test_28(self, test): """IS-12 Protocol Error: Node handles invalid command type""" - # Referencing the Google sheet - # IS-12 (5) Error messages MUST be used by devices to return general error messages when more specific + # Error messages MUST be used by devices to return general error messages when more specific # responses cannot be returned + # https://specs.amwa.tv/is-12/branches/v1.0/docs/Protocol_messaging.html#error-messages command_json = \ self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, @@ -1206,9 +1197,9 @@ def test_28(self, test): def test_29(self, test): """IS-12 Protocol Error: Node handles invalid JSON""" - # Referencing the Google sheet - # IS-12 (5) Error messages MUST be used by devices to return general error messages when more specific + # Error messages MUST be used by devices to return general error messages when more specific # responses cannot be returned + # https://specs.amwa.tv/is-12/branches/v1.0/docs/Protocol_messaging.html#error-messages # Use invalid JSON command_json = {'not_a': 'valid_command'} @@ -1237,9 +1228,9 @@ def test_30(self, test): def test_31(self, test): """MS-05-02 Error: Node handles invalid property identifier""" - # Referencing the Google sheet - # MS-05-02 (15) Devices MUST use the exact status code from NcMethodStatus when errors are encountered + # Devices MUST use the exact status code from NcMethodStatus when errors are encountered # for the following scenarios... + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncmethodresult # Use invalid property id invalid_property_identifier = {'level': 1, 'index': 999} @@ -1253,9 +1244,9 @@ def test_31(self, test): def test_32(self, test): """MS-05-02 Error: Node handles invalid method identifier""" - # Referencing the Google sheet - # MS-05-02 (15) Devices MUST use the exact status code from NcMethodStatus when errors are encountered + # Devices MUST use the exact status code from NcMethodStatus when errors are encountered # for the following scenarios... + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncmethodresult command_json = \ self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, @@ -1272,10 +1263,9 @@ def test_32(self, test): def test_33(self, test): """MS-05-02 Error: Node handles read only error""" - # Try to set a read only property - # Referencing the Google sheet - # MS-05-02 (15) Devices MUST use the exact status code from NcMethodStatus when errors are encountered + # Devices MUST use the exact status code from NcMethodStatus when errors are encountered # for the following scenarios... + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncmethodresult command_json = \ self.is12_utils.create_command_JSON(self.is12_utils.ROOT_BLOCK_OID, @@ -1289,9 +1279,9 @@ def test_33(self, test): def test_34(self, test): """MS-05-02 Error: Node handles GetSequence index out of bounds error""" - # Referencing the Google sheet - # MS-05-02 (15) Devices MUST use the exact status code from NcMethodStatus when errors are encountered + # Devices MUST use the exact status code from NcMethodStatus when errors are encountered # for the following scenarios... + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/Framework.html#ncmethodresult self.create_ncp_socket(test) @@ -1311,12 +1301,10 @@ def test_34(self, test): def test_35(self, test): """Node implements subscription and notification mechanism""" - # Referencing the Google sheet - # MS-05-02 (12) Notification message type - # MS-05-02 (13) Subscription message type - # MS-05-02 (14) Subscription response message type - # MS-05-02 (17) Property Changed events - # MS-05-02 (21) Check notification is received + # https://specs.amwa.tv/ms-05-02/branches/v1.0/docs/NcObject.html#propertychanged-event + # https://specs.amwa.tv/is-12/branches/v1.0/docs/Protocol_messaging.html#notification-message-type + # https://specs.amwa.tv/is-12/branches/v1.0/docs/Protocol_messaging.html#subscription-message-type + # https://specs.amwa.tv/is-12/branches/v1.0/docs/Protocol_messaging.html#subscription-response-message-type device_model = self.query_device_model(test)