From 87e65a731ee0edad576de202ca9e48515a24bedd Mon Sep 17 00:00:00 2001 From: Anirudh9794 Date: Mon, 23 Aug 2021 11:03:14 -0700 Subject: [PATCH] [VCDA-2588] Set up environment for RDE based tests (#1070) (#1155) * [VCDA-2588] Set up environment for RDE based tests (#1070) * set up environment for RDE based CSE tests Signed-off-by: Aniruddha Shamasundar * Added server tests back Signed-off-by: Aniruddha Shamasundar * Clean up defined entity artifacts after the tests are complete Signed-off-by: Aniruddha Shamasundar * Server tests changes Signed-off-by: Aniruddha Shamasundar * Addressed review comments Signed-off-by: Aniruddha Shamasundar * Addressed review comments Signed-off-by: Aniruddha Shamasundar * Fix flake8 errors Signed-off-by: Aniruddha Shamasundar * Address review comments Signed-off-by: Aniruddha Shamasundar * Address review comment Signed-off-by: Aniruddha Shamasundar * Change default template Signed-off-by: Aniruddha Shamasundar --- .../mqi/mqtt_extension_manager.py | 16 + .../system_test_framework/environment.py | 417 +++++++++++-- system_tests_v2/README.md | 141 +++++ system_tests_v2/__init__.py | 0 system_tests_v2/base_config.yaml | 64 ++ system_tests_v2/conftest.py | 124 ++++ system_tests_v2/pytest_logger.py | 10 + system_tests_v2/test_cse_server.py | 590 ++++++++++++++++++ test-requirements.txt | 3 +- tox.ini | 2 +- 10 files changed, 1327 insertions(+), 40 deletions(-) create mode 100644 system_tests_v2/README.md create mode 100644 system_tests_v2/__init__.py create mode 100644 system_tests_v2/base_config.yaml create mode 100644 system_tests_v2/conftest.py create mode 100644 system_tests_v2/pytest_logger.py create mode 100644 system_tests_v2/test_cse_server.py diff --git a/container_service_extension/mqi/mqtt_extension_manager.py b/container_service_extension/mqi/mqtt_extension_manager.py index 51a83f472..7a798a1e1 100644 --- a/container_service_extension/mqi/mqtt_extension_manager.py +++ b/container_service_extension/mqi/mqtt_extension_manager.py @@ -322,6 +322,22 @@ def check_extension_exists(self, ext_urn_id): self._debug_logger.debug(last_response.text) return False + def is_extension_enabled(self, ext_urn_id): + """Check if the MQTT Extension is enabled. + + :param str ext_urn_id: the extension urn id + + :return: boolean indicating if extension is enabled + :rtype: bool + """ + try: + mqtt_ext_obj = self.get_extension_response_body_by_urn(ext_urn_id) + return mqtt_ext_obj['enabled'] + except requests.exceptions.HTTPError: + last_response = self._cloudapi_client.get_last_response() + self._debug_logger.debug(last_response.text) + return False + def setup_extension_token(self, token_name, ext_name, ext_version, ext_vendor, ext_urn_id): """Handle setting up a single extension token. diff --git a/container_service_extension/system_test_framework/environment.py b/container_service_extension/system_test_framework/environment.py index 33ab9e1e7..3653d6c30 100644 --- a/container_service_extension/system_test_framework/environment.py +++ b/container_service_extension/system_test_framework/environment.py @@ -4,6 +4,7 @@ import os from pathlib import Path +from typing import List from click.testing import CliRunner from pyvcloud.vcd.api_extension import APIExtension @@ -12,14 +13,23 @@ from pyvcloud.vcd.exceptions import EntityNotFoundException from pyvcloud.vcd.exceptions import MissingRecordException from pyvcloud.vcd.org import Org +from pyvcloud.vcd.role import Role from pyvcloud.vcd.vdc import VDC from vcd_cli.vcd import vcd -from container_service_extension.common.constants.server_constants import CSE_SERVICE_NAME # noqa: E501 -from container_service_extension.common.constants.server_constants import CSE_SERVICE_NAMESPACE # noqa: E501 -from container_service_extension.common.constants.shared_constants import SYSTEM_ORG_NAME # noqa: E501 +import container_service_extension.common.constants.server_constants as server_constants # noqa: E501 +import container_service_extension.common.constants.shared_constants as shared_constants # noqa: E501 +from container_service_extension.common.utils.core_utils import get_max_api_version # noqa: E501 import container_service_extension.common.utils.pyvcloud_utils as pyvcloud_utils # noqa: E501 +from container_service_extension.installer.right_bundle_manager import RightBundleManager # noqa: E501 from container_service_extension.installer.templates.remote_template_manager import RemoteTemplateManager # noqa: E501 +from container_service_extension.logging.logger import NULL_LOGGER, SERVER_CLOUDAPI_WIRE_LOGGER # noqa: E501 +from container_service_extension.mqi.mqtt_extension_manager import \ + MQTTExtensionManager +import container_service_extension.rde.constants as rde_constants +from container_service_extension.rde.models import common_models +import container_service_extension.rde.schema_service as def_schema_svc +import container_service_extension.rde.utils as rde_utils import container_service_extension.system_test_framework.utils as testutils @@ -42,17 +52,29 @@ BASE_CONFIG_FILEPATH = 'base_config.yaml' ACTIVE_CONFIG_FILEPATH = 'cse_test_config.yaml' TEMPLATE_DEFINITIONS = None +TEMPLATE_COOKBOOK_VERSION = None SCRIPTS_DIR = 'scripts' SSH_KEY_FILEPATH = str(Path.home() / '.ssh' / 'id_rsa.pub') CLI_RUNNER = CliRunner() SYS_ADMIN_TEST_CLUSTER_NAME = 'testclustersystem' +CLUSTER_ADMIN_TEST_CLUSTER_NAME = 'testclusteradmin' +CLUSTER_AUTHOR_TEST_CLUSTER_NAME = 'testclusterauthor' + +# TODO remove legacy test clusters after removing legacy mode ORG_ADMIN_TEST_CLUSTER_NAME = 'testclusteradmin' K8_AUTHOR_TEST_CLUSTER_NAME = 'testclusterk8' # required user info SYS_ADMIN_NAME = 'sys_admin' +CLUSTER_ADMIN_NAME = 'cluster_admin' +CLUSTER_ADMIN_PASSWORD = 'password' +CLUSTER_ADMIN_ROLE_NAME = 'cluster_admin_role' +CLUSTER_AUTHOR_NAME = 'cluster_author' +CLUSTER_AUTHOR_PASSWORD = 'password' +CLUSTER_AUTHOR_ROLE_NAME = 'cluster_author_role' +# TODO remove legacy users after removing legacy mode ORG_ADMIN_NAME = 'org_admin' ORG_ADMIN_PASSWORD = 'password' # nosec: test environment ORG_ADMIN_ROLE_NAME = 'Organization Administrator' @@ -61,6 +83,8 @@ K8_AUTHOR_PASSWORD = 'password' # nosec: test environment K8_AUTHOR_ROLE_NAME = 'k8 Author' +VCD_API_VERSION_TO_USE = None + # config file 'test' section flags TEARDOWN_INSTALLATION = None TEARDOWN_CLUSTERS = None @@ -73,6 +97,9 @@ # Persona login cmd SYS_ADMIN_LOGIN_CMD = None +CLUSTER_ADMIN_LOGIN_CMD = None +CLUSTER_AUTHOR_LOGIN_CMD = None +# TODO remove legacy login command after removing legacy mode ORG_ADMIN_LOGIN_CMD = None K8_AUTHOR_LOGIN_CMD = None USER_LOGOUT_CMD = "logout" @@ -91,6 +118,107 @@ VIEW_PUBLISHED_CATALOG_RIGHT = 'Catalog: View Published Catalogs' +def init_rde_environment(config_filepath=BASE_CONFIG_FILEPATH, logger=NULL_LOGGER): # noqa: E501 + """Set up module variables according to config dict. + + :param str config_filepath: + """ + global CLIENT, ORG_HREF, VDC_HREF, \ + CATALOG_NAME, TEARDOWN_INSTALLATION, TEARDOWN_CLUSTERS, \ + TEMPLATE_DEFINITIONS, TEST_ALL_TEMPLATES, SYS_ADMIN_LOGIN_CMD, \ + CLUSTER_ADMIN_LOGIN_CMD, CLUSTER_AUTHOR_LOGIN_CMD, \ + USERNAME_TO_LOGIN_CMD, USERNAME_TO_CLUSTER_NAME, TEST_ORG_HREF, \ + TEST_VDC_HREF, VCD_API_VERSION_TO_USE + + logger.debug("Setting RDE environement") + config = testutils.yaml_to_dict(config_filepath) + logger.debug(f"Config file used: {config}") + + # download all remote template scripts + rtm = RemoteTemplateManager( + config['broker']['remote_template_cookbook_url'], + legacy_mode=config['service']['legacy_mode']) + template_cookbook = rtm.get_filtered_remote_template_cookbook() + TEMPLATE_DEFINITIONS = template_cookbook['templates'] + rtm.download_all_template_scripts(force_overwrite=True) + + # setup test variables + init_test_vars(config['test'], logger=logger) + + sysadmin_client = Client( + config['vcd']['host'], + verify_ssl_certs=config['vcd']['verify']) + sysadmin_client.set_credentials(BasicLoginCredentials( + config['vcd']['username'], + shared_constants.SYSTEM_ORG_NAME, + config['vcd']['password'])) + + vcd_supported_api_versions = \ + set(sysadmin_client.get_supported_versions_list()) + cse_supported_api_versions = set(shared_constants.SUPPORTED_VCD_API_VERSIONS) # noqa: E501 + common_supported_api_versions = \ + list(cse_supported_api_versions.intersection(vcd_supported_api_versions)) # noqa: E501 + common_supported_api_versions.sort() + max_api_version = get_max_api_version(common_supported_api_versions) + CLIENT = Client(config['vcd']['host'], + api_version=max_api_version, + verify_ssl_certs=config['vcd']['verify']) + credentials = BasicLoginCredentials(config['vcd']['username'], + shared_constants.SYSTEM_ORG_NAME, + config['vcd']['password']) + CLIENT.set_credentials(credentials) + VCD_API_VERSION_TO_USE = max_api_version + logger.debug(f"Using VCD api version: {VCD_API_VERSION_TO_USE}") + + CATALOG_NAME = config['broker']['catalog'] + + SYS_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} system " \ + f"{config['vcd']['username']} " \ + f"-iwp {config['vcd']['password']} " \ + f"-V {VCD_API_VERSION_TO_USE}" + CLUSTER_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} " \ + f"{TEST_ORG}" \ + f" {CLUSTER_ADMIN_NAME} " \ + f"-iwp {CLUSTER_ADMIN_PASSWORD} " \ + f"-V {VCD_API_VERSION_TO_USE}" + + USERNAME_TO_LOGIN_CMD = { + 'sys_admin': SYS_ADMIN_LOGIN_CMD, + 'cluster_admin': CLUSTER_ADMIN_LOGIN_CMD, + 'cluster_author': CLUSTER_AUTHOR_LOGIN_CMD + } + + USERNAME_TO_CLUSTER_NAME = { + 'sys_admin': SYS_ADMIN_TEST_CLUSTER_NAME, + 'cluster_admin': CLUSTER_ADMIN_TEST_CLUSTER_NAME, + 'cluster_author': CLUSTER_AUTHOR_TEST_CLUSTER_NAME + } + + # hrefs for Org and VDC that hosts the catalog + org = pyvcloud_utils.get_org(CLIENT, org_name=config['broker']['org']) + vdc = pyvcloud_utils.get_vdc(CLIENT, vdc_name=config['broker']['vdc'], + org=org) + ORG_HREF = org.href + VDC_HREF = vdc.href + + logger.debug(f"Using template org {org.get_name()} with href {ORG_HREF}") + logger.debug(f"Using template vdc {vdc.name} with href {VDC_HREF}") + + # hrefs for Org and VDC that tests cluster operations + test_org = pyvcloud_utils.get_org(CLIENT, org_name=TEST_ORG) + test_vdc = pyvcloud_utils.get_vdc(CLIENT, vdc_name=TEST_VDC, org=test_org) + TEST_ORG_HREF = test_org.href + TEST_VDC_HREF = test_vdc.href + + logger.debug(f"Using test org {test_org.get_name()} " + f"with href {TEST_ORG_HREF}") + logger.debug(f"Using test vdc {test_vdc.name} with href {TEST_VDC_HREF}") + + create_cluster_admin_role(config['vcd'], logger=logger) + create_cluster_author_role(config['vcd'], logger=logger) + + +# TODO remove after removing legacy mode def init_environment(config_filepath=BASE_CONFIG_FILEPATH): """Set up module variables according to config dict. @@ -100,7 +228,8 @@ def init_environment(config_filepath=BASE_CONFIG_FILEPATH): CATALOG_NAME, TEARDOWN_INSTALLATION, TEARDOWN_CLUSTERS, \ TEMPLATE_DEFINITIONS, TEST_ALL_TEMPLATES, SYS_ADMIN_LOGIN_CMD, \ ORG_ADMIN_LOGIN_CMD, K8_AUTHOR_LOGIN_CMD, USERNAME_TO_LOGIN_CMD, \ - USERNAME_TO_CLUSTER_NAME, TEST_ORG_HREF, TEST_VDC_HREF + USERNAME_TO_CLUSTER_NAME, TEST_ORG_HREF, TEST_VDC_HREF, \ + VCD_API_VERSION_TO_USE, TEMPLATE_COOKBOOK_VERSION config = testutils.yaml_to_dict(config_filepath) @@ -108,6 +237,7 @@ def init_environment(config_filepath=BASE_CONFIG_FILEPATH): RemoteTemplateManager(config['broker']['remote_template_cookbook_url'], legacy_mode=config['service']['legacy_mode']) template_cookbook = rtm.get_filtered_remote_template_cookbook() + TEMPLATE_COOKBOOK_VERSION = rtm.cookbook_version TEMPLATE_DEFINITIONS = template_cookbook['templates'] rtm.download_all_template_scripts(force_overwrite=True) @@ -117,10 +247,11 @@ def init_environment(config_filepath=BASE_CONFIG_FILEPATH): api_version=config['vcd']['api_version'], verify_ssl_certs=config['vcd']['verify']) credentials = BasicLoginCredentials(config['vcd']['username'], - SYSTEM_ORG_NAME, + shared_constants.SYSTEM_ORG_NAME, config['vcd']['password']) CLIENT.set_credentials(credentials) + VCD_API_VERSION_TO_USE = config['vcd']['api_version'] CATALOG_NAME = config['broker']['catalog'] AMQP_USERNAME = config['amqp']['username'] AMQP_PASSWORD = config['amqp']['password'] @@ -128,15 +259,15 @@ def init_environment(config_filepath=BASE_CONFIG_FILEPATH): SYS_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} system " \ f"{config['vcd']['username']} " \ f"-iwp {config['vcd']['password']} " \ - f"-V {config['vcd']['api_version']}" + f"-V {VCD_API_VERSION_TO_USE}" ORG_ADMIN_LOGIN_CMD = f"login {config['vcd']['host']} " \ f"{TEST_ORG}" \ f" {ORG_ADMIN_NAME} -iwp {ORG_ADMIN_PASSWORD} " \ - f"-V {config['vcd']['api_version']}" + f"-V {VCD_API_VERSION_TO_USE}" K8_AUTHOR_LOGIN_CMD = f"login {config['vcd']['host']} " \ f"{TEST_ORG} " \ f"{K8_AUTHOR_NAME} -iwp {K8_AUTHOR_PASSWORD}" \ - f" -V {config['vcd']['api_version']}" + f" -V {VCD_API_VERSION_TO_USE}" USERNAME_TO_LOGIN_CMD = { 'sys_admin': SYS_ADMIN_LOGIN_CMD, @@ -164,7 +295,7 @@ def init_environment(config_filepath=BASE_CONFIG_FILEPATH): create_k8_author_role(config['vcd']) -def init_test_vars(test_config): +def init_test_vars(test_config, logger=NULL_LOGGER): """Initialize all the environment variables that are used for test. :param dict test_config: test section of config.yaml @@ -194,9 +325,11 @@ def init_test_vars(test_config): specified_templates_def.append(template_def) break TEMPLATE_DEFINITIONS = specified_templates_def + logger.debug(f'Loaded template definitions: {TEMPLATE_DEFINITIONS}') -def cleanup_environment(): +def cleanup_environment(logger=NULL_LOGGER): + logger.debug("Logging out.") if CLIENT is not None: CLIENT.logout() @@ -227,10 +360,11 @@ def teardown_active_config(): os.remove(ACTIVE_CONFIG_FILEPATH) +# TODO remove after removing legacy mode def create_k8_author_role(vcd_config: dict): - cmd = f"login {vcd_config['host']} {SYSTEM_ORG_NAME} " \ + cmd = f"login {vcd_config['host']} {shared_constants.SYSTEM_ORG_NAME} " \ f"{vcd_config['username']} -iwp {vcd_config['password']} " \ - f"-V {vcd_config['api_version']}" + f"-V {VCD_API_VERSION_TO_USE}" result = CLI_RUNNER.invoke(vcd, cmd.split(), catch_exceptions=False) assert result.exit_code == 0 cmd = f"org use {TEST_ORG}" @@ -252,11 +386,87 @@ def create_k8_author_role(vcd_config: dict): result.output) -def create_user(username, password, role): +def create_cluster_admin_role(vcd_config: dict, logger=NULL_LOGGER): + """Create cluster_admin role using pre-existing 'vapp author' role. + + :param dict vcd_config: server config file + """ + logger.debug("Creating cluster admin role.") + cmd = f"login {vcd_config['host']} {shared_constants.SYSTEM_ORG_NAME} " \ + f"{vcd_config['username']} -iwp {vcd_config['password']} " \ + f"-V {VCD_API_VERSION_TO_USE}" + result = CLI_RUNNER.invoke(vcd, cmd.split(), catch_exceptions=False) + assert result.exit_code == 0 + cmd = f"org use {TEST_ORG}" + result = CLI_RUNNER.invoke(vcd, cmd.split(), catch_exceptions=False) + assert result.exit_code == 0 + + logger.debug(f"Cloning role {ORG_ADMIN_ROLE_NAME} " + f"to create {CLUSTER_ADMIN_ROLE_NAME}") + result = CLI_RUNNER.invoke( + vcd, ['role', 'clone', ORG_ADMIN_ROLE_NAME, CLUSTER_ADMIN_ROLE_NAME], + catch_exceptions=False) + role_exists = DUPLICATE_NAME in result.stdout + if role_exists: + logger.debug(f"Role {CLUSTER_ADMIN_ROLE_NAME} already exists.") + assert role_exists or result.exit_code == 0, \ + testutils.format_command_info('vcd', cmd, result.exit_code, + result.output) + # Add View right for other published catalogs + logger.debug(f"Publishing {VIEW_PUBLISHED_CATALOG_RIGHT} to " + f"the role {CLUSTER_ADMIN_ROLE_NAME}") + result = CLI_RUNNER.invoke( + vcd, ['role', 'add-right', CLUSTER_ADMIN_ROLE_NAME, + VIEW_PUBLISHED_CATALOG_RIGHT], + catch_exceptions=False) + assert result.exit_code == 0, \ + testutils.format_command_info('vcd', cmd, result.exit_code, + result.output) + logger.debug(f"Successfully created role: {CLUSTER_ADMIN_ROLE_NAME}") + + +def create_cluster_author_role(vcd_config: dict, logger=NULL_LOGGER): + """Create cluster_author role using pre-existing 'org admin' role. + + :param dict vcd_config: server config file + """ + logger.debug("Creating cluster author role.") + cmd = f"login {vcd_config['host']} {shared_constants.SYSTEM_ORG_NAME} " \ + f"{vcd_config['username']} -iwp {vcd_config['password']} " \ + f"-V {VCD_API_VERSION_TO_USE}" + result = CLI_RUNNER.invoke(vcd, cmd.split(), catch_exceptions=False) + assert result.exit_code == 0 + cmd = f"org use {TEST_ORG}" + result = CLI_RUNNER.invoke(vcd, cmd.split(), catch_exceptions=False) + assert result.exit_code == 0 + + logger.debug(f"Cloning role {VAPP_AUTHOR_ROLE_NAME} " + f"to create {CLUSTER_AUTHOR_ROLE_NAME}") + result = CLI_RUNNER.invoke( + vcd, ['role', 'clone', VAPP_AUTHOR_ROLE_NAME, CLUSTER_AUTHOR_ROLE_NAME], # noqa: E501 + catch_exceptions=False) + assert DUPLICATE_NAME in result.stdout or result.exit_code == 0, \ + testutils.format_command_info('vcd', cmd, result.exit_code, + result.output) + # Add View right for other published catalogs + logger.debug(f"Publishing {VIEW_PUBLISHED_CATALOG_RIGHT} to the role " + f"{CLUSTER_AUTHOR_ROLE_NAME}") + result = CLI_RUNNER.invoke( + vcd, ['role', 'add-right', CLUSTER_AUTHOR_ROLE_NAME, + VIEW_PUBLISHED_CATALOG_RIGHT], + catch_exceptions=False) + assert result.exit_code == 0, \ + testutils.format_command_info('vcd', cmd, result.exit_code, + result.output) + logger.debug(f"Successfully created role: {CLUSTER_AUTHOR_ROLE_NAME}") + + +def create_user(username, password, role, logger=NULL_LOGGER): config = testutils.yaml_to_dict(BASE_CONFIG_FILEPATH) - cmd = f"login {config['vcd']['host']} {SYSTEM_ORG_NAME} " \ + cmd = f"login {config['vcd']['host']} " \ + f"{shared_constants.SYSTEM_ORG_NAME} " \ f"{config['vcd']['username']} -iwp {config['vcd']['password']} " \ - f"-V {config['vcd']['api_version']}" + f"-V {VCD_API_VERSION_TO_USE}" result = CLI_RUNNER.invoke(vcd, cmd.split(), catch_exceptions=False) assert result.exit_code == 0 cmd = f"org use {TEST_ORG}" @@ -271,38 +481,38 @@ def create_user(username, password, role): catch_exceptions=False) # no assert here because if the user exists, the exit code will be 2 - # user can already exist but be disabled - # cmd = f"user update {username} --enable" - # result = CLI_RUNNER.invoke(vcd, cmd.split(), catch_exceptions=False) - # assert result.exit_code == 0,\ - # testutils.format_command_info('vcd', cmd, result.exit_code, - # result.output) + logger.debug(f"Successfully created user {username}") -def delete_catalog_item(item_name): +def delete_catalog_item(item_name, logger=NULL_LOGGER): + logger.debug(f"Deleting catalog item: {item_name}") org = Org(CLIENT, href=ORG_HREF) try: org.delete_catalog_item(CATALOG_NAME, item_name) pyvcloud_utils.wait_for_catalog_item_to_resolve(CLIENT, CATALOG_NAME, item_name, org=org) org.reload() - except EntityNotFoundException: - pass + logger.debug(f"Successfully deleted the catalog item: {item_name}") + except EntityNotFoundException as e: + logger.debug(f"Failed to delete catalog item {item_name}: {e}") -def delete_vapp(vapp_name, vdc_href): +def delete_vapp(vapp_name, vdc_href, logger=NULL_LOGGER): + logger.debug(f"Deleting vapp {vapp_name} in vdc {vdc_href}.") vdc = VDC(CLIENT, href=vdc_href) try: task = vdc.delete_vapp(vapp_name, force=True) CLIENT.get_task_monitor().wait_for_success(task) vdc.reload() - except EntityNotFoundException: - pass + logger.debug(f"Successfully deleted the vapp {vapp_name}.") + except EntityNotFoundException as e: + logger.debug(f"Failed to vapp {vapp_name}: {e}") -def delete_catalog(catalog_name=None): +def delete_catalog(catalog_name=None, logger=NULL_LOGGER): if catalog_name is None: catalog_name = CATALOG_NAME + logger.debug(f"Deleting catalog {catalog_name}") org = Org(CLIENT, href=ORG_HREF) try: org.delete_catalog(catalog_name) @@ -311,19 +521,133 @@ def delete_catalog(catalog_name=None): # below causes EntityNotFoundException, catalog not found. # time.sleep(15) # org.reload() + logger.debug(f"Successfully deleted the catalog {catalog_name}") except EntityNotFoundException: - pass + logger.debug(f"Failed to delete catalog {catalog_name}") +# TODO remove after removing legacy mode def unregister_cse(): try: - APIExtension(CLIENT).delete_extension(CSE_SERVICE_NAME, - CSE_SERVICE_NAMESPACE) + APIExtension(CLIENT).delete_extension( + server_constants.CSE_SERVICE_NAME, + server_constants.CSE_SERVICE_NAMESPACE) except MissingRecordException: pass -def catalog_item_exists(catalog_item, catalog_name=None): +def unregister_cse_in_mqtt(logger=NULL_LOGGER): + logger.debug("Unregistering CSE as MQTT extension") + try: + mqtt_ext_manager = MQTTExtensionManager(CLIENT) + mqtt_ext_info = mqtt_ext_manager.get_extension_info( + ext_name=server_constants.CSE_SERVICE_NAME, + ext_version=server_constants.MQTT_EXTENSION_VERSION, + ext_vendor=server_constants.MQTT_EXTENSION_VENDOR) + ext_urn_id = mqtt_ext_info[server_constants.MQTTExtKey.EXT_URN_ID] + mqtt_ext_manager.delete_extension( + ext_name=server_constants.CSE_SERVICE_NAME, + ext_version=server_constants.MQTT_EXTENSION_VERSION, + ext_vendor=server_constants.MQTT_EXTENSION_VENDOR, + ext_urn_id=ext_urn_id) + logger.debug("Successfully unregistered CSE as MQTT extension") + except Exception as e: + logger.debug(f"Failed to unregister CSE from MQTT: {e}") + + +def publish_right_bundle_to_deployment_org(logger=NULL_LOGGER): + try: + rbm = RightBundleManager(CLIENT, logger_debug=logger, log_wire=True) + cse_right_bundle = rbm.get_right_bundle_by_name( + rde_constants.DEF_NATIVE_ENTITY_TYPE_RIGHT_BUNDLE) + test_org_id = TEST_ORG_HREF.split('/')[-1] + rbm.publish_cse_right_bundle_to_tenants( + right_bundle_id=cse_right_bundle['id'], + org_ids=[test_org_id]) + logger.debug( + f"Successfully published native right bundle to orgs {TEST_ORG}") + except Exception as e: + logger.debug(f"Failed to publish native right bundle " + f"to org {TEST_ORG}: {e}") + + +def assign_native_rights(role_name, right_list=None, logger=NULL_LOGGER): + logger.debug(f"Assigning rights {right_list} to the role {role_name}") + if not right_list: + logger.debug(f"Skipping assigning native rights to role {role_name}") + return + try: + test_org = Org(CLIENT, href=TEST_ORG_HREF) + role_resource = test_org.get_role_resource(role_name) + role = Role(CLIENT, resource=role_resource) + initial_right_set = set([r['name'] for r in role.list_rights()]) + right_set = set(right_list) + initial_right_set.update(right_set) + role.add_rights(list(initial_right_set), test_org) + except Exception as e: + logger.debug(f"Failed to assign native rights " + f"{right_list} to role {role_name}: {e} ") + + +def cleanup_rde_artifacts(logger=NULL_LOGGER): + """Cleanup all defined entity related artifacts. + + Deletes the following - + - CSE interface + - Native entity type + """ + try: + rde_version_in_use = rde_utils.get_runtime_rde_version_by_vcd_api_version(CLIENT.get_api_version()) # noqa: E501 + rde_metadata = rde_utils.get_rde_metadata(rde_version_in_use) + cloudapi_client = pyvcloud_utils.get_cloudapi_client_from_vcd_client( + client=CLIENT, + logger_wire=SERVER_CLOUDAPI_WIRE_LOGGER) + schema_svc = def_schema_svc.DefSchemaService(cloudapi_client=cloudapi_client) # noqa: E501 + if rde_constants.RDEMetadataKey.ENTITY_TYPE in rde_metadata: + # delete entitytype + entity_type: common_models.DefEntityType = \ + rde_metadata[rde_constants.RDEMetadataKey.ENTITY_TYPE] + schema_svc.delete_entity_type(entity_type.get_id()) + logger.debug(f"Deleted entity type: {entity_type.name}") + if rde_constants.RDEMetadataKey.INTERFACES in rde_metadata: + # delete interface + interfaces: List[common_models.DefInterface] = \ + rde_metadata[rde_constants.RDEMetadataKey.INTERFACES] + for i in interfaces: + interface_id = i.get_id() + if interface_id != common_models.K8Interface.VCD_INTERFACE.value.get_id(): # noqa: E501 + schema_svc.delete_interface(interface_id) + logger.debug(f"Deleted interface: {i.name}") + except Exception as e: + logger.error(f"Failed to clean up RDE artifacts: {e}") + + +def cleanup_roles_and_users(logger=NULL_LOGGER): + """Cleanup all the new roles and users created. + + Deletes the following + - cluster_author User + - cluster_author_role Role + - cluster_admin User + - cluster_admin_role ROle + """ + user_and_role_list = [ + (CLUSTER_AUTHOR_NAME, CLUSTER_AUTHOR_ROLE_NAME), + (CLUSTER_ADMIN_NAME, CLUSTER_ADMIN_ROLE_NAME) + ] + org = Org(CLIENT, href=TEST_ORG_HREF) + for user_and_role in user_and_role_list: + try: + logger.debug(f"cleaning up user {user_and_role[0]} and " + f"role {user_and_role[1]}") + org.delete_user(user_and_role[0]) + org.delete_role(user_and_role[1]) + except Exception as e: + logger.debug("Exception occured when cleaning up " + f"roles and users: {e}") + + +def catalog_item_exists(catalog_item, catalog_name=None, logger=NULL_LOGGER): if catalog_name is None: catalog_name = CATALOG_NAME org = Org(CLIENT, href=ORG_HREF) @@ -334,23 +658,27 @@ def catalog_item_exists(catalog_item, catalog_name=None): # question. Please use this method only for org admin and sys admins. org.get_catalog_item(catalog_name, catalog_item) return True - except EntityNotFoundException: + except EntityNotFoundException as e: + logger.error(f"Catalog item not found: {e}") return False -def vapp_exists(vapp_name, vdc_href): +def vapp_exists(vapp_name, vdc_href, logger=NULL_LOGGER): vdc = VDC(CLIENT, href=vdc_href) try: vdc.get_vapp(vapp_name) + logger.debug(f"Vapp {vapp_name} found in vdc {vdc.name}") return True except EntityNotFoundException: + logger.debug(f"Vapp {vapp_name} not found in vdc {vdc.name}") return False def is_cse_registered(): try: - APIExtension(CLIENT).get_extension(CSE_SERVICE_NAME, - namespace=CSE_SERVICE_NAMESPACE) + APIExtension(CLIENT).get_extension( + server_constants.CSE_SERVICE_NAME, + namespace=server_constants.CSE_SERVICE_NAMESPACE) return True except MissingRecordException: return False @@ -358,8 +686,9 @@ def is_cse_registered(): def is_cse_registration_valid(routing_key, exchange): try: - ext = APIExtension(CLIENT).get_extension(CSE_SERVICE_NAME, - namespace=CSE_SERVICE_NAMESPACE) # noqa: E501 + ext = APIExtension(CLIENT).get_extension( + server_constants.CSE_SERVICE_NAME, + namespace=server_constants.CSE_SERVICE_NAMESPACE) except MissingRecordException: return False @@ -377,3 +706,15 @@ def check_cse_registration(routing_key, exchange): assert is_cse_registration_valid(routing_key, exchange), \ 'CSE is registered as an extension, but the extension settings ' \ 'on vCD are not the same as config settings.' + + +def check_cse_registration_as_mqtt_extension(logger=NULL_LOGGER): + mqtt_ext_manager = MQTTExtensionManager(CLIENT, debug_logger=logger) + is_cse_registered = mqtt_ext_manager.check_extension_exists( + server_constants.MQTT_EXTENSION_URN) + assert is_cse_registered, \ + 'CSE is not registered as an extension when it should be.' + if is_cse_registered: + assert mqtt_ext_manager.is_extension_enabled( + server_constants.MQTT_EXTENSION_URN), "CSE is registered as an " \ + "extension but the extension is not enabled" diff --git a/system_tests_v2/README.md b/system_tests_v2/README.md new file mode 100644 index 000000000..078c1c28d --- /dev/null +++ b/system_tests_v2/README.md @@ -0,0 +1,141 @@ +# CSE Testing + +## Usage + +```bash +$ pip install -r test-requirements.txt +$ cd container-service-extension/system_tests + +# modify base_config.yaml (more info in next section) +# set up vCD instance (org, ovdc, ovdc network) + +# Run all tests (either works) +$ pytest +$ python -m pytest + +# Run a test module +$ pytest test_cse_server.py + +# Run a test function +$ pytest test_cse_server.py::mytestfunction + +# Useful options +$ pytest --exitfirst # -x stop after first failure +$ pytest --maxfail=2 # stop after 2 failures +$ pytest --verbose # -v increases testing verbosity +$ pytest --capture=no # -s print all output during testing (tells pytest not to capture output) +$ pytest --disable-warnings + +# Common use case (outputs to 'testlog') +$ pytest --disable-warnings -x -v -s test_cse_server.py > testlog +``` + +## base_config.yaml + +Testers should fill out this config file with vCD instance details + +Options for **'test'** section: + +| key | description | value type | possible values | default value | +|-----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|-----------------|---------------| +| teardown_installation | - Affects **test_cse_server.py**
- If True, delete all installation entities (even on test failure)
- If False, do not delete installation entities (even on test failure)
- If omitted, defaults to True | bool | True, False | True | +| teardown_clusters | - Affects **test_cse_client.py**
- If True, delete test cluster on test failure
- If False, do not delete test cluster on test failure
- If omitted, defaults to True
- Successful client tests will not leave clusters up
| bool | True, False | True | +| test_all_templates | - Affects **test_cse_client.py**
- If True, tests cluster operations on all templates found
- If False, tests cluster operations only for 1st template found
- If omitted, defaults to False | bool | True, False | False | + +## Notes + +- Client tests (**test_cse_client.py**) require an cluster admin and cluster author user with the same username and password specified in the config file **vcd** section +- Server tests (**test_cse_server.py**) require you to have a public/private SSH key (RSA) + - These keys should be at: `~/.ssh/id_rsa` and `~/.ssh/id_rsa.pub` + - Keys must not be password protected (to remove key password, use `ssh-keygen -p`) + - ssh-key help: +- More detailed information can be found in the module docstrings + +--- + +## Writing Tests with Pytest and Click + +Before writing a test, first check **conftest.py** for any 'autouse=True' fixtures. Then check +the module itself for any 'autouse=True' fixtures. When writing a test, any fixtures +defined in **conftest.py** can be used. When creating new fixtures, place it in the module +if its functionality is specific to that test module. If the functionality can be used +across multiple test modules, then place it in **conftest.py** + +### Fixtures (setUp, tearDown) + +```python +@pytest.fixture() +def my_fixture(): + print('my_fixture setup') + yield + print('my_fixture teardown') + +@pytest.fixture() +def another_fixture(): + important_variable = {'key': 'value'} + yield important_variable + print('another_fixture teardown') + +def my_test(my_fixture): + assert my_fixture is None + +def my_test_2(another_fixture): + assert isinstance(another_fixture, dict) + print(another_fixture['key']) +``` + +Common keyword arguments for @pytest.fixture() + +| keyword | description | value type | possible values | default value | +|---------|------------------------------------------------------------|------------|------------------------------------------|---------------| +| scope | defines when and how often the fixture should run | str | 'session', 'module', 'class', 'function' | 'function' | +| autouse | if True, fixture runs automatically with respect to @scope | bool | True, False | False | + +- Fixture teardown (after yield) executes even if test raises an exception (including AssertionError) + +- Shared fixtures should be defined in **conftest.py** according to pytest conventions. Fixtures defined here are autodiscovered by pytest and don't need to be imported. + +- Fixtures specific to a test module should be defined in the module directly. + +### Click's CLIRunner + +## Example + +```python +import container_service_extension.system_test_framework.environment as env +from container_service_extension.server_cli import cli +from vcd_cli.vcd import vcd + +# test command: `cse sample --output myconfig.yaml` +cmd = 'sample --output myconfig.yaml' +result = env.CLI_RUNNER.invoke(cli, cmd.split(), catch_exceptions=False) +# catch_exceptions=False tell Click not to swallow exceptions, so we can inspect if something went wrong +assert result.exit_code == 0, f"Command [{cmd}] failed." + +# test command: `vcd cse template list` +cmd = 'cse template list' +result = env.CLI_RUNNER.invoke(vcd, cmd.split(), catch_exceptions=False) +assert result.exit_code == 0, f"Command[{cmd}] failed." +``` + +These small tests using Click's `CLIRunner` and `invoke` function only validate command structure. +These assert statements will pass because the commands themselves are valid, even if an error is thrown during the command execution. + +## Pytest logging mechanism + +pytest-logger is a plugin used to log during test execution. The plugin makes use of hooks to configure logs. +The `pytest_logger.py` file contains the logger which can be imported in different test files. + +The hooks `pytest_logger_config(logger_config)` and `pytest_logger_logdirlink(config)` are configured in conftest.py to create +a symlink `pytest_log` to log directory created by pytest. A different log file will be created for each test executed. + +To log information, please import and use the logger PYTEST_LOGGER defined in `pytest_logger.py` module. + +--- + +### Helpful links + +- Usage and Invocations: +- Fixtures: +- Click Testing: +- Logging: diff --git a/system_tests_v2/__init__.py b/system_tests_v2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/system_tests_v2/base_config.yaml b/system_tests_v2/base_config.yaml new file mode 100644 index 000000000..9a7f52b71 --- /dev/null +++ b/system_tests_v2/base_config.yaml @@ -0,0 +1,64 @@ +# Valid config file +# Fill in fields marked with '???' + +test: + teardown_installation: true # Affects test_cse_server.py. + # if true, delete all installation entities (even on test failure). + # if false, do not delete installation entities (even on test success). + # if this key is omitted, defaults to true. + teardown_clusters: true # Affects test_cse_client.py. + # if true, delete test cluster (env.TEST_CLUSTER_NAME) on test failure. + # if false, do not delete test cluster on test failure. + # if this key is omitted, defaults to true. + # Successful client tests will not leave clusters up. + test_all_templates: true # Affects test_cse_client.py. + # if true, tests cluster deployment on all templates found. + # if false, tests cluster deployment only for first template found. + # if this key is omitted, defaults to false. + test_templates: "???" # Tests will only create these templates if test_all_templates is set to false + # format -> "template_1_name:template_1_revision,template_2_name:template_2_revision" + upgrade_template_repo_url: '???' # + network: '???' # org network within @vdc that will be used during testing + # Should have outbound access to the public internet + org: '???' # vCD org where all the test will be run + storage_profile: '???' # name of the storage profile to use while creating clusters on this org vdc + vdc: '???' # Org VDC powering the org + +mqtt: + verify_ssl: false + +vcd: + host: '???' + log: true + password: '???' + port: 443 + username: '???' + verify: false + +vcs: +- name: '???' + password: '???' + username: '???' + verify: false + +service: + enforce_authorization: false + processors: 5 + log_wire: false + telemetry: + enable: false + legacy_mode: false + +broker: + catalog: cse # public shared catalog within org where the template will be published + default_template_name: ubuntu-16.04_k8-1.21_weave-2.8.1 + # name of the default template to use if none is specified + default_template_revision: 1 # revision of the default template to use if none is specified + ip_allocation_mode: pool # dhcp or pool + network: '???' # org network within @vdc that will be used during the install process to build the template + # Should have outbound access to the public internet + # CSE appliance doesn't need to be connected to this network + org: '???' # vCD org that contains the shared catalog where the master templates will be stored + remote_template_cookbook_url: http://raw.githubusercontent.com/vmware/container-service-extension-templates/dev/template_v2.yaml + storage_profile: '*' # name of the storage profile to use when creating the temporary vApp used to build the template + vdc: '???' # VDC within @org that will be used during the install process to build the template diff --git a/system_tests_v2/conftest.py b/system_tests_v2/conftest.py new file mode 100644 index 000000000..2f5d54052 --- /dev/null +++ b/system_tests_v2/conftest.py @@ -0,0 +1,124 @@ +# container-service-extension +# Copyright (c) 2019 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause + +""" +conftest.py is used by pytest to automatically find shared fixtures. + +Fixtures defined here can be used without importing. +""" +import os + +import pytest +import system_tests_v2.pytest_logger as pytest_logger + +import container_service_extension.system_test_framework.environment as env +import container_service_extension.system_test_framework.utils as testutils + + +def pytest_logger_config(logger_config): + # adds two loggers, which will: + # - log to pytest_logger at 'warn' level + logger_config.add_loggers([pytest_logger.PYTEST_LOGGER_NAME], + stdout_level='warn') + # default --loggers option is set to log pytest_logger at WARN level + logger_config.set_log_option_default(pytest_logger.PYTEST_LOGGER_NAME) + + +def pytest_logger_logdirlink(config): + return os.path.join(os.path.dirname(__file__), pytest_logger.PYTEST_LOG_FILE_NAME) # noqa: E501 + + +@pytest.fixture(scope='session', autouse=True) +def environment(): + """Fixture to setup and teardown the session environment. + + This fixture executes automatically for test session setup and teardown. + Does not have any side effects to vCD. + + Setup tasks: + - initialize variables (org/vdc href, client, mqtt settings) + - create cluster admin and cluster author roles + + Teardown tasks: + - logout client + """ + env.init_rde_environment(logger=pytest_logger.PYTEST_LOGGER) + yield + env.cleanup_environment(logger=pytest_logger.PYTEST_LOGGER) + + +@pytest.fixture(scope='session', autouse=True) +def vcd_users(): + """Fixture to setup required users if they do not exist already. + + This fixture executes automatically for test session setup and teardown. + User credentials are in 'system_test_framework/environment.py' + + Setup tasks: + - create Cluster admin user if it doesn't exist + - create Cluster author user if it doesn't exist + """ + # org_admin -> cluster_admin + # k8_author -> cluster_author + env.create_user(env.CLUSTER_ADMIN_NAME, + env.CLUSTER_ADMIN_PASSWORD, + env.CLUSTER_ADMIN_ROLE_NAME, + logger=pytest_logger.PYTEST_LOGGER) + env.create_user(env.CLUSTER_AUTHOR_NAME, + env.CLUSTER_AUTHOR_PASSWORD, + env.CLUSTER_AUTHOR_ROLE_NAME, + logger=pytest_logger.PYTEST_LOGGER) + + +@pytest.fixture +def config(): + """Fixture to setup and teardown an active config file. + + Usage: add the parameter 'config' to the test function. This 'config' + parameter is the dict representation of the config file, and can be + used in the test function. + + Tasks: + - create config dict from env.BASE_CONFIG_FILEPATH + - create active config file at env.ACTIVE_CONFIG_FILEPATH + - adjust active config file security + + yields config dict + """ + config = env.setup_active_config() + yield config + env.teardown_active_config() + + +@pytest.fixture +def test_config(): + """Fixture to provide 'test' section of test config to individual tests.""" + config = testutils.yaml_to_dict(env.BASE_CONFIG_FILEPATH) + return config['test'] + + +@pytest.fixture +def publish_native_right_bundle(): + """Publish CSE native right bundle to deployment org. + + Usage: add parameter 'publish_native_right_bundle' to the test function. + This fixture will be executed after the test function completes + + Tasks done: + - publish cse native right bundle to deployment org (org specified in + 'test' section of base_config.yaml) + - assign appropriate rights to roles in test org + """ + yield + env.publish_right_bundle_to_deployment_org( + logger=pytest_logger.PYTEST_LOGGER) + env.assign_native_rights(env.CLUSTER_ADMIN_ROLE_NAME, + ["cse:nativeCluster: Full Access", + "cse:nativeCluster: Modify", + "cse:nativeCluster: View"], + logger=pytest_logger.PYTEST_LOGGER) + env.assign_native_rights(env.CLUSTER_AUTHOR_ROLE_NAME, + ["cse:nativeCluster: Modify", + "cse:nativeCluster: View"], + logger=pytest_logger.PYTEST_LOGGER) diff --git a/system_tests_v2/pytest_logger.py b/system_tests_v2/pytest_logger.py new file mode 100644 index 000000000..44ea95675 --- /dev/null +++ b/system_tests_v2/pytest_logger.py @@ -0,0 +1,10 @@ +# container-service-extension +# Copyright (c) 2021 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause + +import logging + + +PYTEST_LOG_FILE_NAME = "pytest_log" +PYTEST_LOGGER_NAME = "log" +PYTEST_LOGGER: logging.Logger = logging.getLogger(PYTEST_LOGGER_NAME) diff --git a/system_tests_v2/test_cse_server.py b/system_tests_v2/test_cse_server.py new file mode 100644 index 000000000..a3308c69f --- /dev/null +++ b/system_tests_v2/test_cse_server.py @@ -0,0 +1,590 @@ +# container-service-extension +# Copyright (c) 2019 VMware, Inc. All Rights Reserved. +# SPDX-License-Identifier: BSD-2-Clause + +import filecmp +import os +import subprocess +import tempfile + +import pytest +from pyvcloud.vcd.exceptions import EntityNotFoundException +from pyvcloud.vcd.vdc import VDC +from system_tests_v2.pytest_logger import PYTEST_LOGGER + +from container_service_extension.common.utils import server_utils +from container_service_extension.installer.config_validator import get_validated_config # noqa: E501 +import container_service_extension.installer.templates.local_template_manager as ltm # noqa: E501 +from container_service_extension.server.cli.server_cli import cli +import container_service_extension.system_test_framework.environment as env +import container_service_extension.system_test_framework.utils as testutils + + +PASSWORD_FOR_CONFIG_ENCRYPTION = "vmware" + +""" +CSE server tests to test validity and functionality of `cse` CLI commands. + +Tests these following commands: +$ cse check cse_test_config.yaml --skip-config-decryption (missing/invalid keys) +$ cse check cse_test_config.yaml --skip-config-decryption (incorrect value types) +$ cse check cse_test_config.yaml -i --skip-config-decryption (cse not installed yet) + +$ cse install --config cse_test_config.yaml --template photon-v2 --skip-config-decryption + --ssh-key ~/.ssh/id_rsa.pub --no-capture + +$ cse install --config cse_test_config.yaml --template photon-v2 --skip-config-decryption +$ cse install --config cse_test_config.yaml --ssh-key ~/.ssh/id_rsa.pub + --update --no-capture --skip-config-decryption +$ cse install --config cse_test_config.yaml --skip-config-decryption + +$ cse check cse_test_config.yaml -i --skip-config-decryption + +$ cse run --config cse_test_config.yaml --skip-config-decryption +$ cse run --config cse_test_config.yaml --skip-check --skip-config-decryption + +NOTE: +- Edit 'base_config.yaml' for your own vCD instance. +- These tests will use your public/private SSH keys (RSA) + - Required keys: '~/.ssh/id_rsa' and '~/.ssh/id_rsa.pub' + - Keys should not be password protected, or tests will fail. + To remove key password, use `ssh-keygen -p`. + - ssh-key help: https://help.github.com/articles/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent/ # noqa +- vCD entities related to CSE (vapps, catalog items) are cleaned up after + tests run, unless 'teardown_installation'=false in 'base_config.yaml'. +- These tests are meant to run in sequence, but can run independently. +- !!! These tests will fail on Windows !!! We generate temporary config + files and restrict its permissions due to the check that + cse install/check performs. This permissions check is incompatible + with Windows, and is a known issue. +- This test module typically takes ~40 minutes to finish. + +TODO() need to check that rights exist when CSE is registered and that rights +don't exist when CSE is not registered. Need a pyvcloud function to check +if a right exists without adding it. Also need functionality to remove CSE +rights when CSE is unregistered. +""" + + +def _remove_cse_artifacts(): + for template in env.TEMPLATE_DEFINITIONS: + env.delete_catalog_item(template['source_ova_name'], + logger=PYTEST_LOGGER) + catalog_item_name = ltm.get_revisioned_template_name( + template['name'], template['revision']) + env.delete_catalog_item(catalog_item_name, + logger=PYTEST_LOGGER) + temp_vapp_name = testutils.get_temp_vapp_name(template['name']) + env.delete_vapp(temp_vapp_name, vdc_href=env.VDC_HREF) + env.delete_catalog(logger=PYTEST_LOGGER) + env.unregister_cse_in_mqtt(logger=PYTEST_LOGGER) + env.cleanup_rde_artifacts(logger=PYTEST_LOGGER) + env.cleanup_roles_and_users(logger=PYTEST_LOGGER) + + +@pytest.fixture(scope='module', autouse=True) +def delete_installation_entities(): + """Fixture to ensure that CSE entities do not exist in vCD. + + This fixture executes automatically for this module's setup and teardown. + + Setup tasks: + - Delete source ova files, vapp templates, temp vapps, catalogs + - Unregister CSE from vCD + + Teardown tasks (only if config key 'teardown_installation'=True): + - Delete source ova files, vapp templates, temp vapps, catalogs + - Unregister CSE from vCD + """ + _remove_cse_artifacts() + yield + if env.TEARDOWN_INSTALLATION: + _remove_cse_artifacts() + + +@pytest.fixture +def unregister_cse_before_test(): + """Fixture to ensure that CSE is not registered to vCD. + + Usage: add the parameter 'unregister_cse' to the test function. + + Note: we don't do teardown unregister_cse(), because the user may want + to preserve the state of vCD after tests run. + """ + env.unregister_cse_in_mqtt() + + +def test_0010_cse_sample(): + """Test that `cse sample` is a valid command. + + Test that `cse sample` command along with every option is an actual + command. Does not test for validity of sample outputs. + """ + cmd = "sample" + result = env.CLI_RUNNER.invoke(cli, cmd.split(), catch_exceptions=False) + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + assert result.exit_code == 0,\ + testutils.format_command_info('cse', cmd, result.exit_code, + result.output) + + output_filepath = 'dummy-output.yaml' + cmd = f'sample --output {output_filepath}' + result = env.CLI_RUNNER.invoke(cli, cmd.split(), catch_exceptions=False) + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + assert result.exit_code == 0,\ + testutils.format_command_info('cse', cmd, result.exit_code, + result.output) + + cmd = f'sample --pks-config --output {output_filepath}' + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + result = env.CLI_RUNNER.invoke(cli, cmd.split(), catch_exceptions=False) + assert result.exit_code == 0,\ + testutils.format_command_info('cse', cmd, result.exit_code, + result.output) + + if os.path.exists(output_filepath): + os.remove(output_filepath) + + +def test_0020_cse_version(): + """Test that `cse version` is a valid command.""" + cmd = "version" + result = env.CLI_RUNNER.invoke(cli, cmd.split(), catch_exceptions=False) + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + assert result.exit_code == 0,\ + testutils.format_command_info('cse', cmd, result.exit_code, + result.output) + + +def test_0030_cse_check(config): + """Test that `cse check` is a valid command. + + Test that `cse check` command along with every option is an actual command. + Does not test for validity of config files or CSE installations. + """ + cmd = f"check {env.ACTIVE_CONFIG_FILEPATH} --skip-config-decryption" + result = env.CLI_RUNNER.invoke(cli, cmd.split(), catch_exceptions=False) + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + assert result.exit_code == 0,\ + testutils.format_command_info('cse', cmd, result.exit_code, + result.output) + + +def test_0040_config_missing_keys(config): + """Test that config files with missing keys don't pass validation.""" + bad_key_config1 = testutils.yaml_to_dict(env.ACTIVE_CONFIG_FILEPATH) + del bad_key_config1['mqtt'] + + bad_key_config2 = testutils.yaml_to_dict(env.ACTIVE_CONFIG_FILEPATH) + del bad_key_config2['vcs'][0]['username'] + + configs = [ + bad_key_config1, + bad_key_config2 + ] + + for config in configs: + testutils.dict_to_yaml_file(config, env.ACTIVE_CONFIG_FILEPATH) + PYTEST_LOGGER.debug(f"Validating config: {config}") + try: + get_validated_config(env.ACTIVE_CONFIG_FILEPATH, + skip_config_decryption=True) + PYTEST_LOGGER.debug("Validation succeeded when it " + "should not have") + assert False, f"{env.ACTIVE_CONFIG_FILEPATH} passed validation " \ + f"when it should not have" + except KeyError as e: + PYTEST_LOGGER.debug("Validation failed as expected due " + f"to invalid keys: {e}") + + +def test_0050_config_invalid_value_types(config): + """Test that configs with invalid value types don't pass validation.""" + bad_values_config1 = testutils.yaml_to_dict(env.ACTIVE_CONFIG_FILEPATH) + bad_values_config1['vcd'] = True + bad_values_config1['vcs'] = 'a' + + bad_values_config2 = testutils.yaml_to_dict(env.ACTIVE_CONFIG_FILEPATH) + bad_values_config2['vcd']['username'] = True + bad_values_config2['vcd']['port'] = 'a' + + bad_values_config3 = testutils.yaml_to_dict(env.ACTIVE_CONFIG_FILEPATH) + bad_values_config3['vcs'][0]['username'] = True + bad_values_config3['vcs'][0]['password'] = 123 + bad_values_config3['vcs'][0]['verify'] = 'a' + + bad_values_config4 = testutils.yaml_to_dict(env.ACTIVE_CONFIG_FILEPATH) + bad_values_config4['broker']['remote_template_cookbook_url'] = 1 + + configs = [ + bad_values_config1, + bad_values_config2, + bad_values_config3, + bad_values_config4 + ] + + for config in configs: + testutils.dict_to_yaml_file(config, env.ACTIVE_CONFIG_FILEPATH) + PYTEST_LOGGER.debug(f"Validating config: {config}") + try: + get_validated_config(env.ACTIVE_CONFIG_FILEPATH, + skip_config_decryption=True) + assert False, f"{env.ACTIVE_CONFIG_FILEPATH} passed validation " \ + f"when it should not have" + except TypeError as e: + PYTEST_LOGGER.debug("Validation failed as expected due " + f"to invalid value: {e}") + pass + + +def test_0060_config_valid(config): + """Test that configs with valid keys and value types pass validation.""" + PYTEST_LOGGER.debug(f"Validating config: {config}") + try: + get_validated_config(env.ACTIVE_CONFIG_FILEPATH, + skip_config_decryption=True) + PYTEST_LOGGER.debug("Validation succeeded as expected.") + except (KeyError, TypeError, ValueError) as e: + PYTEST_LOGGER.debug(f"Failed to validate the config. Error: {e}") + assert False, f"{env.ACTIVE_CONFIG_FILEPATH} did not pass validation" \ + f" when it should have" + + +def test_0070_check_invalid_installation(config): + """Test cse check against config that hasn't been used for installation.""" + try: + cmd = f"check {env.ACTIVE_CONFIG_FILEPATH} --skip-config-decryption --check-install" # noqa: E501 + result = env.CLI_RUNNER.invoke(cli, cmd.split(), catch_exceptions=False) # noqa: E501 + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + assert False, "cse check passed when it should have failed." + except Exception: + pass + + +def test_0080_install_skip_template_creation(config, + unregister_cse_before_test, + publish_native_right_bundle): + """Test install. + + Installation options: '--ssh-key', '--skip-template-creation', + '--skip-config-decryption' + + Tests that installation: + - registers CSE, without installing the templates + + command: cse install --config cse_test_config.yaml + --ssh-key ~/.ssh/id_rsa.pub --skip-config-decryption + --skip-create-templates + required files: ~/.ssh/id_rsa.pub, cse_test_config.yaml, + expected: cse registered, catalog exists, source OVAs do not exist, + temp vapps do not exist, k8s templates do not exist. + """ + cmd = f"install --config {env.ACTIVE_CONFIG_FILEPATH} --ssh-key " \ + f"{env.SSH_KEY_FILEPATH} --skip-template-creation " \ + f"--skip-config-decryption" + result = env.CLI_RUNNER.invoke(cli, cmd.split(), catch_exceptions=False) + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + assert result.exit_code == 0,\ + testutils.format_command_info('cse', cmd, result.exit_code, + result.output) + + # check that cse was registered correctly + env.check_cse_registration_as_mqtt_extension(logger=PYTEST_LOGGER) + remote_template_keys = server_utils.get_template_descriptor_keys( + env.TEMPLATE_COOKBOOK_VERSION) + + for template_config in env.TEMPLATE_DEFINITIONS: + # check that source ova file does not exist in catalog + assert not env.catalog_item_exists( + template_config[remote_template_keys.SOURCE_OVA_NAME], + logger=PYTEST_LOGGER), \ + 'Source ova file exists when it should not.' + + # check that k8s templates does not exist + catalog_item_name = ltm.get_revisioned_template_name( + template_config[remote_template_keys.NAME], + template_config[remote_template_keys.REVISION]) + assert not env.catalog_item_exists( + catalog_item_name, + logger=PYTEST_LOGGER), 'k8s templates exist when they should not.' + + # check that temp vapp does not exists + temp_vapp_name = testutils.get_temp_vapp_name( + template_config[remote_template_keys.NAME]) + assert not env.vapp_exists( + temp_vapp_name, vdc_href=env.VDC_HREF, logger=PYTEST_LOGGER), \ + 'vApp exists when it should not.' + + +@pytest.mark.skipif(not env.TEST_ALL_TEMPLATES, + reason="Configuration specifies 'test_all_templates' as False") # noqa: E501 +def test_0090_install_all_templates(config, unregister_cse_before_test): + """Test install. + + Installation options: '--ssh-key', '--retain-temp-vapp', + '--skip-config-decryption'. + + Tests that installation: + - downloads/uploads ova file, + - creates photon temp vapp, + - creates k8s templates + - skips deleting the temp vapp + - checks that proper packages are installed in the vm in temp vApp + + command: cse install --config cse_test_config.yaml --retain-temp-vapp + --skip-config-decryption --ssh-key ~/.ssh/id_rsa.pub + required files: ~/.ssh/id_rsa.pub, cse_test_config.yaml + expected: cse registered, catalog exists, source OVAs exist, + temp vapps exist, k8s templates exist. + """ + cmd = f"install --config {env.ACTIVE_CONFIG_FILEPATH} --ssh-key " \ + f"{env.SSH_KEY_FILEPATH} --retain-temp-vapp --skip-config-decryption" + result = env.CLI_RUNNER.invoke(cli, cmd.split(), + catch_exceptions=False) + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + assert result.exit_code == 0,\ + testutils.format_command_info('cse', cmd, result.exit_code, + result.output) + + # check that cse was registered correctly + env.check_cse_registration_as_mqtt_extension() + remote_template_keys = server_utils.get_template_descriptor_keys( + env.TEMPLATE_COOKBOOK_VERSION) + + vdc = VDC(env.CLIENT, href=env.VDC_HREF) + for template_config in env.TEMPLATE_DEFINITIONS: + # check that source ova file exists in catalog + assert env.catalog_item_exists( + template_config[remote_template_keys.SOURCE_OVA_NAME], + logger=PYTEST_LOGGER), \ + 'Source ova file does not exist when it should.' + + # check that k8s templates exist + catalog_item_name = ltm.get_revisioned_template_name( + template_config[remote_template_keys.NAME], + template_config[remote_template_keys.REVISION]) + assert env.catalog_item_exists( + catalog_item_name, logger=PYTEST_LOGGER), \ + 'k8s template does not exist when it should.' + + # check that temp vapp exists + temp_vapp_name = testutils.get_temp_vapp_name( + template_config[remote_template_keys.NAME]) + try: + vdc.reload() + vdc.get_vapp(temp_vapp_name) + except EntityNotFoundException: + assert False, 'vApp does not exist when it should.' + + +@pytest.mark.skipif(env.TEST_ALL_TEMPLATES, + reason="Configuration specifies 'test_all_templates' as True") # noqa: E501 +def test_0100_install_select_templates(config, unregister_cse_before_test): + """Tests template installation. + + Tests that selected template installation is done correctly + + command: cse template install template_name template_revision + --config cse_test_config.yaml --ssh-key ~/.ssh/id_rsa.pub + --skip-config-decryption --retain-temp-vapp + required files: cse_test_config.yaml, ~/.ssh/id_rsa.pub, + ubuntu/photon init/cust scripts + expected: cse registered, source OVAs exist, k8s templates exist and + temp vapps exist. + """ + cmd = f"install --config {env.ACTIVE_CONFIG_FILEPATH} --ssh-key " \ + f"{env.SSH_KEY_FILEPATH} --skip-template-creation " \ + f"--skip-config-decryption" + result = env.CLI_RUNNER.invoke(cli, cmd.split(), catch_exceptions=False) + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + assert result.exit_code == 0,\ + testutils.format_command_info('cse', cmd, result.exit_code, + result.output) + + # check that cse was registered correctly + env.check_cse_registration_as_mqtt_extension() + remote_template_keys = server_utils.get_template_descriptor_keys( + env.TEMPLATE_COOKBOOK_VERSION) + + vdc = VDC(env.CLIENT, href=env.VDC_HREF) + for template_config in env.TEMPLATE_DEFINITIONS: + # install the template + cmd = f"template install {template_config[remote_template_keys.NAME]} " \ + f"{template_config['revision']} " \ + f"--config {env.ACTIVE_CONFIG_FILEPATH} " \ + f"--ssh-key {env.SSH_KEY_FILEPATH} " \ + f"--skip-config-decryption --force --retain-temp-vapp" # noqa: E501 + result = env.CLI_RUNNER.invoke( + cli, cmd.split(), catch_exceptions=False) + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + assert result.exit_code == 0,\ + testutils.format_command_info('cse', cmd, result.exit_code, + result.output) + # check that source ova file exists in catalog + assert env.catalog_item_exists( + template_config[remote_template_keys.SOURCE_OVA_NAME], + logger=PYTEST_LOGGER), \ + 'Source ova file does not exists when it should.' + + # check that k8s templates exist + catalog_item_name = ltm.get_revisioned_template_name( + template_config[remote_template_keys.NAME], + template_config[remote_template_keys.REVISION]) + assert env.catalog_item_exists( + catalog_item_name, logger=PYTEST_LOGGER), \ + 'k8s template does not exist when it should.' + + # check that temp vapp exists + temp_vapp_name = testutils.get_temp_vapp_name( + template_config[remote_template_keys.NAME]) + try: + vdc.reload() + vdc.get_vapp(temp_vapp_name) + except EntityNotFoundException: + assert False, 'vApp does not exist when it should.' + + +def test_0110_cse_check_valid_installation(config): + """Tests that `cse check` passes for a valid installation. + + command: cse check cse_test_config.yaml -i -s + expected: check passes + """ + try: + cmd = f"check {env.ACTIVE_CONFIG_FILEPATH} --skip-config-decryption --check-install" # noqa: E501 + result = env.CLI_RUNNER.invoke(cli, + cmd.split(), + catch_exceptions=False) + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + except EntityNotFoundException: + msg = "cse check failed when it should have passed." + PYTEST_LOGGER.debug(msg) + assert False, msg + + +def test_0120_cse_run(config): + """Test `cse run` command. + + Run cse server as a subprocess with a timeout. If we + reach the timeout, then cse server was valid and running. Exiting the + process before then means that server run failed somehow. + + commands: + $ cse run -c cse_test_config + $ cse run -c cse_test_config --skip-check --skip-config-decryption + """ + cmds = [ + f"cse run -c {env.ACTIVE_CONFIG_FILEPATH} --skip-config-decryption", + f"cse run -c {env.ACTIVE_CONFIG_FILEPATH} --skip-check --skip-config-decryption" # noqa: E501 + ] + + for cmd in cmds: + p = None + try: + if os.name == 'nt': + p = subprocess.Popen(cmd, shell=True) + else: + p = subprocess.Popen(cmd.split()) + p.wait(timeout=env.WAIT_INTERVAL * 2) # 1 minute + msg = f"`{cmd}` failed with return code {p.returncode}" + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(msg) + assert False, msg + except subprocess.TimeoutExpired: + pass + finally: + try: + if p: + if os.name == 'nt': + subprocess.run(f"taskkill /f /pid {p.pid} /t") + else: + p.terminate() + except OSError: + pass + + +def test_0130_cse_encrypt_decrypt_with_password_from_stdin(config): + """Test `cse encrypt plain-config.yaml and cse decrypt encrypted-config`. + + Get the password for encrypt/decrypt from stdin. + + 1. Execute `cse encrypt` on plain-config file and store the encrypted file. + 2. Execute `cse decrypt` on the encrypted config file get decrypted file. + 3. Compare plain-config file and decrypted config file and check result. + """ + encrypted_file = tempfile.NamedTemporaryFile() + cmd = f"encrypt {env.ACTIVE_CONFIG_FILEPATH} -o {encrypted_file.name}" # noqa: E501 + result = env.CLI_RUNNER.invoke(cli, + cmd.split(), + input=PASSWORD_FOR_CONFIG_ENCRYPTION, + catch_exceptions=False) + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + + # Run `cse decrypt` on the encrypted config file from previous step + decrypted_file = tempfile.NamedTemporaryFile() + cmd = f"decrypt {encrypted_file.name} -o {decrypted_file.name}" + result = env.CLI_RUNNER.invoke(cli, + cmd.split(), + input=PASSWORD_FOR_CONFIG_ENCRYPTION, + catch_exceptions=False) + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + + # File comparison also include content comparison + assert filecmp.cmp(env.ACTIVE_CONFIG_FILEPATH, decrypted_file.name, + shallow=False) + + +def test_0140_cse_encrypt_decrypt_with_password_from_environment_var(config): + """Test `cse encrypt plain-config.yaml and cse decrypt encrypted-config`. + + Get the password for encrypt/decrypt from environment variable. + + 1. Execute `cse encrypt` on plain-config file and store the encrypted file. + 2. Execute `cse decrypt` on the encrypted config file get decrypted file. + 3. Compare plain-config file and decrypted config file and check result. + """ + os.environ['CSE_CONFIG_PASSWORD'] = PASSWORD_FOR_CONFIG_ENCRYPTION + encrypted_file = tempfile.NamedTemporaryFile() + cmd = f"encrypt {env.ACTIVE_CONFIG_FILEPATH} -o {encrypted_file.name}" # noqa: E501 + result = env.CLI_RUNNER.invoke(cli, cmd.split(), catch_exceptions=False) + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + + # Run `cse decrypt` on the encrypted config file from previous step + decrypted_file = tempfile.NamedTemporaryFile() + cmd = f"decrypt {encrypted_file.name} -o {decrypted_file.name}" + result = env.CLI_RUNNER.invoke(cli, cmd.split(), catch_exceptions=False) + PYTEST_LOGGER.debug(f"Executing command: {cmd}") + PYTEST_LOGGER.debug(f"Exit code: {result.exit_code}") + PYTEST_LOGGER.debug(f"Output: {result.output}") + + # File comparison also include content comparison + assert filecmp.cmp(env.ACTIVE_CONFIG_FILEPATH, decrypted_file.name, + shallow=False) diff --git a/test-requirements.txt b/test-requirements.txt index bcd0eb458..892ba08b8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,4 +3,5 @@ flake8 flake8-import-order flake8-docstrings pytest -pydocstyle < 4 \ No newline at end of file +pydocstyle < 4 +pytest-logger diff --git a/tox.ini b/tox.ini index 74bfa1f79..8930397d0 100644 --- a/tox.ini +++ b/tox.ini @@ -29,4 +29,4 @@ deps = [testenv:flake8] deps = {[testenv]deps} -commands = flake8 container_service_extension system_tests +commands = flake8 container_service_extension system_tests system_tests_v2