From 8fc3657a13a062eb9642e137da623855f9f3343d Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Tue, 19 Mar 2024 12:21:09 +0000 Subject: [PATCH 1/7] wip: Add external integration for ExampleDataSource --- nautobot_ssot/jobs/examples.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/nautobot_ssot/jobs/examples.py b/nautobot_ssot/jobs/examples.py index a3164fc23..f9df6f38f 100644 --- a/nautobot_ssot/jobs/examples.py +++ b/nautobot_ssot/jobs/examples.py @@ -9,7 +9,9 @@ from django.urls import reverse from nautobot.dcim.models import Location, LocationType -from nautobot.extras.jobs import StringVar +from nautobot.extras.choices import SecretsGroupAccessTypeChoices, SecretsGroupSecretTypeChoices +from nautobot.extras.jobs import ObjectVar, StringVar +from nautobot.extras.models import ExternalIntegration from nautobot.ipam.models import Prefix from nautobot.tenancy.models import Tenant @@ -378,6 +380,12 @@ def load(self): class ExampleDataSource(DataSource): """Sync Region and Site data from a remote Nautobot instance into the local Nautobot instance.""" + source = ObjectVar( + model=ExternalIntegration, + queryset=ExternalIntegration.objects.all(), + display_field="display", + label="Nautobot Demo Instance", + ) source_url = StringVar( description="Remote Nautobot instance to load Sites and Regions from", default="https://demo.nautobot.com" ) @@ -408,13 +416,27 @@ def data_mappings(cls): ) def run( - self, dryrun, memory_profiling, source_url, source_token, *args, **kwargs + self, dryrun, memory_profiling, source, source_url, source_token, *args, **kwargs ): # pylint:disable=arguments-differ """Run sync.""" self.dryrun = dryrun self.memory_profiling = memory_profiling - self.source_url = source_url - self.source_token = source_token + try: + if source: + self.logger.info(f"Using external integration '{source}'") + self.source_url = source.remote_url + secrets_group = source.secrets_group + self.source_token = secrets_group.get_secret_value( + access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, + secret_type=SecretsGroupSecretTypeChoices.TYPE_TOKEN, + ) + else: + self.source_url = source_url + self.source_token = source_token + except Exception as e: + self.logger.error(f"Error setting up job: {e}") + raise + super().run(dryrun, memory_profiling, *args, **kwargs) def load_source_adapter(self): From 43cfa6e9d988c0256fc7d4126f8f771c3a1f181c Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Thu, 21 Mar 2024 09:59:19 +0000 Subject: [PATCH 2/7] feat: Add ExternalIntegration to AristaCV --- nautobot_ssot/integrations/aristacv/jobs.py | 177 ++++++++++++++++---- nautobot_ssot/jobs/examples.py | 1 + 2 files changed, 149 insertions(+), 29 deletions(-) diff --git a/nautobot_ssot/integrations/aristacv/jobs.py b/nautobot_ssot/integrations/aristacv/jobs.py index b83ef1fa9..f945bbec8 100644 --- a/nautobot_ssot/integrations/aristacv/jobs.py +++ b/nautobot_ssot/integrations/aristacv/jobs.py @@ -1,23 +1,100 @@ # pylint: disable=invalid-name,too-few-public-methods """Jobs for CloudVision integration with SSoT app.""" +from typing import Mapping +from typing import Optional +from urllib.parse import urlparse + from django.templatetags.static import static from django.urls import reverse - -from nautobot.dcim.models import DeviceType -from nautobot.extras.jobs import Job, BooleanVar +from nautobot.core.settings_funcs import is_truthy from nautobot.core.utils.lookup import get_route_for_model -from nautobot_ssot.jobs.base import DataTarget, DataSource, DataMapping +from nautobot.dcim.models import DeviceType +from nautobot.extras.choices import SecretsGroupAccessTypeChoices +from nautobot.extras.choices import SecretsGroupSecretTypeChoices +from nautobot.extras.jobs import BooleanVar +from nautobot.extras.jobs import Job +from nautobot.extras.jobs import ObjectVar +from nautobot.extras.models import ExternalIntegration +from nautobot.extras.models import SecretsGroup from nautobot_ssot.integrations.aristacv.constant import APP_SETTINGS from nautobot_ssot.integrations.aristacv.diffsync.adapters.cloudvision import CloudvisionAdapter from nautobot_ssot.integrations.aristacv.diffsync.adapters.nautobot import NautobotAdapter from nautobot_ssot.integrations.aristacv.diffsync.models import nautobot from nautobot_ssot.integrations.aristacv.utils.cloudvision import CloudvisionApi - +from nautobot_ssot.jobs.base import DataMapping +from nautobot_ssot.jobs.base import DataSource +from nautobot_ssot.jobs.base import DataTarget name = "SSoT - Arista CloudVision" # pylint: disable=invalid-name +def _get_settings(source: Optional[ExternalIntegration]) -> dict: + source_config = source.extra_config if source and isinstance(source.extra_config, Mapping) else {} + # On premise is a default behavior for ExternalIntegration + is_on_prem = bool(source_config.get("is_on_prem", True) if source else APP_SETTINGS.get("aristacv_cvp_host")) + + settings = { + "is_on_prem": is_on_prem, + "delete_devices_on_sync": is_truthy( + APP_SETTINGS.get("aristacv_delete_devices_on_sync", nautobot.DEFAULT_DELETE_DEVICES_ON_SYNC) + ), + "from_cloudvision_default_site": APP_SETTINGS.get( + "aristacv_from_cloudvision_default_site", nautobot.DEFAULT_SITE + ), + "from_cloudvision_default_device_role": APP_SETTINGS.get( + "aristacv_from_cloudvision_default_device_role", nautobot.DEFAULT_DEVICE_ROLE + ), + "from_cloudvision_default_device_role_color": APP_SETTINGS.get( + "aristacv_from_cloudvision_default_device_role_color", nautobot.DEFAULT_DEVICE_ROLE_COLOR + ), + "apply_import_tag": is_truthy(APP_SETTINGS.get("aristacv_apply_import_tag", nautobot.APPLY_IMPORT_TAG)), + "import_active": APP_SETTINGS.get("aristacv_import_active"), + "verify": APP_SETTINGS.get("aristacv_verify"), + "cvp_host": APP_SETTINGS.get("aristacv_cvp_host"), + "cvp_user": APP_SETTINGS.get("aristacv_cvp_user"), + "cvp_password": APP_SETTINGS.get("aristacv_cvp_password"), + "cvp_token": APP_SETTINGS.get("aristacv_cvp_token"), + "cvp_port": APP_SETTINGS.get("aristacv_cvp_port"), + } + + if not source: + return settings + + if isinstance(source.verify_ssl, bool): + settings["verify"] = source.verify_ssl + + if is_on_prem: + parsed_url = urlparse(source.remote_url) # type: ignore + if parsed_url: + settings["cvp_host"] = parsed_url.hostname + settings["cvp_port"] = parsed_url.port + else: + settings["cvaas_url"] = source.remote_url + return settings + + secrets_group: SecretsGroup = source.secrets_group # type: ignore + if not secrets_group: + return settings + + settings["cvp_user"] = secrets_group.get_secret_value( + access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, + secret_type=SecretsGroupSecretTypeChoices.TYPE_USERNAME, + ) + settings["cvp_password"] = secrets_group.get_secret_value( + access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, + secret_type=SecretsGroupSecretTypeChoices.TYPE_PASSWORD, + ) + settings["cvp_token"] = secrets_group.get_secret_value( + access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, + secret_type=SecretsGroupSecretTypeChoices.TYPE_TOKEN, + ) + + settings.update(source_config) + + return settings + + class MissingConfigSetting(Exception): """Exception raised for missing configuration settings. @@ -35,6 +112,13 @@ def __init__(self, setting): class CloudVisionDataSource(DataSource, Job): # pylint: disable=abstract-method """CloudVision SSoT Data Source.""" + source = ObjectVar( + model=ExternalIntegration, + queryset=ExternalIntegration.objects.all(), + display_field="display", + label="Arista CloudVision External Integration", + description="ExternalIntegration containing information for connecting to Arista CloudVision", + ) debug = BooleanVar(description="Enable for more verbose debug logging") class Meta: @@ -72,7 +156,7 @@ def config_information(cls): "aristacv_from_cloudvision_default_device_role_color", nautobot.DEFAULT_DEVICE_ROLE_COLOR ), "Apply import tag": str(APP_SETTINGS.get("aristacv_apply_import_tag", nautobot.APPLY_IMPORT_TAG)), - "Import Active": str(APP_SETTINGS.get("aristacv_import_active", "True")) + "Import Active": str(APP_SETTINGS.get("aristacv_import_active", "True")), # Password and Token are intentionally omitted! } @@ -100,18 +184,18 @@ def data_mappings(cls): def load_source_adapter(self): """Load data from CloudVision into DiffSync models.""" - if not APP_SETTINGS.get("aristacv_from_cloudvision_default_site"): + if not self.source_settings["from_cloudvision_default_site"]: self.logger.error( - "App setting `aristacv_from_cloudvision_default_site` is not defined. This setting is required for the App to function." + "App setting `from_cloudvision_default_site` is not defined. This setting is required for the App to function." ) - raise MissingConfigSetting(setting="aristacv_from_cloudvision_default_site") - if not APP_SETTINGS.get("aristacv_from_cloudvision_default_device_role"): + raise MissingConfigSetting(setting="from_cloudvision_default_site") + if not self.source_settings["from_cloudvision_default_device_role"]: self.logger.error( - "App setting `aristacv_from_cloudvision_default_device_role` is not defined. This setting is required for the App to function." + "App setting `from_cloudvision_default_device_role` is not defined. This setting is required for the App to function." ) - raise MissingConfigSetting(setting="aristacv_from_cloudvision_default_device_role") + raise MissingConfigSetting(setting="from_cloudvision_default_device_role") if self.debug: - if APP_SETTINGS.get("aristacv_delete_devices_on_sync"): + if self.source_settings["delete_devices_on_sync"]: self.logger.warning( "Devices not present in Cloudvision but present in Nautobot will be deleted from Nautobot." ) @@ -121,12 +205,12 @@ def load_source_adapter(self): ) self.logger.info("Connecting to CloudVision") with CloudvisionApi( - cvp_host=APP_SETTINGS["aristacv_cvp_host"], - cvp_port=APP_SETTINGS.get("aristacv_cvp_port", "8443"), - verify=APP_SETTINGS["aristacv_verify"], - username=APP_SETTINGS["aristacv_cvp_user"], - password=APP_SETTINGS["aristacv_cvp_password"], - cvp_token=APP_SETTINGS["aristacv_cvp_token"], + cvp_host=self.source_settings["cvp_host"], + cvp_port=self.source_settings["cvp_port"], + verify=self.source_settings["verify"], + username=self.source_settings["cvp_user"], + password=self.source_settings["cvp_password"], + cvp_token=self.source_settings["cvp_token"], ) as client: self.logger.info("Loading data from CloudVision") self.source_adapter = CloudvisionAdapter(job=self, conn=client) @@ -139,18 +223,39 @@ def load_target_adapter(self): self.target_adapter.load() def run( # pylint: disable=arguments-differ, too-many-arguments, duplicate-code - self, dryrun, memory_profiling, debug, *args, **kwargs + self, + source, + dryrun, + memory_profiling, + debug, + *args, + **kwargs, ): """Perform data synchronization.""" self.debug = debug self.dryrun = dryrun self.memory_profiling = memory_profiling + + try: + self.source_settings = _get_settings(source) + except Exception as exc: + # TBD: Why is this exception swallowed? + self.logger.error(exc) + raise + super().run(dryrun=self.dryrun, memory_profiling=self.memory_profiling, *args, **kwargs) class CloudVisionDataTarget(DataTarget, Job): # pylint: disable=abstract-method """CloudVision SSoT Data Target.""" + target = ObjectVar( + model=ExternalIntegration, + queryset=ExternalIntegration.objects.all(), + display_field="display", + label="Arista CloudVision External Integration", + description="ExternalIntegration containing information for connecting to Arista CloudVision", + ) debug = BooleanVar(description="Enable for more verbose debug logging") class Meta: @@ -169,7 +274,7 @@ def config_information(cls): "Server type": "On prem", "CloudVision host": APP_SETTINGS.get("aristacv_cvp_host"), "Username": APP_SETTINGS.get("aristacv_cvp_user"), - "Verify": str(APP_SETTINGS.get("aristacv_verify")) + "Verify": str(APP_SETTINGS.get("aristacv_verify")), # Password is intentionally omitted! } return { @@ -192,7 +297,7 @@ def load_source_adapter(self): def load_target_adapter(self): """Load data from CloudVision into DiffSync models.""" if self.debug: - if APP_SETTINGS.get("aristacv_delete_devices_on_sync"): + if self.target_settings["delete_devices_on_sync"]: self.logger.warning( "Devices not present in Cloudvision but present in Nautobot will be deleted from Nautobot." ) @@ -202,24 +307,38 @@ def load_target_adapter(self): ) self.logger.info("Connecting to CloudVision") with CloudvisionApi( - cvp_host=APP_SETTINGS["aristacv_cvp_host"], - cvp_port=APP_SETTINGS.get("aristacv_cvp_port", "8443"), - verify=APP_SETTINGS["aristacv_verify"], - username=APP_SETTINGS["aristacv_cvp_user"], - password=APP_SETTINGS["aristacv_cvp_password"], - cvp_token=APP_SETTINGS["aristacv_cvp_token"], + cvp_host=self.target_settings["cvp_host"], + cvp_port=self.target_settings["cvp_port"], + verify=self.target_settings["verify"], + username=self.target_settings["cvp_user"], + password=self.target_settings["cvp_password"], + cvp_token=self.target_settings["cvp_token"], ) as client: self.logger.info("Loading data from CloudVision") self.target_adapter = CloudvisionAdapter(job=self, conn=client) self.target_adapter.load() def run( # pylint: disable=arguments-differ, too-many-arguments, duplicate-code - self, dryrun, memory_profiling, debug, *args, **kwargs + self, + target, + dryrun, + memory_profiling, + debug, + *args, + **kwargs, ): """Perform data synchronization.""" self.debug = debug self.dryrun = dryrun self.memory_profiling = memory_profiling + + try: + self.target_settings = _get_settings(target) + except Exception as exc: + # TBD: Why is this exception swallowed? + self.logger.error(exc) + raise + super().run(dryrun=self.dryrun, memory_profiling=self.memory_profiling, *args, **kwargs) diff --git a/nautobot_ssot/jobs/examples.py b/nautobot_ssot/jobs/examples.py index f9df6f38f..3d3a1baa9 100644 --- a/nautobot_ssot/jobs/examples.py +++ b/nautobot_ssot/jobs/examples.py @@ -434,6 +434,7 @@ def run( self.source_url = source_url self.source_token = source_token except Exception as e: + # TBD: Why are these exceptions swallowed? self.logger.error(f"Error setting up job: {e}") raise From b904ffe6a06238ca9ff911da06765f3665b7ac82 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Thu, 21 Mar 2024 10:02:27 +0000 Subject: [PATCH 3/7] chore: Added changelog fragment --- changes/398.changed | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/398.changed diff --git a/changes/398.changed b/changes/398.changed new file mode 100644 index 000000000..9b449207a --- /dev/null +++ b/changes/398.changed @@ -0,0 +1 @@ +Changed Arista Cloud Vision jobs to optionally use ExternalIntegration. From 042926b0c07d497a9022a3fcea31541df5726a4a Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Mon, 25 Mar 2024 14:52:50 +0000 Subject: [PATCH 4/7] feat: Add `aristacv_external_integration_name` plugin config option --- nautobot_ssot/__init__.py | 1 + .../aristacv/{constant.py => constants.py} | 26 +- .../aristacv/diffsync/adapters/cloudvision.py | 17 +- .../aristacv/diffsync/adapters/nautobot.py | 5 +- .../aristacv/diffsync/models/cloudvision.py | 43 ++-- .../aristacv/diffsync/models/nautobot.py | 49 ++-- nautobot_ssot/integrations/aristacv/jobs.py | 209 ++++------------ .../integrations/aristacv/signals.py | 6 +- nautobot_ssot/integrations/aristacv/types.py | 25 ++ .../aristacv/utils/cloudvision.py | 95 +++---- .../integrations/aristacv/utils/nautobot.py | 234 ++++++++++++++---- nautobot_ssot/jobs/examples.py | 48 ++-- .../aristacv/test_cloudvision_adapter.py | 11 +- nautobot_ssot/tests/aristacv/test_jobs.py | 71 +++--- .../tests/aristacv/test_utils_cloudvision.py | 60 +++-- .../tests/aristacv/test_utils_nautobot.py | 107 +++----- 16 files changed, 509 insertions(+), 498 deletions(-) rename nautobot_ssot/integrations/aristacv/{constant.py => constants.py} (88%) create mode 100644 nautobot_ssot/integrations/aristacv/types.py diff --git a/nautobot_ssot/__init__.py b/nautobot_ssot/__init__.py index aa1a5c41f..27bc4bc1c 100644 --- a/nautobot_ssot/__init__.py +++ b/nautobot_ssot/__init__.py @@ -74,6 +74,7 @@ class NautobotSSOTAppConfig(NautobotAppConfig): "aristacv_from_cloudvision_default_site": "", "aristacv_hostname_patterns": [], "aristacv_import_active": False, + "aristacv_external_integration_name": "", "aristacv_role_mappings": {}, "aristacv_site_mappings": {}, "aristacv_verify": True, diff --git a/nautobot_ssot/integrations/aristacv/constant.py b/nautobot_ssot/integrations/aristacv/constants.py similarity index 88% rename from nautobot_ssot/integrations/aristacv/constant.py rename to nautobot_ssot/integrations/aristacv/constants.py index 5a30f014a..876a26988 100644 --- a/nautobot_ssot/integrations/aristacv/constant.py +++ b/nautobot_ssot/integrations/aristacv/constants.py @@ -1,14 +1,18 @@ """Storage of data that will not change throughout the life cycle of the application.""" -from django.conf import settings - - -def _read_settings() -> dict: - config = settings.PLUGINS_CONFIG["nautobot_ssot"] - return config - - -APP_SETTINGS = _read_settings() +ARISTA_PLATFORM = "arista.eos.eos" +CLOUDVISION_PLATFORM = "Arista EOS-CloudVision" +DEFAULT_APPLY_IMPORT_TAG = False +DEFAULT_CREATE_CONTROLLER = False +DEFAULT_CVAAS_URL = "https://www.arista.io" +DEFAULT_DELETE_DEVICES_ON_SYNC = False +DEFAULT_DEVICE_ROLE = "network" +DEFAULT_DEVICE_ROLE_COLOR = "ff0000" +DEFAULT_DEVICE_STATUS = "cloudvision_imported" +DEFAULT_DEVICE_STATUS_COLOR = "ff0000" +DEFAULT_IMPORT_ACTIVE = False +DEFAULT_SITE = "cloudvision_imported" +DEFAULT_VERIFY_SSL = True PORT_TYPE_MAP = { "xcvr1000BaseT": "1000base-t", @@ -83,7 +87,3 @@ def _read_settings() -> dict: "400GBASE-2FR4": "400gbase-x-osfp", "400GBASE-ZR": "400gbase-x-qsfpdd", } - -CLOUDVISION_PLATFORM = "Arista EOS-CloudVision" - -ARISTA_PLATFORM = "arista.eos.eos" diff --git a/nautobot_ssot/integrations/aristacv/diffsync/adapters/cloudvision.py b/nautobot_ssot/integrations/aristacv/diffsync/adapters/cloudvision.py index 40c460f2e..074772bae 100644 --- a/nautobot_ssot/integrations/aristacv/diffsync/adapters/cloudvision.py +++ b/nautobot_ssot/integrations/aristacv/diffsync/adapters/cloudvision.py @@ -7,7 +7,6 @@ from diffsync import DiffSync from diffsync.exceptions import ObjectAlreadyExists, ObjectNotFound -from nautobot_ssot.integrations.aristacv.constant import APP_SETTINGS from nautobot_ssot.integrations.aristacv.diffsync.models.cloudvision import ( CloudvisionCustomField, CloudvisionDevice, @@ -17,6 +16,7 @@ CloudvisionIPAddress, CloudvisionIPAssignment, ) +from nautobot_ssot.integrations.aristacv.types import CloudVisionAppConfig from nautobot_ssot.integrations.aristacv.utils import cloudvision @@ -41,8 +41,13 @@ def __init__(self, *args, job=None, conn: cloudvision.CloudvisionApi, **kwargs): def load_devices(self): """Load devices from CloudVision.""" - if APP_SETTINGS.get("aristacv_create_controller"): - cvp_version = cloudvision.get_cvp_version() + config: CloudVisionAppConfig = self.job.app_config + if config.hostname_patterns and not (config.site_mappings and config.role_mappings): + self.job.logger.warning( + "Configuration found for aristacv_hostname_patterns but no aristacv_site_mappings or aristacv_role_mappings. Please ensure your mappings are defined." + ) + if config.create_controller: + cvp_version = cloudvision.get_cvp_version(config) cvp_ver_cf = self.cf(name="arista_eos", value=cvp_version, device_name="CloudVision") try: self.add(cvp_ver_cf) @@ -258,10 +263,4 @@ def load_device_tags(self, device): def load(self): """Load devices and associated data from CloudVision.""" - if APP_SETTINGS.get("aristacv_hostname_patterns") and not ( - APP_SETTINGS.get("aristacv_site_mappings") and APP_SETTINGS.get("aristacv_role_mappings") - ): - self.job.logger.warning( - "Configuration found for aristacv_hostname_patterns but no aristacv_site_mappings or aristacv_role_mappings. Please ensure your mappings are defined." - ) self.load_devices() diff --git a/nautobot_ssot/integrations/aristacv/diffsync/adapters/nautobot.py b/nautobot_ssot/integrations/aristacv/diffsync/adapters/nautobot.py index 8cc12e337..5e994a549 100644 --- a/nautobot_ssot/integrations/aristacv/diffsync/adapters/nautobot.py +++ b/nautobot_ssot/integrations/aristacv/diffsync/adapters/nautobot.py @@ -11,7 +11,6 @@ from diffsync import DiffSync from diffsync.exceptions import ObjectNotFound, ObjectAlreadyExists -from nautobot_ssot.integrations.aristacv.constant import APP_SETTINGS from nautobot_ssot.integrations.aristacv.diffsync.models.nautobot import ( NautobotDevice, NautobotCustomField, @@ -21,6 +20,7 @@ NautobotIPAssignment, NautobotPort, ) +from nautobot_ssot.integrations.aristacv.types import CloudVisionAppConfig from nautobot_ssot.integrations.aristacv.utils import nautobot @@ -166,8 +166,9 @@ def sync_complete(self, source: DiffSync, *args, **kwargs): self.job.logger.warning(f"Deletion failed for protected object: {nautobot_object}. {err}") self.objects_to_delete[grouping] = [] + config: CloudVisionAppConfig = self.job.app_config # type: ignore # if Controller is created we need to ensure all imported Devices have RelationshipAssociation to it. - if APP_SETTINGS.get("aristacv_create_controller"): + if config.create_controller: self.job.logger.info("Creating Relationships between CloudVision and connected Devices.") controller_relation = OrmRelationship.objects.get(label="Controller -> Device") device_ct = ContentType.objects.get_for_model(OrmDevice) diff --git a/nautobot_ssot/integrations/aristacv/diffsync/models/cloudvision.py b/nautobot_ssot/integrations/aristacv/diffsync/models/cloudvision.py index 015e8f5c0..5763c2556 100644 --- a/nautobot_ssot/integrations/aristacv/diffsync/models/cloudvision.py +++ b/nautobot_ssot/integrations/aristacv/diffsync/models/cloudvision.py @@ -1,5 +1,4 @@ -"""Cloudvision DiffSync models for AristaCV SSoT.""" -from nautobot_ssot.integrations.aristacv.constant import APP_SETTINGS +"""CloudVision DiffSync models for AristaCV SSoT.""" from nautobot_ssot.integrations.aristacv.diffsync.models.base import ( Device, CustomField, @@ -9,11 +8,12 @@ IPAssignment, Port, ) +from nautobot_ssot.integrations.aristacv.types import CloudVisionAppConfig from nautobot_ssot.integrations.aristacv.utils.cloudvision import CloudvisionApi class CloudvisionDevice(Device): - """Cloudvision Device model.""" + """CloudVision Device model.""" @classmethod def create(cls, diffsync, ids, attrs): @@ -30,7 +30,7 @@ def delete(self): class CloudvisionPort(Port): - """Cloudvision Port model.""" + """CloudVision Port model.""" @classmethod def create(cls, diffsync, ids, attrs): @@ -47,7 +47,7 @@ def delete(self): class CloudvisionNamespace(Namespace): - """Cloudvision Namespace model.""" + """CloudVision Namespace model.""" @classmethod def create(cls, diffsync, ids, attrs): @@ -67,7 +67,7 @@ def delete(self): class CloudvisionPrefix(Prefix): - """Cloudvision IPAdress model.""" + """CloudVision IPAdress model.""" @classmethod def create(cls, diffsync, ids, attrs): @@ -87,7 +87,7 @@ def delete(self): class CloudvisionIPAddress(IPAddress): - """Cloudvision IPAdress model.""" + """CloudVision IPAdress model.""" @classmethod def create(cls, diffsync, ids, attrs): @@ -107,7 +107,7 @@ def delete(self): class CloudvisionIPAssignment(IPAssignment): - """Cloudvision IPAssignment model.""" + """CloudVision IPAssignment model.""" @classmethod def create(cls, diffsync, ids, attrs): @@ -127,24 +127,19 @@ def delete(self): class CloudvisionCustomField(CustomField): - """Cloudvision CustomField model.""" + """CloudVision CustomField model.""" @staticmethod - def connect_cvp(): - """Connect to Cloudvision gRPC endpoint.""" - return CloudvisionApi( - cvp_host=APP_SETTINGS["aristacv_cvp_host"], - cvp_port=APP_SETTINGS.get("aristacv_cvp_port", "8443"), - verify=APP_SETTINGS["aristacv_verify"], - username=APP_SETTINGS["aristacv_cvp_user"], - password=APP_SETTINGS["aristacv_cvp_password"], - cvp_token=APP_SETTINGS["aristacv_cvp_token"], - ) + def connect_cvp(config: CloudVisionAppConfig): + """Connect to CloudVision gRPC endpoint.""" + return CloudvisionApi(config) @classmethod def create(cls, diffsync, ids, attrs): """Create a user tag in cvp.""" - cvp = cls.connect_cvp() + config: CloudVisionAppConfig = diffsync.job.app_config # type: ignore + # TBD: Isn't this a performance bottleneck? We are connecting to CVP for each operation. + cvp = cls.connect_cvp(config) cvp.create_tag(ids["name"], attrs["value"]) # Create mapping from device_name to CloudVision device_id device_ids = {dev["hostname"]: dev["device_id"] for dev in cvp.get_devices()} @@ -159,7 +154,9 @@ def create(cls, diffsync, ids, attrs): def update(self, attrs): """Update user tag in cvp.""" - cvp = self.connect_cvp() + config: CloudVisionAppConfig = self.diffsync.job.app_config # type: ignore + # TBD: Isn't this a performance bottleneck? We are connecting to CVP for each operation. + cvp = self.connect_cvp(config) remove = set(self.device_name) - set(attrs["devices"]) add = set(attrs["devices"]) - set(self.device_name) # Create mapping from device_name to CloudVision device_id @@ -180,7 +177,9 @@ def update(self, attrs): def delete(self): """Delete user tag applied to devices in cvp.""" - cvp = self.connect_cvp() + config: CloudVisionAppConfig = self.diffsync.job.app_config # type: ignore + # TBD: Isn't this performance bottleneck? We are connecting to CVP for each operation. + cvp = self.connect_cvp(config) device_ids = {dev["hostname"]: dev["device_id"] for dev in cvp.get_devices()} for device in self.device_name: cvp.remove_tag_from_device(device_ids[device], self.name, self.value) diff --git a/nautobot_ssot/integrations/aristacv/diffsync/models/nautobot.py b/nautobot_ssot/integrations/aristacv/diffsync/models/nautobot.py index a67708862..8ce5c20ac 100644 --- a/nautobot_ssot/integrations/aristacv/diffsync/models/nautobot.py +++ b/nautobot_ssot/integrations/aristacv/diffsync/models/nautobot.py @@ -14,10 +14,10 @@ from nautobot.ipam.models import IPAddressToInterface import distutils -from nautobot_ssot.integrations.aristacv.constant import ( - APP_SETTINGS, +from nautobot_ssot.integrations.aristacv.constants import ( ARISTA_PLATFORM, CLOUDVISION_PLATFORM, + DEFAULT_DEVICE_ROLE_COLOR, ) from nautobot_ssot.integrations.aristacv.diffsync.models.base import ( Device, @@ -28,6 +28,7 @@ Port, Prefix, ) +from nautobot_ssot.integrations.aristacv.types import CloudVisionAppConfig from nautobot_ssot.integrations.aristacv.utils import nautobot try: @@ -38,15 +39,6 @@ print("Device Lifecycle app isn't installed so will revert to CustomField for OS version.") LIFECYCLE_MGMT = False - -# TODO: Move to constant.py -DEFAULT_SITE = "cloudvision_imported" -DEFAULT_DEVICE_ROLE = "network" -DEFAULT_DEVICE_ROLE_COLOR = "ff0000" -DEFAULT_DEVICE_STATUS = "cloudvision_imported" -DEFAULT_DEVICE_STATUS_COLOR = "ff0000" -DEFAULT_DELETE_DEVICES_ON_SYNC = False -APPLY_IMPORT_TAG = False MISSING_CUSTOM_FIELDS = [] @@ -56,40 +48,35 @@ class NautobotDevice(Device): @classmethod def create(cls, diffsync, ids, attrs): """Create device object in Nautobot.""" - site_code, role_code = nautobot.parse_hostname(ids["name"].lower()) - site_map = APP_SETTINGS.get("aristacv_site_mappings") - role_map = APP_SETTINGS.get("aristacv_role_mappings") + config: CloudVisionAppConfig = diffsync.job.app_config # type: ignore + site_code, role_code = nautobot.parse_hostname(ids["name"].lower(), config.hostname_patterns) + site_map = config.site_mappings + role_map = config.role_mappings if site_code and site_code in site_map: site = nautobot.verify_site(site_map[site_code]) elif "CloudVision" in ids["name"]: - if APP_SETTINGS.get("aristacv_controller_site"): - site = nautobot.verify_site(APP_SETTINGS["aristacv_controller_site"]) + if config.controller_site: + site = nautobot.verify_site(config.controller_site) else: site = nautobot.verify_site("CloudVision") else: - site = nautobot.verify_site(APP_SETTINGS.get("aristacv_from_cloudvision_default_site", DEFAULT_SITE)) + site = nautobot.verify_site(config.from_cloudvision_default_site) if role_code and role_code in role_map: role = nautobot.verify_device_role_object( role_map[role_code], - APP_SETTINGS.get( - "aristacv_from_cloudvision_default_device_role_color", - DEFAULT_DEVICE_ROLE_COLOR, - ), + config.from_cloudvision_default_device_role_color, ) elif "CloudVision" in ids["name"]: role = nautobot.verify_device_role_object("Controller", DEFAULT_DEVICE_ROLE_COLOR) else: role = nautobot.verify_device_role_object( - APP_SETTINGS.get("aristacv_from_cloudvision_default_device_role", DEFAULT_DEVICE_ROLE), - APP_SETTINGS.get( - "aristacv_from_cloudvision_default_device_role_color", - DEFAULT_DEVICE_ROLE_COLOR, - ), + config.from_cloudvision_default_device_role, + config.from_cloudvision_default_device_role_color, ) - if APP_SETTINGS.get("aristacv_create_controller") and "CloudVision" in ids["name"]: + if config.create_controller and "CloudVision" in ids["name"]: platform = OrmPlatform.objects.get(name=CLOUDVISION_PLATFORM) else: platform = OrmPlatform.objects.get(name=ARISTA_PLATFORM) @@ -106,7 +93,7 @@ def create(cls, diffsync, ids, attrs): serial=attrs["serial"] if attrs.get("serial") else "", ) - if APP_SETTINGS.get("aristacv_apply_import_tag", APPLY_IMPORT_TAG): + if config.apply_import_tag: import_tag = nautobot.verify_import_tag() new_device.tags.add(import_tag) try: @@ -143,7 +130,8 @@ def update(self, attrs): def delete(self): """Delete device object in Nautobot.""" - if APP_SETTINGS.get("aristacv_delete_devices_on_sync", DEFAULT_DELETE_DEVICES_ON_SYNC): + config: CloudVisionAppConfig = self.diffsync.job.app_config # type: ignore + if config.delete_devices_on_sync: self.diffsync.job.logger.warning(f"Device {self.name} will be deleted per app settings.") device = OrmDevice.objects.get(id=self.uuid) self.diffsync.objects_to_delete["devices"].append(device) @@ -242,7 +230,8 @@ def update(self, attrs): def delete(self): """Delete Interface in Nautobot.""" - if APP_SETTINGS.get("aristacv_delete_devices_on_sync"): + config: CloudVisionAppConfig = self.diffsync.job.app_config # type: ignore + if config.delete_devices_on_sync: super().delete() if self.diffsync.job.debug: self.diffsync.job.logger.warning(f"Interface {self.name} for {self.device} will be deleted.") diff --git a/nautobot_ssot/integrations/aristacv/jobs.py b/nautobot_ssot/integrations/aristacv/jobs.py index f945bbec8..58fc028fa 100644 --- a/nautobot_ssot/integrations/aristacv/jobs.py +++ b/nautobot_ssot/integrations/aristacv/jobs.py @@ -1,27 +1,16 @@ # pylint: disable=invalid-name,too-few-public-methods """Jobs for CloudVision integration with SSoT app.""" -from typing import Mapping -from typing import Optional -from urllib.parse import urlparse from django.templatetags.static import static from django.urls import reverse -from nautobot.core.settings_funcs import is_truthy from nautobot.core.utils.lookup import get_route_for_model from nautobot.dcim.models import DeviceType -from nautobot.extras.choices import SecretsGroupAccessTypeChoices -from nautobot.extras.choices import SecretsGroupSecretTypeChoices from nautobot.extras.jobs import BooleanVar from nautobot.extras.jobs import Job -from nautobot.extras.jobs import ObjectVar -from nautobot.extras.models import ExternalIntegration -from nautobot.extras.models import SecretsGroup - -from nautobot_ssot.integrations.aristacv.constant import APP_SETTINGS from nautobot_ssot.integrations.aristacv.diffsync.adapters.cloudvision import CloudvisionAdapter from nautobot_ssot.integrations.aristacv.diffsync.adapters.nautobot import NautobotAdapter -from nautobot_ssot.integrations.aristacv.diffsync.models import nautobot from nautobot_ssot.integrations.aristacv.utils.cloudvision import CloudvisionApi +from nautobot_ssot.integrations.aristacv.utils.nautobot import get_config from nautobot_ssot.jobs.base import DataMapping from nautobot_ssot.jobs.base import DataSource from nautobot_ssot.jobs.base import DataTarget @@ -29,72 +18,6 @@ name = "SSoT - Arista CloudVision" # pylint: disable=invalid-name -def _get_settings(source: Optional[ExternalIntegration]) -> dict: - source_config = source.extra_config if source and isinstance(source.extra_config, Mapping) else {} - # On premise is a default behavior for ExternalIntegration - is_on_prem = bool(source_config.get("is_on_prem", True) if source else APP_SETTINGS.get("aristacv_cvp_host")) - - settings = { - "is_on_prem": is_on_prem, - "delete_devices_on_sync": is_truthy( - APP_SETTINGS.get("aristacv_delete_devices_on_sync", nautobot.DEFAULT_DELETE_DEVICES_ON_SYNC) - ), - "from_cloudvision_default_site": APP_SETTINGS.get( - "aristacv_from_cloudvision_default_site", nautobot.DEFAULT_SITE - ), - "from_cloudvision_default_device_role": APP_SETTINGS.get( - "aristacv_from_cloudvision_default_device_role", nautobot.DEFAULT_DEVICE_ROLE - ), - "from_cloudvision_default_device_role_color": APP_SETTINGS.get( - "aristacv_from_cloudvision_default_device_role_color", nautobot.DEFAULT_DEVICE_ROLE_COLOR - ), - "apply_import_tag": is_truthy(APP_SETTINGS.get("aristacv_apply_import_tag", nautobot.APPLY_IMPORT_TAG)), - "import_active": APP_SETTINGS.get("aristacv_import_active"), - "verify": APP_SETTINGS.get("aristacv_verify"), - "cvp_host": APP_SETTINGS.get("aristacv_cvp_host"), - "cvp_user": APP_SETTINGS.get("aristacv_cvp_user"), - "cvp_password": APP_SETTINGS.get("aristacv_cvp_password"), - "cvp_token": APP_SETTINGS.get("aristacv_cvp_token"), - "cvp_port": APP_SETTINGS.get("aristacv_cvp_port"), - } - - if not source: - return settings - - if isinstance(source.verify_ssl, bool): - settings["verify"] = source.verify_ssl - - if is_on_prem: - parsed_url = urlparse(source.remote_url) # type: ignore - if parsed_url: - settings["cvp_host"] = parsed_url.hostname - settings["cvp_port"] = parsed_url.port - else: - settings["cvaas_url"] = source.remote_url - return settings - - secrets_group: SecretsGroup = source.secrets_group # type: ignore - if not secrets_group: - return settings - - settings["cvp_user"] = secrets_group.get_secret_value( - access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, - secret_type=SecretsGroupSecretTypeChoices.TYPE_USERNAME, - ) - settings["cvp_password"] = secrets_group.get_secret_value( - access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, - secret_type=SecretsGroupSecretTypeChoices.TYPE_PASSWORD, - ) - settings["cvp_token"] = secrets_group.get_secret_value( - access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, - secret_type=SecretsGroupSecretTypeChoices.TYPE_TOKEN, - ) - - settings.update(source_config) - - return settings - - class MissingConfigSetting(Exception): """Exception raised for missing configuration settings. @@ -112,51 +35,34 @@ def __init__(self, setting): class CloudVisionDataSource(DataSource, Job): # pylint: disable=abstract-method """CloudVision SSoT Data Source.""" - source = ObjectVar( - model=ExternalIntegration, - queryset=ExternalIntegration.objects.all(), - display_field="display", - label="Arista CloudVision External Integration", - description="ExternalIntegration containing information for connecting to Arista CloudVision", - ) debug = BooleanVar(description="Enable for more verbose debug logging") class Meta: """Meta data for DataSource.""" name = "CloudVision ⟹ Nautobot" - data_source = "Cloudvision" + data_source = "CloudVision" data_source_icon = static("nautobot_ssot_aristacv/cvp_logo.png") description = "Sync system tag data from CloudVision to Nautobot" @classmethod def config_information(cls): """Dictionary describing the configuration of this DataSource.""" - if APP_SETTINGS.get("aristacv_cvp_host"): - server_type = "On prem" - host = APP_SETTINGS.get("aristacv_cvp_host") - else: - server_type = "CVaaS" - host = APP_SETTINGS.get("aristacv_cvaas_url") + config = get_config() + + print(100 * "A") + print(config) return { - "Server type": server_type, - "CloudVision host": host, - "Username": APP_SETTINGS.get("aristacv_cvp_user"), - "Verify": str(APP_SETTINGS.get("aristacv_verify")), - "Delete devices on sync": APP_SETTINGS.get( - "aristacv_delete_devices_on_sync", str(nautobot.DEFAULT_DELETE_DEVICES_ON_SYNC) - ), - "New device default site": APP_SETTINGS.get( - "aristacv_from_cloudvision_default_site", nautobot.DEFAULT_SITE - ), - "New device default role": APP_SETTINGS.get( - "aristacv_from_cloudvision_default_device_role", nautobot.DEFAULT_DEVICE_ROLE - ), - "New device default role color": APP_SETTINGS.get( - "aristacv_from_cloudvision_default_device_role_color", nautobot.DEFAULT_DEVICE_ROLE_COLOR - ), - "Apply import tag": str(APP_SETTINGS.get("aristacv_apply_import_tag", nautobot.APPLY_IMPORT_TAG)), - "Import Active": str(APP_SETTINGS.get("aristacv_import_active", "True")), + "Server Type": "On prem" if config.is_on_premise else "CVaaS", + "CloudVision URL": config.url, + "User Name": config.cvp_user, + "Verify SSL": str(config.verify_ssl), + "Delete Devices On Sync": config.delete_devices_on_sync, + "New Device Default Site": config.from_cloudvision_default_site, + "New Device Default Role": config.from_cloudvision_default_device_role, + "New Device Default Role Color": config.from_cloudvision_default_device_role_color, + "Apply Import Tag": str(config.apply_import_tag), + "Import Active": str(config.import_active), # Password and Token are intentionally omitted! } @@ -182,36 +88,34 @@ def data_mappings(cls): DataMapping("topology_type", None, "Topology Type", None), ) + def __init__(self, *args, **kwargs): + """Initialize the CloudVision Data Target.""" + super().__init__(*args, **kwargs) + self.app_config = get_config() + def load_source_adapter(self): """Load data from CloudVision into DiffSync models.""" - if not self.source_settings["from_cloudvision_default_site"]: + if not self.app_config.from_cloudvision_default_site: self.logger.error( "App setting `from_cloudvision_default_site` is not defined. This setting is required for the App to function." ) raise MissingConfigSetting(setting="from_cloudvision_default_site") - if not self.source_settings["from_cloudvision_default_device_role"]: + if not self.app_config.from_cloudvision_default_device_role: self.logger.error( "App setting `from_cloudvision_default_device_role` is not defined. This setting is required for the App to function." ) raise MissingConfigSetting(setting="from_cloudvision_default_device_role") if self.debug: - if self.source_settings["delete_devices_on_sync"]: + if self.app_config.delete_devices_on_sync: self.logger.warning( - "Devices not present in Cloudvision but present in Nautobot will be deleted from Nautobot." + "Devices not present in CloudVision but present in Nautobot will be deleted from Nautobot." ) else: self.logger.warning( - "Devices not present in Cloudvision but present in Nautobot will not be deleted from Nautobot." + "Devices not present in CloudVision but present in Nautobot will not be deleted from Nautobot." ) self.logger.info("Connecting to CloudVision") - with CloudvisionApi( - cvp_host=self.source_settings["cvp_host"], - cvp_port=self.source_settings["cvp_port"], - verify=self.source_settings["verify"], - username=self.source_settings["cvp_user"], - password=self.source_settings["cvp_password"], - cvp_token=self.source_settings["cvp_token"], - ) as client: + with CloudvisionApi(self.app_config) as client: self.logger.info("Loading data from CloudVision") self.source_adapter = CloudvisionAdapter(job=self, conn=client) self.source_adapter.load() @@ -224,7 +128,6 @@ def load_target_adapter(self): def run( # pylint: disable=arguments-differ, too-many-arguments, duplicate-code self, - source, dryrun, memory_profiling, debug, @@ -236,26 +139,12 @@ def run( # pylint: disable=arguments-differ, too-many-arguments, duplicate-code self.dryrun = dryrun self.memory_profiling = memory_profiling - try: - self.source_settings = _get_settings(source) - except Exception as exc: - # TBD: Why is this exception swallowed? - self.logger.error(exc) - raise - super().run(dryrun=self.dryrun, memory_profiling=self.memory_profiling, *args, **kwargs) class CloudVisionDataTarget(DataTarget, Job): # pylint: disable=abstract-method """CloudVision SSoT Data Target.""" - target = ObjectVar( - model=ExternalIntegration, - queryset=ExternalIntegration.objects.all(), - display_field="display", - label="Arista CloudVision External Integration", - description="ExternalIntegration containing information for connecting to Arista CloudVision", - ) debug = BooleanVar(description="Enable for more verbose debug logging") class Meta: @@ -269,17 +158,19 @@ class Meta: @classmethod def config_information(cls): """Dictionary describing the configuration of this DataTarget.""" - if APP_SETTINGS.get("aristacv_cvp_host"): + config = get_config() + + if config.is_on_premise: return { - "Server type": "On prem", - "CloudVision host": APP_SETTINGS.get("aristacv_cvp_host"), - "Username": APP_SETTINGS.get("aristacv_cvp_user"), - "Verify": str(APP_SETTINGS.get("aristacv_verify")), + "Server Type": "On prem", + "CloudVision URL": config.url, + "Verify": str(config.verify_ssl), + "User Name": config.cvp_user, # Password is intentionally omitted! } return { - "Server type": "CVaaS", - "CloudVision host": APP_SETTINGS.get("aristacv_cvaas_url"), + "Server Type": "CVaaS", + "CloudVision URL": config.url, # Token is intentionally omitted! } @@ -288,6 +179,11 @@ def data_mappings(cls): """List describing the data mappings involved in this DataTarget.""" return (DataMapping("Tags", reverse("extras:tag_list"), "Device Tags", None),) + def __init__(self, *args, **kwargs): + """Initialize the CloudVision Data Target.""" + super().__init__(*args, **kwargs) + self.app_config = get_config() + def load_source_adapter(self): """Load data from Nautobot into DiffSync models.""" self.logger.info("Loading data from Nautobot") @@ -297,30 +193,22 @@ def load_source_adapter(self): def load_target_adapter(self): """Load data from CloudVision into DiffSync models.""" if self.debug: - if self.target_settings["delete_devices_on_sync"]: + if self.app_config.delete_devices_on_sync: self.logger.warning( - "Devices not present in Cloudvision but present in Nautobot will be deleted from Nautobot." + "Devices not present in CloudVision but present in Nautobot will be deleted from Nautobot." ) else: self.logger.warning( - "Devices not present in Cloudvision but present in Nautobot will not be deleted from Nautobot." + "Devices not present in CloudVision but present in Nautobot will not be deleted from Nautobot." ) self.logger.info("Connecting to CloudVision") - with CloudvisionApi( - cvp_host=self.target_settings["cvp_host"], - cvp_port=self.target_settings["cvp_port"], - verify=self.target_settings["verify"], - username=self.target_settings["cvp_user"], - password=self.target_settings["cvp_password"], - cvp_token=self.target_settings["cvp_token"], - ) as client: + with CloudvisionApi(self.app_config) as client: self.logger.info("Loading data from CloudVision") self.target_adapter = CloudvisionAdapter(job=self, conn=client) self.target_adapter.load() def run( # pylint: disable=arguments-differ, too-many-arguments, duplicate-code self, - target, dryrun, memory_profiling, debug, @@ -332,13 +220,6 @@ def run( # pylint: disable=arguments-differ, too-many-arguments, duplicate-code self.dryrun = dryrun self.memory_profiling = memory_profiling - try: - self.target_settings = _get_settings(target) - except Exception as exc: - # TBD: Why is this exception swallowed? - self.logger.error(exc) - raise - super().run(dryrun=self.dryrun, memory_profiling=self.memory_profiling, *args, **kwargs) diff --git a/nautobot_ssot/integrations/aristacv/signals.py b/nautobot_ssot/integrations/aristacv/signals.py index 46748e6e1..fda54c728 100644 --- a/nautobot_ssot/integrations/aristacv/signals.py +++ b/nautobot_ssot/integrations/aristacv/signals.py @@ -5,7 +5,7 @@ from django.db.models.signals import post_migrate from nautobot.extras.choices import CustomFieldTypeChoices, RelationshipTypeChoices -from nautobot_ssot.integrations.aristacv.constant import APP_SETTINGS +from nautobot_ssot.integrations.aristacv.utils.nautobot import get_config # pylint: disable-next=unused-argument @@ -15,7 +15,7 @@ def register_signals(sender): post_migrate.connect(post_migrate_create_manufacturer) post_migrate.connect(post_migrate_create_platform) - if APP_SETTINGS.get("aristacv_create_controller"): + if get_config().create_controller: post_migrate.connect(post_migrate_create_controller_relationship) @@ -134,7 +134,7 @@ def post_migrate_create_platform(apps=global_apps, **kwargs): }, ) - if APP_SETTINGS.get("aristacv_create_controller"): + if get_config().create_controller: Platform.objects.get_or_create( name="Arista EOS-CloudVision", manufacturer=Manufacturer.objects.get(name="Arista"), diff --git a/nautobot_ssot/integrations/aristacv/types.py b/nautobot_ssot/integrations/aristacv/types.py new file mode 100644 index 000000000..ed87e2244 --- /dev/null +++ b/nautobot_ssot/integrations/aristacv/types.py @@ -0,0 +1,25 @@ +"""Arista CloudVision Type Definitions.""" + +from typing import NamedTuple + + +class CloudVisionAppConfig(NamedTuple): + """Arista CloudVision Configuration.""" + + is_on_premise: bool + url: str + verify_ssl: bool + cvp_user: str + cvp_password: str + token: str + delete_devices_on_sync: bool + from_cloudvision_default_site: str + from_cloudvision_default_device_role: str + from_cloudvision_default_device_role_color: str + apply_import_tag: bool + import_active: bool + hostname_patterns: list + site_mappings: dict + role_mappings: dict + controller_site: str + create_controller: bool diff --git a/nautobot_ssot/integrations/aristacv/utils/cloudvision.py b/nautobot_ssot/integrations/aristacv/utils/cloudvision.py index 5757ca4a3..3dfff318c 100644 --- a/nautobot_ssot/integrations/aristacv/utils/cloudvision.py +++ b/nautobot_ssot/integrations/aristacv/utils/cloudvision.py @@ -3,6 +3,7 @@ import ssl from datetime import datetime from typing import Any, Iterable, List, Optional, Tuple, Union +from urllib.parse import urlparse import google.protobuf.timestamp_pb2 as pbts import grpc @@ -23,7 +24,8 @@ from cloudvision.Connector.codec.custom_types import FrozenDict from cloudvision.Connector.grpc_client.grpcClient import create_query, to_pbts -from nautobot_ssot.integrations.aristacv.constant import APP_SETTINGS, PORT_TYPE_MAP +from nautobot_ssot.integrations.aristacv.constants import PORT_TYPE_MAP +from nautobot_ssot.integrations.aristacv.types import CloudVisionAppConfig RPC_TIMEOUT = 30 TIME_TYPE = Union[pbts.Timestamp, datetime] @@ -42,66 +44,51 @@ def __init__(self, error_code, message): class CloudvisionApi: # pylint: disable=too-many-instance-attributes, too-many-arguments - """Arista Cloudvision gRPC client.""" + """Arista CloudVision gRPC client.""" AUTH_KEY_PATH = "access_token" - def __init__( - self, - cvp_host: str, - cvp_port: str = "", - verify: bool = True, - username: str = "", - password: str = "", - cvp_token: str = "", - ): - """Create Cloudvision API connection.""" + def __init__(self, config: CloudVisionAppConfig): + """Create CloudVision API connection.""" self.metadata = None - self.cvp_host = cvp_host - self.cvp_port = cvp_port - self.cvp_url = f"{cvp_host}:{cvp_port}" - self.verify = verify - self.username = username - self.password = password - self.cvp_token = cvp_token - - # If CVP_HOST is defined, we assume an on-prem installation. - if self.cvp_host: - if self.verify: + + parsed_url = urlparse(config.url) + if not parsed_url.hostname or not parsed_url.port: + raise ValueError("Invalid URL provided for CloudVision") + token = config.token + if config.is_on_premise: + if config.verify_ssl: channel_creds = grpc.ssl_channel_credentials() else: channel_creds = grpc.ssl_channel_credentials( - bytes(ssl.get_server_certificate((self.cvp_host, int(self.cvp_port))), "utf-8") + bytes(ssl.get_server_certificate((parsed_url.hostname, parsed_url.port)), "utf-8") ) - if self.cvp_token: - call_creds = grpc.access_token_call_credentials(self.cvp_token) - elif self.username != "" and self.password != "": # nosec + if token: + call_creds = grpc.access_token_call_credentials(token) + elif config.cvp_user != "" and config.cvp_password != "": # nosec response = requests.post( # nosec - f"https://{self.cvp_host}/cvpservice/login/authenticate.do", - auth=(self.username, self.password), + f"{parsed_url.hostname}:{parsed_url.port}/cvpservice/login/authenticate.do", + auth=(config.cvp_user, config.cvp_password), timeout=60, - verify=self.verify, + verify=config.verify_ssl, ) session_id = response.json().get("sessionId") if not session_id: error_code = response.json().get("errorCode") error_message = response.json().get("errorMessage") raise AuthFailure(error_code, error_message) - if not self.cvp_token: - self.cvp_token = session_id + token = session_id call_creds = grpc.access_token_call_credentials(session_id) else: raise AuthFailure( error_code="Missing Credentials", message="Unable to authenticate due to missing credentials." ) - self.metadata = ((self.AUTH_KEY_PATH, self.cvp_token),) - # Set up credentials for CVaaS using supplied token. + self.metadata = ((self.AUTH_KEY_PATH, token),) else: - self.cvp_url = APP_SETTINGS.get("aristacv_cvaas_url", "www.arista.io:443") - call_creds = grpc.access_token_call_credentials(self.cvp_token) + call_creds = grpc.access_token_call_credentials(token) channel_creds = grpc.ssl_channel_credentials() conn_creds = grpc.composite_channel_credentials(channel_creds, call_creds) - self.comm_channel = grpc.secure_channel(self.cvp_url, conn_creds) + self.comm_channel = grpc.secure_channel(f"{parsed_url.hostname}:{parsed_url.port}", conn_creds) self.__client = rtr_client.RouterV1Stub(self.comm_channel) self.__auth_client = rtr_client.AuthStub(self.comm_channel) self.__search_client = rtr_client.SearchStub(self.comm_channel) @@ -266,10 +253,10 @@ def search( # pylint:disable=dangerous-default-value, too-many-locals return (self.decode_batch(nb) for nb in res) -def get_devices(client): +def get_devices(client, import_active: bool): """Get devices from CloudVision inventory.""" device_stub = services.DeviceServiceStub(client) - if APP_SETTINGS.get("aristacv_import_active"): + if import_active: req = services.DeviceStreamRequest( partial_eq_filter=[models.Device(streaming_status=models.STREAMING_STATUS_ACTIVE)] ) @@ -440,7 +427,7 @@ def get_device_type(client: CloudvisionApi, dId: str): """Returns the type of the device: modular/fixed. Args: - client (CloudvisionApi): Cloudvision connection. + client (CloudvisionApi): CloudVision connection. dId (str): Device ID to determine type for. Returns: @@ -462,7 +449,7 @@ def get_interfaces_chassis(client: CloudvisionApi, dId): """Gets information about interfaces for a modular device. Args: - client (CloudvisionApi): Cloudvision connection. + client (CloudvisionApi): CloudVision connection. dId (str): Device ID to determine type for. """ # Fetch the list of slices/linecards @@ -502,7 +489,7 @@ def get_interfaces_fixed(client: CloudvisionApi, dId: str): """Gets information about interfaces for a fixed system device. Args: - client (CloudvisionApi): Cloudvision connection. + client (CloudvisionApi): CloudVision connection. dId (str): Device ID to determine type for. """ pathElts = ["Sysdb", "interface", "status", "eth", "phy", "slice", "1", "intfStatus", Wildcard()] @@ -534,7 +521,7 @@ def get_interface_transceiver(client: CloudvisionApi, dId: str, interface: str): """Gets transceiver information for specified interface on specific device. Args: - client (CloudvisionApi): Cloudvision connection. + client (CloudvisionApi): CloudVision connection. dId (str): Device ID to determine transceiver type for. interface (str): Name of interface to get transceiver information for. """ @@ -559,7 +546,7 @@ def get_interface_mode(client: CloudvisionApi, dId: str, interface: str): """Gets interface mode, ie access/trunked. Args: - client (CloudvisionApi): Cloudvision connection. + client (CloudvisionApi): CloudVision connection. dId (str): Device ID to determine type for. interface (str): Name of interface to get mode information for. """ @@ -625,7 +612,7 @@ def get_interface_description(client: CloudvisionApi, dId: str, interface: str): """Gets interface description. Args: - client (CloudvisionApi): Cloudvision connection. + client (CloudvisionApi): CloudVision connection. dId (str): Device ID to get description for. interface (str): Name of interface to get description for. """ @@ -644,7 +631,7 @@ def get_interface_vrf(client: CloudvisionApi, dId: str, interface: str) -> str: """Gets interface VRF. Args: - client (CloudvisionApi): Cloudvision connection. + client (CloudvisionApi): CloudVision connection. dId (str): Device ID to determine type for. interface (str): Name of interface to get mode information for. """ @@ -663,7 +650,7 @@ def get_ip_interfaces(client: CloudvisionApi, dId: str): """Gets interfaces with IP Addresses configured from specified device. Args: - client (CloudvisionApi): Cloudvision connection. + client (CloudvisionApi): CloudVision connection. dId (str): Device ID to retrieve IP Addresses and associated interfaces for. """ pathElts = ["Sysdb", "ip", "config", "ipIntfConfig", Wildcard()] @@ -686,7 +673,7 @@ def get_ip_interfaces(client: CloudvisionApi, dId: str): return ip_intfs -def get_cvp_version(): +def get_cvp_version(config: CloudVisionAppConfig): """Returns CloudVision portal version. Returns: @@ -694,19 +681,19 @@ def get_cvp_version(): """ client = CvpClient() try: - if APP_SETTINGS.get("aristacv_cvp_token") and not APP_SETTINGS.get("aristacv_cvp_host"): + if config.token and not config.is_on_premise: client.connect( - nodes=[APP_SETTINGS["aristacv_cvaas_url"]], + nodes=[config.url], username="", password="", # nosec: B106 is_cvaas=True, - api_token=APP_SETTINGS.get("aristacv_cvp_token"), + api_token=config.token, ) else: client.connect( - nodes=[APP_SETTINGS["aristacv_cvp_host"]], - username=APP_SETTINGS.get("aristacv_cvp_user"), - password=APP_SETTINGS.get("aristacv_cvp_password"), + nodes=[config.url], + username=config.cvp_user, + password=config.cvp_password, is_cvaas=False, ) except CvpLoginError as err: diff --git a/nautobot_ssot/integrations/aristacv/utils/nautobot.py b/nautobot_ssot/integrations/aristacv/utils/nautobot.py index f6378ea57..52de5a00a 100644 --- a/nautobot_ssot/integrations/aristacv/utils/nautobot.py +++ b/nautobot_ssot/integrations/aristacv/utils/nautobot.py @@ -1,10 +1,32 @@ """Utility functions for Nautobot ORM.""" + +import logging import re +from typing import Mapping +from urllib.parse import urlparse + +from django.conf import settings from django.contrib.contenttypes.models import ContentType -from nautobot.dcim.models import Device, DeviceType, Location, LocationType, Manufacturer -from nautobot.extras.models import Role, Status, Tag, Relationship +from nautobot.core.models.utils import slugify +from nautobot.core.settings_funcs import is_truthy +from nautobot.dcim.models import Device +from nautobot.dcim.models import DeviceType +from nautobot.dcim.models import Location +from nautobot.dcim.models import LocationType +from nautobot.dcim.models import Manufacturer +from nautobot.extras.choices import SecretsGroupAccessTypeChoices +from nautobot.extras.choices import SecretsGroupSecretTypeChoices +from nautobot.extras.models import ExternalIntegration +from nautobot.extras.models import Relationship +from nautobot.extras.models import Role +from nautobot.extras.models import Secret +from nautobot.extras.models import SecretsGroup +from nautobot.extras.models import SecretsGroupAssociation +from nautobot.extras.models import Status +from nautobot.extras.models import Tag -from nautobot_ssot.integrations.aristacv.constant import APP_SETTINGS +from nautobot_ssot.integrations.aristacv import constants +from nautobot_ssot.integrations.aristacv.types import CloudVisionAppConfig try: from nautobot_device_lifecycle_mgmt.models import SoftwareLCM # noqa: F401 # pylint: disable=unused-import @@ -15,6 +37,168 @@ LIFECYCLE_MGMT = False +logger = logging.getLogger(__name__) + + +def _get_or_create_integration(integration_name: str, config: dict) -> ExternalIntegration: + slugified_integration_name = slugify(integration_name) + integration_env_name = slugified_integration_name.upper().replace("-", "_") + + integration, created = ExternalIntegration.objects.get_or_create( + name=integration_name, + defaults={ + "remote_url": config.pop("url"), + "verify_ssl": config.pop("verify_ssl", False), + "extra_config": config, + }, + ) + if not created: + return integration + + secrets_group = SecretsGroup.objects.create(name=f"{slugified_integration_name}-group") + secret_token = Secret.objects.create( + name=f"{slugified_integration_name}-token", + provider="environment-variable", + parameters={"variable": f"{integration_env_name}_TOKEN"}, + ) + SecretsGroupAssociation.objects.create( + secret=secret_token, + secrets_group=secrets_group, + access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, + secret_type=SecretsGroupSecretTypeChoices.TYPE_TOKEN, + ) + secret_password = Secret.objects.create( + name=f"{slugified_integration_name}-password", + provider="environment-variable", + parameters={"variable": f"{integration_env_name}_PASSWORD"}, + ) + SecretsGroupAssociation.objects.create( + secret=secret_password, + secrets_group=secrets_group, + access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, + secret_type=SecretsGroupSecretTypeChoices.TYPE_PASSWORD, + ) + secret_user = Secret.objects.create( + name=f"{slugified_integration_name}-user", + provider="environment-variable", + parameters={"variable": f"{integration_env_name}_USER"}, + ) + SecretsGroupAssociation.objects.create( + secret=secret_user, + secrets_group=secrets_group, + access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, + secret_type=SecretsGroupSecretTypeChoices.TYPE_USERNAME, + ) + integration.secrets_group = secrets_group + integration.validated_save() + return integration + + +def get_config() -> CloudVisionAppConfig: + """Get Arista CloudVision configuration from Nautobot settings. + + Reads configuration from external integration if specified by `aristacv_external_integration_name` app configuration. + + Keeps backward compatibility with previous configuration settings. + + Create a new integration if specified but not found. + """ + app_settings: dict = settings.PLUGINS_CONFIG["nautobot_ssot"] # type: ignore + + config = { + "is_on_premise": bool(app_settings.get("aristacv_cvp_host")), + "delete_devices_on_sync": is_truthy( + app_settings.get("aristacv_delete_devices_on_sync", constants.DEFAULT_DELETE_DEVICES_ON_SYNC) + ), + "from_cloudvision_default_site": app_settings.get( + "aristacv_from_cloudvision_default_site", constants.DEFAULT_SITE + ), + "from_cloudvision_default_device_role": app_settings.get( + "aristacv_from_cloudvision_default_device_role", constants.DEFAULT_DEVICE_ROLE + ), + "from_cloudvision_default_device_role_color": app_settings.get( + "aristacv_from_cloudvision_default_device_role_color", constants.DEFAULT_DEVICE_ROLE_COLOR + ), + "apply_import_tag": is_truthy( + app_settings.get("aristacv_apply_import_tag", constants.DEFAULT_APPLY_IMPORT_TAG) + ), + "import_active": is_truthy(app_settings.get("aristacv_import_active", constants.DEFAULT_IMPORT_ACTIVE)), + "verify_ssl": is_truthy(app_settings.get("aristacv_verify", constants.DEFAULT_VERIFY_SSL)), + "token": app_settings.get("aristacv_cvp_token", ""), + "cvp_user": app_settings.get("aristacv_cvp_user", ""), + "cvp_password": app_settings.get("aristacv_cvp_password", ""), + "hostname_patterns": app_settings.get("aristacv_hostname_patterns", []), + "site_mappings": app_settings.get("aristacv_site_mappings", {}), + "role_mappings": app_settings.get("aristacv_role_mappings", {}), + "controller_site": app_settings.get("aristacv_controller_site", ""), + "create_controller": is_truthy( + app_settings.get("aristacv_create_controller", constants.DEFAULT_CREATE_CONTROLLER) + ), + } + + if config["is_on_premise"]: + url = app_settings.get("aristacv_cvp_host", "") + if not url.startswith("http"): + url = f"https://{url}" + parsed_url = urlparse(url) + port = parsed_url.port or app_settings.get("aristacv_cvp_port", 443) + config["url"] = f"{parsed_url.scheme}://{parsed_url.hostname}:{port}" + else: + url = app_settings.get("aristacv_cvaas_url", constants.DEFAULT_CVAAS_URL) + if not url.startswith("http"): + url = f"https://{url}" + parsed_url = urlparse(url) + config["url"] = f"{parsed_url.scheme}://{parsed_url.hostname}:{parsed_url.port or 443}" + + def convert(): + expected_fields = set(CloudVisionAppConfig._fields) + for key in list(config): + if key not in expected_fields: + logger.warning(f"Unexpected key found in Arista CloudVision config: {key}") + config.pop(key) + + for key in expected_fields - set(config): + logger.warning(f"Missing key in Arista CloudVision config: {key}") + config[key] = "" + + return CloudVisionAppConfig(**config) + + integration_name = app_settings.get("aristacv_external_integration_name") + if not integration_name: + return convert() + + integration = _get_or_create_integration(integration_name, {**config}) + integration_config: Mapping = integration.extra_config # type: ignore + if not isinstance(integration.extra_config, Mapping): + integration_config = config + + if isinstance(integration.verify_ssl, bool): + config["verify_ssl"] = integration.verify_ssl + + config["url"] = integration.remote_url + + config.update(integration_config) + + secrets_group: SecretsGroup = integration.secrets_group # type: ignore + if not secrets_group: + return convert() + + config["cvp_user"] = secrets_group.get_secret_value( + access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, + secret_type=SecretsGroupSecretTypeChoices.TYPE_USERNAME, + ) + config["cvp_password"] = secrets_group.get_secret_value( + access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, + secret_type=SecretsGroupSecretTypeChoices.TYPE_PASSWORD, + ) + config["token"] = secrets_group.get_secret_value( + access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, + secret_type=SecretsGroupSecretTypeChoices.TYPE_TOKEN, + ) + + return convert() + + def verify_site(site_name): """Verifies whether site in app config is created. If not, creates site. @@ -26,12 +210,12 @@ def verify_site(site_name): try: site_obj = Location.objects.get(name=site_name, location_type=loc_type) except Location.DoesNotExist: - site_obj = Location( + status, _ = Status.objects.get_or_create(name="Staging") + site_obj = Location.objects.create( name=site_name, - status=Status.objects.get(name="Staging"), + status=status, location_type=loc_type, ) - site_obj.validated_save() return site_obj @@ -39,7 +223,7 @@ def verify_device_type_object(device_type): """Verifies whether device type object already exists in Nautobot. If not, creates specified device type. Args: - device_type (str): Device model gathered from Cloudvision. + device_type (str): Device model gathered from CloudVision. """ try: device_type_obj = DeviceType.objects.get(model=device_type) @@ -101,14 +285,12 @@ def get_device_version(device): return version -def parse_hostname(hostname: str): +def parse_hostname(hostname: str, hostname_patterns: list): """Parse a device's hostname to find site and role. Args: hostname (str): Device hostname to be parsed for site and role. """ - hostname_patterns = APP_SETTINGS.get("aristacv_hostname_patterns") - site, role = None, None for pattern in hostname_patterns: match = re.search(pattern=pattern, string=hostname) @@ -118,35 +300,3 @@ def parse_hostname(hostname: str): if "role" in match.groupdict() and match.group("role"): role = match.group("role") return (site, role) - - -def get_site_from_map(site_code: str): - """Get name of Site from site_mapping based upon sitecode. - - Args: - site_code (str): Site code from device hostname. - - Returns: - str|None: Name of Site if site code found else None. - """ - site_map = APP_SETTINGS.get("aristacv_site_mappings") - site_name = None - if site_code in site_map: - site_name = site_map[site_code] - return site_name - - -def get_role_from_map(role_code: str): - """Get name of Role from role_mapping based upon role code in hostname. - - Args: - role_code (str): Role code from device hostname. - - Returns: - str|None: Name of Device Role if role code found else None. - """ - role_map = APP_SETTINGS.get("aristacv_role_mappings") - role_name = None - if role_code in role_map: - role_name = role_map[role_code] - return role_name diff --git a/nautobot_ssot/jobs/examples.py b/nautobot_ssot/jobs/examples.py index 3d3a1baa9..66e8ed163 100644 --- a/nautobot_ssot/jobs/examples.py +++ b/nautobot_ssot/jobs/examples.py @@ -3,26 +3,31 @@ # Skip colon check for multiple statements on one line. # flake8: noqa: E701 -from typing import Optional, Mapping, List +from typing import List +from typing import Mapping +from typing import Optional from uuid import UUID + +import requests +from diffsync import DiffSync +from diffsync.enum import DiffSyncFlags from django.templatetags.static import static from django.urls import reverse - -from nautobot.dcim.models import Location, LocationType -from nautobot.extras.choices import SecretsGroupAccessTypeChoices, SecretsGroupSecretTypeChoices -from nautobot.extras.jobs import ObjectVar, StringVar +from nautobot.dcim.models import Location +from nautobot.dcim.models import LocationType +from nautobot.extras.choices import SecretsGroupAccessTypeChoices +from nautobot.extras.choices import SecretsGroupSecretTypeChoices +from nautobot.extras.jobs import ObjectVar +from nautobot.extras.jobs import StringVar from nautobot.extras.models import ExternalIntegration from nautobot.ipam.models import Prefix from nautobot.tenancy.models import Tenant -from diffsync import DiffSync -from diffsync.enum import DiffSyncFlags - -import requests - -from nautobot_ssot.contrib import NautobotModel, NautobotAdapter -from nautobot_ssot.jobs.base import DataMapping, DataSource, DataTarget - +from nautobot_ssot.contrib import NautobotAdapter +from nautobot_ssot.contrib import NautobotModel +from nautobot_ssot.jobs.base import DataMapping +from nautobot_ssot.jobs.base import DataSource +from nautobot_ssot.jobs.base import DataTarget # In a more complex Job, you would probably want to move the DiffSyncModel subclasses into a separate Python module(s). @@ -415,9 +420,16 @@ def data_mappings(cls): DataMapping("Prefix (remote)", None, "Prefix (local)", reverse("ipam:prefix_list")), ) - def run( - self, dryrun, memory_profiling, source, source_url, source_token, *args, **kwargs - ): # pylint:disable=arguments-differ + def run( # pylint: disable=too-many-arguments, arguments-differ + self, + dryrun, + memory_profiling, + source, + source_url, + source_token, + *args, + **kwargs, + ): """Run sync.""" self.dryrun = dryrun self.memory_profiling = memory_profiling @@ -433,9 +445,9 @@ def run( else: self.source_url = source_url self.source_token = source_token - except Exception as e: + except Exception as error: # TBD: Why are these exceptions swallowed? - self.logger.error(f"Error setting up job: {e}") + self.logger.error("Error setting up job: %s", error) raise super().run(dryrun, memory_profiling, *args, **kwargs) diff --git a/nautobot_ssot/tests/aristacv/test_cloudvision_adapter.py b/nautobot_ssot/tests/aristacv/test_cloudvision_adapter.py index e2229e418..9c64bd92b 100644 --- a/nautobot_ssot/tests/aristacv/test_cloudvision_adapter.py +++ b/nautobot_ssot/tests/aristacv/test_cloudvision_adapter.py @@ -1,9 +1,10 @@ -"""Unit tests for the Cloudvision DiffSync adapter class.""" +"""Unit tests for the CloudVision DiffSync adapter class.""" import ipaddress from unittest.mock import MagicMock, patch -from nautobot.extras.models import JobResult from nautobot.core.testing import TransactionTestCase +from nautobot.extras.models import JobResult + from nautobot_ssot.integrations.aristacv.diffsync.adapters.cloudvision import ( CloudvisionAdapter, ) @@ -49,12 +50,10 @@ def setUp(self): ) self.cvp = CloudvisionAdapter(job=self.job, conn=self.client) - @patch.dict( - "nautobot_ssot.integrations.aristacv.constant.APP_SETTINGS", - {"aristacv_create_controller": False}, - ) def test_load_devices(self): """Test the load_devices() adapter method.""" + # Update config namedtuple `create_controller` to False + self.job.app_config = self.job.app_config._replace(create_controller=False) with patch( "nautobot_ssot.integrations.aristacv.utils.cloudvision.get_devices", self.cloudvision.get_devices, diff --git a/nautobot_ssot/tests/aristacv/test_jobs.py b/nautobot_ssot/tests/aristacv/test_jobs.py index ab89b4f48..b3bb87ce3 100644 --- a/nautobot_ssot/tests/aristacv/test_jobs.py +++ b/nautobot_ssot/tests/aristacv/test_jobs.py @@ -1,6 +1,5 @@ -"""Test Cloudvision Jobs.""" -from unittest.mock import patch - +"""Test CloudVision Jobs.""" +from django.test import override_settings from django.urls import reverse from nautobot.core.testing import TestCase @@ -8,13 +7,13 @@ class CloudVisionDataSourceJobTest(TestCase): - """Test the Cloudvision DataSource Job.""" + """Test the CloudVision DataSource Job.""" def test_metadata(self): """Verify correctness of the Job Meta attributes.""" self.assertEqual("CloudVision ⟹ Nautobot", jobs.CloudVisionDataSource.name) self.assertEqual("CloudVision ⟹ Nautobot", jobs.CloudVisionDataSource.Meta.name) - self.assertEqual("Cloudvision", jobs.CloudVisionDataSource.data_source) + self.assertEqual("CloudVision", jobs.CloudVisionDataSource.data_source) self.assertEqual("Sync system tag data from CloudVision to Nautobot", jobs.CloudVisionDataSource.description) def test_data_mapping(self): # pylint: disable=too-many-statements @@ -101,53 +100,55 @@ def test_data_mapping(self): # pylint: disable=too-many-statements self.assertEqual("Topology Type", mappings[15].target_name) self.assertIsNone(mappings[15].target_url) - @patch.dict( - "nautobot_ssot.integrations.aristacv.constant.APP_SETTINGS", - { - "aristacv_cvp_host": "https://localhost", - "aristacv_cvp_user": "admin", - "aristacv_verify": True, - "aristacv_delete_devices_on_sync": True, - "aristacv_from_cloudvision_default_site": "HQ", - "aristacv_from_cloudvision_default_device_role": "Router", - "aristacv_from_cloudvision_default_device_role_color": "ff0000", - "aristacv_apply_import_tag": True, - "aristacv_import_active": True, + @override_settings( + PLUGINS_CONFIG={ + "nautobot_ssot": { + "aristacv_cvp_host": "https://localhost", + "aristacv_cvp_user": "admin", + "aristacv_verify": True, + "aristacv_delete_devices_on_sync": True, + "aristacv_from_cloudvision_default_site": "HQ", + "aristacv_from_cloudvision_default_device_role": "Router", + "aristacv_from_cloudvision_default_device_role_color": "ff0000", + "aristacv_apply_import_tag": True, + "aristacv_import_active": True, + }, }, ) def test_config_information_on_prem(self): """Verify the config_information() API for on-prem.""" config_information = jobs.CloudVisionDataSource.config_information() - self.assertEqual(config_information["Server type"], "On prem") - self.assertEqual(config_information["CloudVision host"], "https://localhost") - self.assertEqual(config_information["Username"], "admin") - self.assertEqual(config_information["Verify"], "True") - self.assertEqual(config_information["Delete devices on sync"], True) - self.assertEqual(config_information["New device default site"], "HQ") - self.assertEqual(config_information["New device default role"], "Router") - self.assertEqual(config_information["New device default role color"], "ff0000") - self.assertEqual(config_information["Apply import tag"], "True") + self.assertEqual(config_information["Server Type"], "On prem") + self.assertEqual(config_information["CloudVision URL"], "https://localhost:443") + self.assertEqual(config_information["Verify SSL"], "True") + self.assertEqual(config_information["User Name"], "admin") + self.assertEqual(config_information["Delete Devices On Sync"], True) + self.assertEqual(config_information["New Device Default Site"], "HQ") + self.assertEqual(config_information["New Device Default Role"], "Router") + self.assertEqual(config_information["New Device Default Role Color"], "ff0000") + self.assertEqual(config_information["Apply Import Tag"], "True") self.assertEqual(config_information["Import Active"], "True") - @patch.dict( - "nautobot_ssot.integrations.aristacv.constant.APP_SETTINGS", - { - "aristacv_cvaas_url": "https://www.arista.io", - "aristacv_cvp_user": "admin", + @override_settings( + PLUGINS_CONFIG={ + "nautobot_ssot": { + "aristacv_cvaas_url": "https://www.arista.io", + "aristacv_cvp_user": "admin", + }, }, ) def test_config_information_cvaas(self): """Verify the config_information() API for CVaaS.""" config_information = jobs.CloudVisionDataSource.config_information() - self.assertEqual(config_information["Server type"], "CVaaS") - self.assertEqual(config_information["CloudVision host"], "https://www.arista.io") - self.assertEqual(config_information["Username"], "admin") + self.assertEqual(config_information["Server Type"], "CVaaS") + self.assertEqual(config_information["CloudVision URL"], "https://www.arista.io:443") + self.assertEqual(config_information["User Name"], "admin") class CloudVisionDataTargetJobTest(TestCase): - """Test the Cloudvision DataTarget Job.""" + """Test the CloudVision DataTarget Job.""" def test_metadata(self): """Verify correctness of the Job Meta attributes.""" diff --git a/nautobot_ssot/tests/aristacv/test_utils_cloudvision.py b/nautobot_ssot/tests/aristacv/test_utils_cloudvision.py index 64dddc022..fe325032f 100644 --- a/nautobot_ssot/tests/aristacv/test_utils_cloudvision.py +++ b/nautobot_ssot/tests/aristacv/test_utils_cloudvision.py @@ -1,48 +1,62 @@ -"""Tests of Cloudvision utility methods.""" -from unittest.mock import MagicMock, patch -from parameterized import parameterized +"""Tests of CloudVision utility methods.""" + +from unittest.mock import MagicMock +from unittest.mock import patch -from nautobot.core.testing import TestCase from cloudvision.Connector.codec.custom_types import FrozenDict +from django.test import override_settings +from nautobot.core.testing import TestCase +from parameterized import parameterized from nautobot_ssot.integrations.aristacv.utils import cloudvision +from nautobot_ssot.integrations.aristacv.utils.nautobot import get_config from nautobot_ssot.tests.aristacv.fixtures import fixtures class TestCloudvisionApi(TestCase): - """Test Cloudvision Api client and methods.""" + """Test CloudVision Api client and methods.""" databases = ("default", "job_logs") + @override_settings( + PLUGINS_CONFIG={ + "nautobot_ssot": { + "aristacv_cvp_host": "localhost", + "aristacv_verify": True, + }, + }, + ) def test_auth_failure_exception(self): """Test that AuthFailure is thrown when no credentials are passed.""" + config = get_config() with self.assertRaises(cloudvision.AuthFailure): - cloudvision.CloudvisionApi(cvp_host="https://localhost", username="", password="", verify=True) # nosec - - @patch.dict( - "nautobot_ssot.integrations.aristacv.constant.APP_SETTINGS", - {"aristacv_cvaas_url": "www.arista.io:443"}, + cloudvision.CloudvisionApi(config) # nosec + + @override_settings( + PLUGINS_CONFIG={ + "nautobot_ssot": { + "aristacv_cvaas_url": "www.arista.io:443", + "aristacv_cvp_token": "1234567890abcdef", + }, + }, ) def test_auth_cvass_with_token(self): """Test that authentication against CVaaS with token works.""" - client = cloudvision.CloudvisionApi(cvp_host=None, cvp_token="1234567890abcdef") # nosec - self.assertEqual(client.cvp_url, "www.arista.io:443") - self.assertEqual(client.cvp_token, "1234567890abcdef") + config = get_config() + cloudvision.CloudvisionApi(config) + self.assertEqual(config.url, "https://www.arista.io:443") + self.assertEqual(config.token, "1234567890abcdef") class TestCloudvisionUtils(TestCase): - """Test Cloudvision utility methods.""" + """Test CloudVision utility methods.""" databases = ("default", "job_logs") def setUp(self): - """Setup mock Cloudvision client.""" + """Setup mock CloudVision client.""" self.client = MagicMock() - @patch.dict( - "nautobot_ssot.integrations.aristacv.constant.APP_SETTINGS", - {"aristacv_import_active": False}, - ) def test_get_all_devices(self): """Test get_devices function for active and inactive devices.""" device1 = MagicMock() @@ -69,14 +83,10 @@ def test_get_all_devices(self): device_svc_stub.DeviceServiceStub.return_value.GetAll.return_value = device_list with patch("nautobot_ssot.integrations.aristacv.utils.cloudvision.services", device_svc_stub): - results = cloudvision.get_devices(client=self.client) + results = cloudvision.get_devices(client=self.client, import_active=False) expected = fixtures.DEVICE_FIXTURE self.assertEqual(results, expected) - @patch.dict( - "nautobot_ssot.integrations.aristacv.constant.APP_SETTINGS", - {"aristacv_import_active": True}, - ) def test_get_active_devices(self): """Test get_devices function for active devices.""" device1 = MagicMock() @@ -94,7 +104,7 @@ def test_get_active_devices(self): device_svc_stub.DeviceServiceStub.return_value.GetAll.return_value = device_list with patch("nautobot_ssot.integrations.aristacv.utils.cloudvision.services", device_svc_stub): - results = cloudvision.get_devices(client=self.client) + results = cloudvision.get_devices(client=self.client, import_active=True) expected = [ { "device_id": "JPE12345678", diff --git a/nautobot_ssot/tests/aristacv/test_utils_nautobot.py b/nautobot_ssot/tests/aristacv/test_utils_nautobot.py index 070169d05..61cc3a0dd 100644 --- a/nautobot_ssot/tests/aristacv/test_utils_nautobot.py +++ b/nautobot_ssot/tests/aristacv/test_utils_nautobot.py @@ -1,9 +1,12 @@ -"""Tests of Cloudvision utility methods.""" +"""Tests of CloudVision utility methods.""" from unittest import skip from unittest.mock import MagicMock, patch + +from django.test import override_settings +from nautobot.core.testing import TestCase from nautobot.dcim.models import DeviceType, Location, LocationType, Manufacturer from nautobot.extras.models import Relationship, Role, Status, Tag -from nautobot.core.testing import TestCase + from nautobot_ssot.integrations.aristacv.utils import nautobot @@ -102,99 +105,53 @@ def test_get_device_version_dlc_exception(self): result = nautobot.get_device_version(mock_device) self.assertEqual(result, "1.0") - @patch.dict( - "nautobot_ssot.integrations.aristacv.constant.APP_SETTINGS", - { - "aristacv_hostname_patterns": [r"(?P\w{2,3}\d+)-(?P\w+)-\d+"], - "aristacv_site_mappings": {"ams01": "Amsterdam"}, - "aristacv_role_mappings": {"leaf": "leaf"}, + @override_settings( + PLUGINS_CONFIG={ + "nautobot_ssot": { + "aristacv_hostname_patterns": [r"(?P\w{2,3}\d+)-(?P\w+)-\d+"], + "aristacv_site_mappings": {"ams01": "Amsterdam"}, + "aristacv_role_mappings": {"leaf": "leaf"}, + }, }, ) def test_parse_hostname(self): """Test the parse_hostname method.""" + config = nautobot.get_config() host = "ams01-leaf-01" - results = nautobot.parse_hostname(host) + results = nautobot.parse_hostname(host, config.hostname_patterns) expected = ("ams01", "leaf") self.assertEqual(results, expected) - @patch.dict( - "nautobot_ssot.integrations.aristacv.constant.APP_SETTINGS", - { - "aristacv_hostname_patterns": [r"(?P\w{2,3}\d+)-.+-\d+"], - "aristacv_site_mappings": {"ams01": "Amsterdam"}, - "aristacv_role_mappings": {}, + @override_settings( + PLUGINS_CONFIG={ + "nautobot_ssot": { + "aristacv_hostname_patterns": [r"(?P\w{2,3}\d+)-.+-\d+"], + "aristacv_site_mappings": {"ams01": "Amsterdam"}, + "aristacv_role_mappings": {}, + }, }, ) def test_parse_hostname_only_site(self): """Test the parse_hostname method with only site specified.""" + config = nautobot.get_config() host = "ams01-leaf-01" - results = nautobot.parse_hostname(host) + results = nautobot.parse_hostname(host, config.hostname_patterns) expected = ("ams01", None) self.assertEqual(results, expected) - @patch.dict( - "nautobot_ssot.integrations.aristacv.constant.APP_SETTINGS", - { - "aristacv_hostname_patterns": [r".+-(?P\w+)-\d+"], - "aristacv_site_mappings": {}, - "aristacv_role_mappings": {"leaf": "leaf"}, + @override_settings( + PLUGINS_CONFIG={ + "nautobot_ssot": { + "aristacv_hostname_patterns": [r".+-(?P\w+)-\d+"], + "aristacv_site_mappings": {}, + "aristacv_role_mappings": {"leaf": "leaf"}, + }, }, ) def test_parse_hostname_only_role(self): """Test the parse_hostname method with only role specified.""" + config = nautobot.get_config() host = "ams01-leaf-01" - results = nautobot.parse_hostname(host) + results = nautobot.parse_hostname(host, config.hostname_patterns) expected = (None, "leaf") self.assertEqual(results, expected) - - @patch.dict( - "nautobot_ssot.integrations.aristacv.constant.APP_SETTINGS", - { - "aristacv_hostname_patterns": [r"(?P\w{2,3}\d+)-(?P\w+)-\d+"], - "aristacv_site_mappings": {"ams01": "Amsterdam"}, - }, - ) - def test_get_site_from_map_success(self): - """Test the get_site_from_map method with response.""" - results = nautobot.get_site_from_map("ams01") - expected = "Amsterdam" - self.assertEqual(results, expected) - - @patch.dict( - "nautobot_ssot.integrations.aristacv.constant.APP_SETTINGS", - { - "aristacv_hostname_patterns": [r"(?P\w{2,3}\d+)-(?P\w+)-\d+"], - "aristacv_site_mappings": {}, - }, - ) - def test_get_site_from_map_fail(self): - """Test the get_site_from_map method with failed response.""" - results = nautobot.get_site_from_map("dc01") - expected = None - self.assertEqual(results, expected) - - @patch.dict( - "nautobot_ssot.integrations.aristacv.constant.APP_SETTINGS", - { - "aristacv_hostname_patterns": [r"(?P\w{2,3}\d+)-(?P\w+)-\d+"], - "aristacv_role_mappings": {"edge": "Edge Router"}, - }, - ) - def test_get_role_from_map_success(self): - """Test the get_role_from_map method with response.""" - results = nautobot.get_role_from_map("edge") - expected = "Edge Router" - self.assertEqual(results, expected) - - @patch.dict( - "nautobot_ssot.integrations.aristacv.constant.APP_SETTINGS", - { - "aristacv_hostname_patterns": [r"(?P\w{2,3}\d+)-(?P\w+)-\d+"], - "aristacv_role_mappings": {}, - }, - ) - def test_get_role_from_map_fail(self): - """Test the get_role_from_map method with failed response.""" - results = nautobot.get_role_from_map("rtr") - expected = None - self.assertEqual(results, expected) From 782118299233a1aa4109b21b3a373ac3fd5a1309 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Mon, 25 Mar 2024 15:18:36 +0000 Subject: [PATCH 5/7] cleanup --- nautobot_ssot/integrations/aristacv/jobs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nautobot_ssot/integrations/aristacv/jobs.py b/nautobot_ssot/integrations/aristacv/jobs.py index 58fc028fa..efbb63018 100644 --- a/nautobot_ssot/integrations/aristacv/jobs.py +++ b/nautobot_ssot/integrations/aristacv/jobs.py @@ -50,8 +50,6 @@ def config_information(cls): """Dictionary describing the configuration of this DataSource.""" config = get_config() - print(100 * "A") - print(config) return { "Server Type": "On prem" if config.is_on_premise else "CVaaS", "CloudVision URL": config.url, From d66903cb4b5647670998b9343f5c8305285a4635 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Thu, 4 Apr 2024 08:58:23 +0000 Subject: [PATCH 6/7] feat: Demo job script --- development/run_example_job.py | 89 ++++++++++++++++++++++++++++++++++ tasks.py | 78 ++++++++++++++++++++--------- 2 files changed, 143 insertions(+), 24 deletions(-) create mode 100644 development/run_example_job.py diff --git a/development/run_example_job.py b/development/run_example_job.py new file mode 100644 index 000000000..ebbeabb6b --- /dev/null +++ b/development/run_example_job.py @@ -0,0 +1,89 @@ +"""Executes a job locally for testing purposes. + +To run this script use the following command: + +``` +invoke nbshell \ + --plain \ + --file development/run_example_job.py \ + --env RUN_SSOT_TARGET_JOB=False \ + --env RUN_SSOT_JOB_DRY_RUN=True +``` + +Passing environment variables to the script is optional. The script will default to running the data source job with a dry run enabled. +""" + +import json +import os + +from django.core.management import call_command +from nautobot.core.settings_funcs import is_truthy +from nautobot.extras.choices import SecretsGroupAccessTypeChoices +from nautobot.extras.choices import SecretsGroupSecretTypeChoices +from nautobot.extras.models import ExternalIntegration +from nautobot.extras.models import Job +from nautobot.extras.models import Secret +from nautobot.extras.models import SecretsGroup +from nautobot.extras.models import SecretsGroupAssociation + +_TOKEN = 40 * "a" +os.environ["NAUTOBOT_DEMO_TOKEN"] = _TOKEN + +_NAUTOBOT_DEMO_URL = "https://demo.nautobot.com" +_DRY_RUN = is_truthy(os.getenv("RUN_SSOT_JOB_DRY_RUN", "True")) + +module_name = "nautobot_ssot.jobs.examples" +is_target_job = is_truthy(os.getenv("RUN_SSOT_TARGET_JOB", "False")) +job_class_name = "ExampleDataTarget" if is_target_job else "ExampleDataSource" + +job = Job.objects.get(module_name=module_name, job_class_name=job_class_name) +if not job.enabled: + job.enabled = True + job.validated_save() + +nautobot_demo, created = ExternalIntegration.objects.get_or_create( + name="Nautobot Demo", + defaults={ + "remote_url": _NAUTOBOT_DEMO_URL, + "verify_ssl": False, + }, +) + +if created: + secret = Secret.objects.create( + name="nautobot-demo-token", + provider="environment-variable", + parameters={"variable": "NAUTOBOT_DEMO_TOKEN"}, + ) + secrets_group = SecretsGroup.objects.create(name="Nautobot Demo Group") + SecretsGroupAssociation.objects.create( + secret=secret, + secrets_group=secrets_group, + access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, + secret_type=SecretsGroupSecretTypeChoices.TYPE_TOKEN, + ) + nautobot_demo.secrets_group = secrets_group + nautobot_demo.validated_save() + +data: dict = { + "debug": True, + "dryrun": _DRY_RUN, + "memory_profiling": False, +} + +if is_target_job: + data["target"] = str(nautobot_demo.pk) + data["target_url"] = _NAUTOBOT_DEMO_URL + data["target_token"] = _TOKEN +else: + data["source"] = str(nautobot_demo.pk) + data["source_url"] = _NAUTOBOT_DEMO_URL + data["source_token"] = _TOKEN + +call_command( + "runjob", + f"{module_name}.{job_class_name}", + data=json.dumps(data), + username="admin", + local=True, # Enable to run the job locally (not as a celery task) +) diff --git a/tasks.py b/tasks.py index bec9260aa..c54ee1b0e 100644 --- a/tasks.py +++ b/tasks.py @@ -88,6 +88,38 @@ def _await_healthy_container(context, container_id): sleep(1) +def _read_command_env(values) -> dict: + """Reads the environment variables from the values and returns a dictionary. + + Examples: + >>> _read_command_env('VAR1=VALUE1') + {'VAR1': 'VALUE1'} + os.environ["VAR2"] = "VALUE2" + >>> _read_command_env(['VAR1=VALUE1', 'VAR2', 'VAR3']) + {'VAR1': 'VALUE1', 'VAR2': 'VALUE2', 'VAR3': ''} + >>> _read_command_env({'VAR1': 'VALUE1', 'VAR2': 'ANOHTER_VALUE'}) + {'VAR1': 'VALUE1', 'VAR2': 'ANOHTER_VALUE'} + """ + if not values: + return {} + + if isinstance(values, dict): + return values + + def read(envs): + if isinstance(envs, str): + if "=" in envs: + name, value = envs.split("=") + yield name, value + else: + yield envs, os.getenv(envs, "") + else: + for env in envs: + yield from read(env) + + return dict(read(values)) + + def task(function=None, *args, **kwargs): """Task decorator to override the default Invoke task decorator and add each task to the invoke namespace.""" @@ -148,32 +180,26 @@ def docker_compose(context, command, **kwargs): def run_command(context, command, **kwargs): """Wrapper to run a command locally or inside the nautobot container.""" + env = _read_command_env(kwargs.pop("env", None)) if is_truthy(context.nautobot_ssot.local): - if "command_env" in kwargs: - kwargs["env"] = { - **kwargs.get("env", {}), - **kwargs.pop("command_env"), - } - context.run(command, **kwargs) + return context.run(command, **kwargs, env=env) + + # Check if nautobot is running, no need to start another nautobot container to run a command + docker_compose_status = "ps --services --filter status=running" + results = docker_compose(context, docker_compose_status, hide="out") + if "nautobot" in results.stdout: + compose_command = "exec" else: - # Check if nautobot is running, no need to start another nautobot container to run a command - docker_compose_status = "ps --services --filter status=running" - results = docker_compose(context, docker_compose_status, hide="out") - if "nautobot" in results.stdout: - compose_command = "exec" - else: - compose_command = "run --rm --entrypoint=''" + compose_command = "run --rm --entrypoint=''" - if "command_env" in kwargs: - command_env = kwargs.pop("command_env") - for key, value in command_env.items(): - compose_command += f' --env="{key}={value}"' + for env_name in env: + compose_command += f" --env={env_name}" - compose_command += f" -- nautobot {command}" + compose_command += f" -- nautobot {command}" - pty = kwargs.pop("pty", True) + pty = kwargs.pop("pty", True) - docker_compose(context, compose_command, pty=pty, **kwargs) + return docker_compose(context, compose_command, **kwargs, pty=pty, env=env) # ------------------------------------------------------------------------------ @@ -342,19 +368,23 @@ def logs(context, service="", follow=False, tail=0): @task( help={ "file": "Python file to execute", - "env": "Environment variables to pass to the command", + "env": "Environment variables to pass to the command e.g.: `--env VAR1=VALUE1 --env VAR2`", "plain": "Flag to run nbshell in plain mode (default: False)", }, + iterable=["env"], ) -def nbshell(context, file="", env={}, plain=False): - """Launch an interactive nbshell session.""" +def nbshell(context, file="", env=None, plain=False): + """Launch an interactive nbshell session. + + Files passed to the command can't contain unindented empty lines, as it breaks the nbshell interpreter. + """ command = [ "nautobot-server", "nbshell", "--plain" if plain else "", f"< '{file}'" if file else "", ] - run_command(context, " ".join(command), pty=not bool(file), command_env=env) + run_command(context, " ".join(command), pty=not bool(file), env=env) @task From c190f62c193af06e88cac9d8255e1c82afbd74fa Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Thu, 11 Apr 2024 08:34:19 +0200 Subject: [PATCH 7/7] Update nautobot_ssot/integrations/aristacv/utils/nautobot.py Co-authored-by: Justin Drew <2396364+jdrew82@users.noreply.github.com> --- nautobot_ssot/integrations/aristacv/utils/nautobot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nautobot_ssot/integrations/aristacv/utils/nautobot.py b/nautobot_ssot/integrations/aristacv/utils/nautobot.py index 52de5a00a..32642aac5 100644 --- a/nautobot_ssot/integrations/aristacv/utils/nautobot.py +++ b/nautobot_ssot/integrations/aristacv/utils/nautobot.py @@ -210,7 +210,9 @@ def verify_site(site_name): try: site_obj = Location.objects.get(name=site_name, location_type=loc_type) except Location.DoesNotExist: - status, _ = Status.objects.get_or_create(name="Staging") + status, created = Status.objects.get_or_create(name="Staging") + if created: + status.content_types.add(ContentType.objects.get_for_model(Location)) site_obj = Location.objects.create( name=site_name, status=status,