Update Integrations for Controllers/ExternalIntegration

development/development.env

@@ -55,7 +55,6 @@ NAUTOBOT_SSOT_ACI_TAG_DOWN_COLOR="FF3333"
 NAUTOBOT_SSOT_ACI_MANUFACTURER_NAME="Cisco"
 NAUTOBOT_SSOT_ACI_IGNORE_TENANTS="[mgmt,infra]"
 NAUTOBOT_SSOT_ACI_COMMENTS="Created by ACI SSoT Integration"
-NAUTOBOT_SSOT_ACI_SITE="Data Center"
 
 NAUTOBOT_SSOT_ENABLE_ARISTACV="False"
 NAUTOBOT_ARISTACV_CONTROLLER_SITE=""

development/nautobot_config.py

@@ -141,12 +141,8 @@
     #     "slack_api_token": os.getenv("SLACK_API_TOKEN"),
     #     "slack_signing_secret": os.getenv("SLACK_SIGNING_SECRET"),
     #     "session_cache_timeout": 3600,
-    #     "ipfabric_api_token": os.getenv("IPFABRIC_API_TOKEN"),
-    #     "ipfabric_host": os.getenv("IPFABRIC_HOST"),
     # },
     "nautobot_ssot": {
-        # URL and credentials should be configured as environment variables on the host system
-        "aci_apics": {x: os.environ[x] for x in os.environ if "APIC" in x},
         # Tag which will be created and applied to all synchronized objects.
         "aci_tag": os.getenv("NAUTOBOT_SSOT_ACI_TAG"),
         "aci_tag_color": os.getenv("NAUTOBOT_SSOT_ACI_TAG_COLOR"),
@@ -161,23 +157,21 @@
         "aci_ignore_tenants": os.getenv("NAUTOBOT_SSOT_ACI_IGNORE_TENANTS", "").split(","),
         # The below value will appear in the Comments field on objects created in Nautobot
         "aci_comments": os.getenv("NAUTOBOT_SSOT_ACI_COMMENTS"),
-        # Site to associate objects. Specify existing, or a new site with this name will be created.
-        "aci_site": os.getenv("NAUTOBOT_SSOT_ACI_SITE"),
-        "aristacv_apply_import_tag": is_truthy(os.getenv("NAUTOBOT_ARISTACV_IMPORT_TAG", False)),
+        "aristacv_apply_import_tag": is_truthy(os.getenv("NAUTOBOT_ARISTACV_IMPORT_TAG", "false")),
         "aristacv_controller_site": os.getenv("NAUTOBOT_ARISTACV_CONTROLLER_SITE", ""),
-        "aristacv_create_controller": is_truthy(os.getenv("NAUTOBOT_ARISTACV_CREATE_CONTROLLER", False)),
+        "aristacv_create_controller": is_truthy(os.getenv("NAUTOBOT_ARISTACV_CREATE_CONTROLLER", "false")),
         "aristacv_cvaas_url": os.getenv("NAUTOBOT_ARISTACV_CVAAS_URL", "www.arista.io:443"),
         "aristacv_cvp_host": os.getenv("NAUTOBOT_ARISTACV_CVP_HOST", ""),
         "aristacv_cvp_password": os.getenv("NAUTOBOT_ARISTACV_CVP_PASSWORD", ""),
         "aristacv_cvp_port": os.getenv("NAUTOBOT_ARISTACV_CVP_PORT", "443"),
         "aristacv_cvp_token": os.getenv("NAUTOBOT_ARISTACV_CVP_TOKEN", ""),
         "aristacv_cvp_user": os.getenv("NAUTOBOT_ARISTACV_CVP_USERNAME", ""),
-        "aristacv_delete_devices_on_sync": is_truthy(os.getenv("NAUTOBOT_ARISTACV_DELETE_ON_SYNC", False)),
+        "aristacv_delete_devices_on_sync": is_truthy(os.getenv("NAUTOBOT_ARISTACV_DELETE_ON_SYNC", "false")),
         "aristacv_from_cloudvision_default_device_role": "network",
         "aristacv_from_cloudvision_default_device_role_color": "ff0000",
         "aristacv_from_cloudvision_default_site": "cloudvision_imported",
         "aristacv_hostname_patterns": [r"(?P<site>\w{2,3}\d+)-(?P<role>\w+)-\d+"],
-        "aristacv_import_active": is_truthy(os.getenv("NAUTOBOT_ARISTACV_IMPORT_ACTIVE", False)),
+        "aristacv_import_active": is_truthy(os.getenv("NAUTOBOT_ARISTACV_IMPORT_ACTIVE", "false")),
         "aristacv_role_mappings": {
             "bb": "backbone",
             "edge": "edge",
@@ -190,7 +184,7 @@
             "ams01": "Amsterdam",
             "atl01": "Atlanta",
         },
-        "aristacv_verify": is_truthy(os.getenv("NAUTOBOT_ARISTACV_VERIFY", True)),
+        "aristacv_verify": is_truthy(os.getenv("NAUTOBOT_ARISTACV_VERIFY", "true")),
         "enable_aci": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_ACI")),
         "enable_aristacv": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_ARISTACV")),
         "enable_device42": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_DEVICE42")),
@@ -199,10 +193,6 @@
         "enable_itential": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_ITENTIAL")),
         "enable_servicenow": is_truthy(os.getenv("NAUTOBOT_SSOT_ENABLE_SERVICENOW")),
         "hide_example_jobs": is_truthy(os.getenv("NAUTOBOT_SSOT_HIDE_EXAMPLE_JOBS")),
-        "device42_host": os.getenv("NAUTOBOT_SSOT_DEVICE42_HOST", ""),
-        "device42_username": os.getenv("NAUTOBOT_SSOT_DEVICE42_USERNAME", ""),
-        "device42_password": os.getenv("NAUTOBOT_SSOT_DEVICE42_PASSWORD", ""),
-        "device42_verify_ssl": False,
         "device42_defaults": {
             "site_status": "Active",
             "rack_status": "Active",
@@ -230,13 +220,9 @@
         "infoblox_password": os.getenv("NAUTOBOT_SSOT_INFOBLOX_PASSWORD"),
         "infoblox_url": os.getenv("NAUTOBOT_SSOT_INFOBLOX_URL"),
         "infoblox_username": os.getenv("NAUTOBOT_SSOT_INFOBLOX_USERNAME"),
-        "infoblox_verify_ssl": is_truthy(os.getenv("NAUTOBOT_SSOT_INFOBLOX_VERIFY_SSL", True)),
+        "infoblox_verify_ssl": is_truthy(os.getenv("NAUTOBOT_SSOT_INFOBLOX_VERIFY_SSL", "true")),
         "infoblox_wapi_version": os.getenv("NAUTOBOT_SSOT_INFOBLOX_WAPI_VERSION", "v2.12"),
         "infoblox_network_view": os.getenv("NAUTOBOT_SSOT_INFOBLOX_NETWORK_VIEW", ""),
-        "ipfabric_api_token": os.getenv("NAUTOBOT_SSOT_IPFABRIC_API_TOKEN"),
-        "ipfabric_host": os.getenv("NAUTOBOT_SSOT_IPFABRIC_HOST"),
-        "ipfabric_ssl_verify": is_truthy(os.getenv("NAUTOBOT_SSOT_IPFABRIC_SSL_VERIFY", "False")),
-        "nautobot_host": os.getenv("NAUTOBOT_HOST"),
         "servicenow_instance": os.getenv("SERVICENOW_INSTANCE", ""),
         "servicenow_password": os.getenv("SERVICENOW_PASSWORD", ""),
         "servicenow_username": os.getenv("SERVICENOW_USERNAME", ""),

docs/admin/integrations/aci_setup.md

@@ -12,11 +12,13 @@ pip install nautobot-ssot[aci]
 
 ## Configuration
 
+!!! note
+    Legacy configuration settings defined in `nautobot_config.py` for `aci_apics` is now deprecated. All information related to communicating to an APIC has been updated to use the Controller and its related ExternalIntegration objects.
+
 Integration behavior can be controlled with the following settings:
 
 | Setting Name<br>(* required) | Type | Description |
 |---|:---:|---|
-| <p>aci_apics_*</p> |  | Per-APIC settings. See per-APIC settings section for details. |
 | aci_tag* | _String_ | Tag which is created and applied to all <br>synchronized objects. |
 | aci_tag_color* | _String_ | Hex color code used for the tag. |
 | aci_tag_up* | _String_ | Tag indicating the state applied to synchronized <br>interfaces. |
@@ -33,8 +35,6 @@ Below is an example snippet from `nautobot_config.py` that demonstrates how to e
 PLUGINS_CONFIG = {
     "nautobot_ssot": {
         "enable_aci": True,
-        # URL and credentials should be configured as environment variables on the host system
-        "aci_apics": {x: os.environ[x] for x in os.environ if "APIC" in x},
         # Tag which will be created and applied to all synchronized objects.
         "aci_tag": "ACI",
         "aci_tag_color": "0047AB",
@@ -54,36 +54,21 @@ PLUGINS_CONFIG = {
 
 ### Per-APIC settings
 
-The APIC URL and credentials need to be created as environment variables on the host system.
-
-You can configure multiple APIC instances for synchronization. To do this, append `_` character, followed by an identifier, to the names of environment variables.
-
-In the example below, configured APIC uses `NTC` for an identifier. Instead of `NTC` you could, for example, use `CHCG01` to configure an APIC instance in your Chicago facility.
-
-```bash
-export NAUTOBOT_APIC_BASE_URI_NTC=https://aci.cloud.networktocode.com
-export NAUTOBOT_APIC_USERNAME_NTC=admin
-export NAUTOBOT_APIC_PASSWORD_NTC=not_so_secret_password
-export NAUTOBOT_APIC_VERIFY_NTC=False
-export NAUTOBOT_APIC_SITE_NTC="NTC ACI"
-export NAUTOBOT_APIC_TENANT_PREFIX_NTC="NTC_ACI"
-```
-
-The identifier is used to select APIC from the SSoT dashboard when initiating a synchronization job:
+All APIC specific settings have been updated to use the Controller and related ExternalIntegration objects. The ExternalIntegration object that is assigned to the Controller will define the APIC base URL, user credentials, and SSL verification. It will also have a `tenant_prefix` key in the `extra_config` section of the ExternalIntegration to define the Tenant prefix.
 
-![image](../../images/aci-dashboard-apic.png)
+The `aci_apics` setting from the `nautobot_config.py` file is no longer used and any configuration found for it will be automatically migrated into a Controller and an ExternalIntegration object.
 
 ## Nautobot Objects Affected by Settings
 
-A Site will be created in Nautobot with the name specified in the `NAUTOBOT_APIC_SITE` environment variable and resources created by the integration will be assigned to this site.
+The Job form has been updated to allow specifying a Location for the imported objects. If that is left unspecified then the Location associated to the specified Controller will be used.
 
-Tenants imported from ACI will be prepended with the unique name specified by the corresponding `TENANT_PREFIX` variable. This uniquely identifies tenants which might have the same name, but belong to two different APIC clusters.
+Tenants imported from ACI will be prefixed with the unique name specified by the corresponding `tenant_prefix` key in the Controller's associated ExternalIntegration `extra_config`. This uniquely identifies tenants which might have the same name, but belong to two different APIC clusters.
 
 ## Configuring Device Templates
 
 To create a new Nautobot Device Type mapping to a specific ACI leaf or spine switch model you need to provide YAML file describing that model.  This model definition includes interface template with the ports and transceiver types (ex. 10GE SFP+) specification.
 
-The YAML files need to be placed in the `nautobot_ssot/integrations/aci/diffsync/device-types` directory. Their names need to match the model name as it appears in the ACI  Fabric Membership area of the APIC dashboard.
+The YAML files need to be placed in the `nautobot_ssot/integrations/aci/diffsync/device-types` directory. Their names need to match the model name as it appears in the ACI Fabric Membership area of the APIC dashboard.
 
 For example, given a Model name of `N9K-C9396PX` as shown below, the YAML file should be named `N9K-C9396PX.yaml`.
 
@@ -114,15 +99,13 @@ There are example YAML files for a few common switch models in `nautobot_ssot/in
     PLUGINS_CONFIG = {
         # "nautobot_ssot_aci": {  REMOVE THIS APP CONFIGURATION
         #     MOVE CONFIGURATION TO `nautobot_ssot` SECTION
-        #     "apics": {x: os.environ[x] for x in os.environ if "NAUTOBOT_APIC" in x},
         #     "tag": "ACI",
         #     ...
         # }
         "nautobot_ssot": {
             # Enable Cisco ACI integration
             "enable_aci": True,
             # Following lines are moved from `nautobot_ssot_aci` and prefixed with `aci_`
-            "aci_apics": {x: os.environ[x] for x in os.environ if "NAUTOBOT_APIC" in x},
             "aci_tag": "ACI",
             ...
         }
@@ -133,5 +116,4 @@ There are example YAML files for a few common switch models in `nautobot_ssot/in
     Configuration keys are prefixed with `aci_`.
 
 !!! note
-    Environment variables defining APICs access must contain `APIC`.
     Other environment variables for this integration are prefixed with `NAUTOBOT_SSOT_ACI_`.

docs/admin/integrations/device42_setup.md

@@ -12,16 +12,21 @@ pip install nautobot-ssot[device42]
 
 ## Configuration
 
-Integration behavior can be controlled with the following settings:
+Access to your Device42 instance is defined using the [ExternalIntegration](https://docs.nautobot.com/projects/core/en/stable/user-guide/platform-functionality/externalintegration/) model which allows you to utilize this integration with multiple instances concurrently. Please bear in mind that it will synchronize all data 1:1 with the specified instance to match exactly, meaning it will delete data missing from an instance, unless you have the `device42_delete_on_sync` option set to False.  Each ExternalIntegration must specify a SecretsGroup that contains Secrets that contain the Device42 Username and Password to authenticate against that instance. You can find Secrets and SecretsGroups available under the Secrets menu.
 
-| Configuration Variable | Type    | Usage                                                                                                 |
-| ---------------------- | ------- | ----------------------------------------------------------------------------------------------------- |
-| device42_host          | string  | This defines the FQDN of the Device42 instance, ie `https://device42.example.com`.                    |
-| device42_username      | string  | This defines the username of the account used to connect to the Device42 API endpoint.                |
-| device42_password      | string  | This defines the password of the account used to connect to the Device42 API endpoint.                |
-| device42_verify        | boolean | This denotes whether SSL validation of the Device42 endpoint should be enabled or not.                |
+![Device42 Username](../../images/device42_username.png)
 
-When creating Sites and Racks in Nautobot it is required to define a Status for each. It is also required to define a Role for your Device when created. You may define the default for each of those objects being imported with the respective values in your `nautobot_config.py` file.
+![Device42 Password](../../images/device42_password.png)
+
+![Device42 SecretsGroup](../../images/device42_secretsgroup.png)
+
+Please note that is it imperative for the SecretsGroup used with the specified Controller uses HTTP(S) Access type and Username and Password respectively for each Secret. Also note that the name of the Secrets or SecretsGroup are irrelevant but are recommended to be relevant to the instance in question.
+
+Once the SecretsGroup is created you'll need to create the ExternalIntegration. You'll find this under the Extensibility menu.
+
+![Device42 ExternalIntegration](../../images/device42_externalintegration.png)
+
+When creating Locations and Racks in Nautobot it is required to define a Status for each. It is also required to define a Role for your Device when created. You may define the default for each of those objects being imported with the respective values in your `nautobot_config.py` file.
 
 | Configuration Variable                              | Type   | Usage                                                      | Default              |
 | --------------------------------------------------- | ------ | ---------------------------------------------------------- | -------------------- |

docs/user/integrations/device42.md

@@ -32,7 +32,7 @@ From the Dashboard, you can also view more information about the App by clicking
 
 ![Detail View](../../images/device42_detail-view.png)
 
-To start the synchronization, simply click the `Sync Now` button on the Dashboard to start the Job. You should be presented with the Job form below:
+To start the synchronization, simply select the ExternalIntegration that corresponds to your desired instance and then click the `Sync Now` button on the Dashboard to start the Job. You should be presented with the Job form below:
 
 ![Job Form](../../images/device42_job-form.png)
 

nautobot_ssot/__init__.py

@@ -1,17 +1,17 @@
 """App declaration for nautobot_ssot."""
 
+import logging
 import os
 from importlib import metadata
 
-
+import packaging
 from django.conf import settings
-from nautobot.extras.plugins import NautobotAppConfig
 from nautobot.core.settings_funcs import is_truthy
-import packaging
+from nautobot.extras.plugins import NautobotAppConfig
 
 from nautobot_ssot.integrations.utils import each_enabled_integration_module
-from nautobot_ssot.utils import logger
 
+logger = logging.getLogger("nautobot.ssot")
 __version__ = metadata.version(__name__)
 
 
@@ -25,8 +25,7 @@
 ]
 
 _MIN_NAUTOBOT_VERSION = {
-    "nautobot_ssot_aristacv": "2.1",
-    "nautobot_ssot_infoblox": "2.1",
+    "nautobot_ssot_aci": "2.2",
 }
 
 
@@ -69,10 +68,9 @@ class NautobotSSOTAppConfig(NautobotAppConfig):
     description = "Nautobot app that enables Single Source of Truth.  Allows users to aggregate distributed data sources and/or distribute Nautobot data to other data sources such as databases and SDN controllers."
     base_url = "ssot"
     required_settings = []
-    min_version = "2.0.0"
+    min_version = "2.1.0"
     max_version = "2.9999"
     default_settings = {
-        "aci_apics": [],
         "aci_tag": "",
         "aci_tag_color": "",
         "aci_tag_up": "",

nautobot_ssot/integrations/aci/diffsync/adapters/aci.py

@@ -2,7 +2,6 @@
 
 # pylint: disable=duplicate-code
 
-
 import logging
 import os
 import re
@@ -20,7 +19,6 @@
 from nautobot_ssot.integrations.aci.diffsync.models import NautobotInterface
 from nautobot_ssot.integrations.aci.diffsync.models import NautobotIPAddress
 from nautobot_ssot.integrations.aci.diffsync.models import NautobotPrefix
-from nautobot_ssot.integrations.aci.diffsync.client import AciApi
 from nautobot_ssot.integrations.aci.diffsync.utils import load_yamlfile
 
 
@@ -52,7 +50,7 @@ class AciAdapter(Adapter):
         "ip_address",
     ]
 
-    def __init__(self, *args, job=None, sync=None, client, **kwargs):
+    def __init__(self, *args, job=None, sync=None, client, tenant_prefix, **kwargs):
         """Initialize ACI.
 
         Args:
@@ -63,15 +61,9 @@ def __init__(self, *args, job=None, sync=None, client, **kwargs):
         super().__init__(*args, **kwargs)
         self.job = job
         self.sync = sync
-        self.conn = AciApi(
-            username=client["username"],
-            password=client["password"],
-            base_uri=client["base_uri"],
-            verify=client["verify"],
-            site=client["site"],
-        )
-        self.site = client.get("site")
-        self.tenant_prefix = client.get("tenant_prefix")
+        self.conn = client
+        self.site = client.site
+        self.tenant_prefix = tenant_prefix
         self.nodes = self.conn.get_nodes()
         self.controllers = self.conn.get_controllers()
         self.nodes.update(self.controllers)
@@ -426,6 +418,11 @@ def load_devices(self):
                 pod_id=value["pod_id"],
                 site=self.site,
                 site_tag=self.site,
+                controller_group=(
+                    self.job.apic.controller_managed_device_group.name
+                    if self.job.apic.controller_managed_device_group
+                    else ""
+                ),
             )
             self.add(new_device)
 

nautobot_ssot/integrations/aci/diffsync/adapters/nautobot.py

@@ -54,20 +54,19 @@ class NautobotAdapter(Adapter):
         "ip_address",
     ]
 
-    def __init__(self, *args, job=None, sync=None, client, **kwargs):
+    def __init__(self, *args, job=None, sync=None, site_name: str, **kwargs):
         """Initialize Nautobot.
 
         Args:
             job (object, optional): Nautobot job. Defaults to None.
             sync (object, optional): Nautobot DiffSync. Defaults to None.
-            client (object): ACI credentials.
+            site_name (str): Name of Site to filter objects on.
         """
         super().__init__(*args, **kwargs)
         self.job = job
         self.sync = sync
-        self.site = client.get("site")
+        self.site = site_name
         self.site_tag = Tag.objects.get_or_create(name=self.site)[0]
-        self.tenant_prefix = client.get("tenant_prefix")
 
     def sync_complete(self, source: Adapter, *args, **kwargs):
         """Clean up function for DiffSync sync.
@@ -185,6 +184,9 @@ def load_devices(self):
                 node_id=nbdevice.custom_field_data["aci_node_id"],
                 pod_id=nbdevice.custom_field_data["aci_pod_id"],
                 site_tag=self.site,
+                controller_group=(
+                    nbdevice.controller_managed_device_group.name if nbdevice.controller_managed_device_group else ""
+                ),
             )
             self.add(_device)