From 9e9ff5177b0a5cca0cc0188f7586e500a1905d68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?zoe=C2=A0=E2=9C=A8?= <9274242+zoedt@users.noreply.github.com> Date: Thu, 26 Sep 2024 09:38:53 -0400 Subject: [PATCH 1/9] First pass at sending topology link metadata Fix pydantic model --- cisco_aci/assets/configuration/spec.yaml | 2 +- cisco_aci/datadog_checks/cisco_aci/api.py | 10 +++ .../cisco_aci/data/conf.yaml.example | 2 +- cisco_aci/datadog_checks/cisco_aci/fabric.py | 8 +- cisco_aci/datadog_checks/cisco_aci/helpers.py | 8 ++ cisco_aci/datadog_checks/cisco_aci/models.py | 78 +++++++++++++++++++ cisco_aci/datadog_checks/cisco_aci/ndm.py | 66 +++++++++++++++- 7 files changed, 170 insertions(+), 4 deletions(-) diff --git a/cisco_aci/assets/configuration/spec.yaml b/cisco_aci/assets/configuration/spec.yaml index 98b3c3d4d8502..f466b50fbe0b5 100644 --- a/cisco_aci/assets/configuration/spec.yaml +++ b/cisco_aci/assets/configuration/spec.yaml @@ -101,7 +101,7 @@ files: example: default - name: send_ndm_metadata description: | - Set to `true` to enable Network Device Monitoring metadata (for devices and interfaces) to be sent. + Set to `true` to enable Network Device Monitoring metadata (for devices, interfaces, topology) to be sent. value: type: boolean example: False diff --git a/cisco_aci/datadog_checks/cisco_aci/api.py b/cisco_aci/datadog_checks/cisco_aci/api.py index d84ddca48f210..24b7d1e5f8e96 100644 --- a/cisco_aci/datadog_checks/cisco_aci/api.py +++ b/cisco_aci/datadog_checks/cisco_aci/api.py @@ -297,6 +297,16 @@ def get_eth_stats(self, pod, node, eth): response = self.make_request(path) return self._parse_response(response) + def get_lldp_adj_ep(self): + path = '/api/node/class/lldpAdjEp.json' + response = self.make_request(path) + return self._parse_response(response) + + def get_cdp_adj_ep(self): + path = '/api/node/class/cdpAdjEp.json' + response = self.make_request(path) + return self._parse_response(response) + def get_eqpt_capacity(self, eqpt): base_path = '/api/class/eqptcapacityEntity.json' base_query = 'query-target=self&rsp-subtree-include=stats&rsp-subtree-class=' diff --git a/cisco_aci/datadog_checks/cisco_aci/data/conf.yaml.example b/cisco_aci/datadog_checks/cisco_aci/data/conf.yaml.example index fabcec25daf24..fb2715c64aefa 100644 --- a/cisco_aci/datadog_checks/cisco_aci/data/conf.yaml.example +++ b/cisco_aci/datadog_checks/cisco_aci/data/conf.yaml.example @@ -131,7 +131,7 @@ instances: # namespace: default ## @param send_ndm_metadata - boolean - optional - default: false - ## Set to `true` to enable Network Device Monitoring metadata (for devices and interfaces) to be sent. + ## Set to `true` to enable Network Device Monitoring metadata (for devices, interfaces, topology) to be sent. # # send_ndm_metadata: false diff --git a/cisco_aci/datadog_checks/cisco_aci/fabric.py b/cisco_aci/datadog_checks/cisco_aci/fabric.py index 1a99ebbb3425b..ff885eea1d3bb 100644 --- a/cisco_aci/datadog_checks/cisco_aci/fabric.py +++ b/cisco_aci/datadog_checks/cisco_aci/fabric.py @@ -47,8 +47,14 @@ def collect(self): pods = self.submit_pod_health(fabric_pods) devices, interfaces = self.submit_nodes_health_and_metadata(fabric_nodes, pods) if self.ndm_enabled(): + # get topology link metadata + lldp_adj_eps = self.api.get_lldp_adj_eps() + cdp_adj_eps = self.api.get_cdp_adj_eps() + device_map = ndm.get_device_ip_mapping(devices) + links = ndm.create_topology_link_metadata(lldp_adj_eps, cdp_adj_eps, device_map, self.namespace) + collect_timestamp = int(time.time()) - batches = ndm.batch_payloads(self.namespace, devices, interfaces, collect_timestamp) + batches = ndm.batch_payloads(self.namespace, devices, interfaces, links, collect_timestamp) for batch in batches: self.event_platform_event(json.dumps(batch.model_dump(exclude_none=True)), "network-devices-metadata") diff --git a/cisco_aci/datadog_checks/cisco_aci/helpers.py b/cisco_aci/datadog_checks/cisco_aci/helpers.py index ac2cd56f40a7e..62bd4e63bc4b8 100644 --- a/cisco_aci/datadog_checks/cisco_aci/helpers.py +++ b/cisco_aci/datadog_checks/cisco_aci/helpers.py @@ -11,6 +11,7 @@ EPG_REGEX = re.compile('/epg-([^/]+)/') IP_REGEX = re.compile('/ip-([^/]+)/') NODE_REGEX = re.compile('node-([0-9]+)') +ETH_REGEX = re.compile('\[([^]]*)\]') def parse_capacity_tags(dn): @@ -84,6 +85,13 @@ def get_node_from_dn(dn): """ return _get_value_from_dn(NODE_REGEX, dn) +def get_eth_id_from_dn(dn): + """ + This parses the interface ID (eth) from a dn designator. They look like this: + topology/pod-1/node-101/sys/lldp/inst/if-[eth1/49]/adj-1 + """ + return _get_value_from_dn(ETH_REGEX, dn) + def _get_value_from_dn(regex, dn): if not dn: diff --git a/cisco_aci/datadog_checks/cisco_aci/models.py b/cisco_aci/datadog_checks/cisco_aci/models.py index 3081d8ffd685c..078051a7ea78e 100644 --- a/cisco_aci/datadog_checks/cisco_aci/models.py +++ b/cisco_aci/datadog_checks/cisco_aci/models.py @@ -8,6 +8,9 @@ from pydantic import BaseModel, ConfigDict, Field, computed_field, field_validator, model_validator +from . import helpers + + """ Cisco ACI Response Models """ @@ -77,6 +80,43 @@ def ethpm_phys_if(self) -> Optional[EthpmPhysIf]: return None +class LldpAdjAttributes(BaseModel): + chassis_id_t: Optional[str] = Field(default=None, alias="chassisIdT") + chassis_id_v: Optional[str] = Field(default=None, alias="chassisIdV") + dn: Optional[str] = None + mgmt_ip: Optional[str] = Field(default=None, alias="mgmtIp") + mgmt_port_mac: Optional[str] = Field(default=None, alias="mgmtPortMac") + port_desc: Optional[str] = Field(default=None, alias="portDesc") + port_id_t: Optional[str] = Field(default=None, alias="portIdT") + port_id_v: Optional[str] = Field(default=None, alias="portIdV") + sys_desc: Optional[str] = Field(default=None, alias="sysDesc") + sys_name: Optional[str] = Field(default=None, alias="sysName") + + @computed_field + @property + def local_device_dn(self) -> str: + # example: topology/pod-1/node-101/sys/lldp/inst/if-[eth1/49]/adj-1 + return helpers.get_hostname_from_dn(self.dn) + + @computed_field + @property + def remote_device_index(self) -> str: + # example: topology/pod-1/paths-201/path-ep-[eth1/1] + # use regex to extract port alias from square brackets - ex: eth1/1 + return helpers.get_eth_id_from_dn(self.port_desc) + + @computed_field + @property + def local_port_id(self) -> str: + # example: topology/pod-1/paths-201/path-ep-[eth1/1] + # use regex to extract port alias from square brackets - ex: eth1/1 + return helpers.get_eth_id_from_dn(self.dn) + + +class LldpAdjEp(BaseModel): + attributes: dict = LldpAdjAttributes + + """ NDM Models """ @@ -96,6 +136,9 @@ class DeviceMetadata(BaseModel): device_type: Optional[str] = Field(default=None) integration: Optional[str] = Field(default='cisco-aci') + # non-exported fields + pod_node_id: Optional[str] = Field(default=None, exclude=True) + @computed_field @property def status(self) -> int: @@ -195,10 +238,45 @@ class InterfaceMetadataList(BaseModel): interface_metadata: list = Field(default_factory=list) +class TopologyLinkDevice(BaseModel): + dd_id: Optional[str] = None + id: Optional[str] = None + id_type: Optional[str] = None + name: Optional[str] = None + description: Optional[str] = None + ip_address: Optional[str] = None + + +class TopologyLinkInterface(BaseModel): + dd_id: Optional[str] = None + id: Optional[str] = None + id_type: Optional[str] = None + description: Optional[str] = None + + +class TopologyLinkSide(BaseModel): + device: Optional[TopologyLinkDevice] = None + interface: Optional[TopologyLinkInterface] = None + + +class SourceType(StrEnum): + LLDP = "lldp" + CDP = "cdp" + OTHER = "OTHER" + + +class TopologyLinkMetadata(BaseModel): + id: Optional[str] = None + source_type = Optional[SourceType] + local = Optional[TopologyLinkSide] + remote = Optional[TopologyLinkSide] + + class NetworkDevicesMetadata(BaseModel): namespace: str = None devices: Optional[list[DeviceMetadata]] = Field(default_factory=list) interfaces: Optional[list[InterfaceMetadata]] = Field(default_factory=list) + links: Optional[list[TopologyLinkMetadata]] = Field(default_factory=list) collect_timestamp: Optional[int] = None size: Optional[int] = Field(default=0, exclude=True) diff --git a/cisco_aci/datadog_checks/cisco_aci/ndm.py b/cisco_aci/datadog_checks/cisco_aci/ndm.py index 5681350d6b39f..76f57f1da5981 100644 --- a/cisco_aci/datadog_checks/cisco_aci/ndm.py +++ b/cisco_aci/datadog_checks/cisco_aci/ndm.py @@ -5,11 +5,19 @@ from datadog_checks.cisco_aci.models import ( DeviceMetadata, InterfaceMetadata, + LldpAdjEp, NetworkDevicesMetadata, Node, PhysIf, + SourceType, + TopologyLinkDevice, + TopologyLinkInterface, + TopologyLinkMetadata, + TopologyLinkSide, ) +from . import helpers + VENDOR_CISCO = 'cisco' PAYLOAD_METADATA_BATCH_SIZE = 100 @@ -37,6 +45,7 @@ def create_node_metadata(node_attrs, tags, namespace): version=node.attributes.version, serial_number=node.attributes.serial, device_type=node.attributes.device_type, + pod_node_id=helpers.get_hostname_from_dn(node.attributes.dn), ) return device @@ -61,6 +70,55 @@ def create_interface_metadata(phys_if, address, namespace): return interface +def create_topology_link_metadata(lldp_adj_eps, cdp_adj_eps, device_map, namespace): + """ + Create a TopologyLinkMetadata object from LLDP or CDP + """ + lldp_adj_eps_list = LldpAdjEp() + for lldp_adj_ep in lldp_adj_eps_list: + local_device_id = device_map.get(lldp_adj_ep.attributes.local_device_dn) + remote_entry_unique_id = lldp_adj_ep.attributes.local_port_id + "." + lldp_adj_ep.attributes.remote_port_id + + local = TopologyLinkSide( + # TODO: need to grab the device id from mapping + device=TopologyLinkDevice(dd_id=local_device_id), + # TODO: double check resolve the local interface id + interface=TopologyLinkInterface( + dd_id='', id=lldp_adj_ep.attributes.local_port_id, id_type='interface_name' + ), + ) + remote = TopologyLinkSide( + # this is all good afaik + device=TopologyLinkDevice( + name=lldp_adj_ep.attributes.system_name, + description=lldp_adj_ep.attributes.system_desc, + id=lldp_adj_ep.attributes.chassis_id_v, + id_type=lldp_adj_ep.attributes.chassis_id_t, + ip_address=lldp_adj_ep.attributes.mgmt_ip, + ), + # TODO: check on the interface alias/name for resolution vs. taken what's given to us + interface=TopologyLinkInterface( + id=lldp_adj_ep.attributes.remote_port_id, + id_type=lldp_adj_ep.attributes.port_id_t, + description=lldp_adj_ep.attributes.port_desc, + ), + ) + yield TopologyLinkMetadata( + id='{}:{}'.format(local_device_id, remote_entry_unique_id), + source_type=SourceType.LLDP, + local=local, + remote=remote, + ) + + +def get_device_ip_mapping(devices): + devices_map = {} + for device in devices: + key = device.pod_node_id + devices_map[key] = device.ip_address + return devices_map + + def get_device_info(device): """ Get device ID and node ID from a device object @@ -72,7 +130,7 @@ def get_device_info(device): return device.id, node_id -def batch_payloads(namespace, devices, interfaces, collect_ts): +def batch_payloads(namespace, devices, interfaces, links, collect_ts): """ Batch payloads into NetworkDevicesMetadata objects """ @@ -89,6 +147,12 @@ def batch_payloads(namespace, devices, interfaces, collect_ts): yield current_payload network_devices_metadata = new_payload + for link in links: + current_payload, new_payload = append_to_payload(link, network_devices_metadata, namespace, collect_ts) + if new_payload: + yield current_payload + network_devices_metadata = new_payload + yield network_devices_metadata From b6609f63c428c2aec5e71d8d431532d7b27ac17f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?zoe=C2=A0=E2=9C=A8?= <9274242+zoedt@users.noreply.github.com> Date: Thu, 26 Sep 2024 09:59:53 -0400 Subject: [PATCH 2/9] Add changelog :-) --- cisco_aci/changelog.d/18675.added | 1 + 1 file changed, 1 insertion(+) create mode 100644 cisco_aci/changelog.d/18675.added diff --git a/cisco_aci/changelog.d/18675.added b/cisco_aci/changelog.d/18675.added new file mode 100644 index 0000000000000..027112714322f --- /dev/null +++ b/cisco_aci/changelog.d/18675.added @@ -0,0 +1 @@ +[NDM] [Cisco ACI] Support submitting topology metadata From a50e00cc7c50ecdc084f80adf6cd82642648d3a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?zoe=C2=A0=E2=9C=A8?= <9274242+zoedt@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:09:01 -0400 Subject: [PATCH 3/9] Clean up + add some test cases --- cisco_aci/datadog_checks/cisco_aci/api.py | 4 +- cisco_aci/datadog_checks/cisco_aci/helpers.py | 12 +++- cisco_aci/datadog_checks/cisco_aci/models.py | 29 ++++++--- cisco_aci/datadog_checks/cisco_aci/ndm.py | 25 ++++---- cisco_aci/tests/common.py | 4 ++ .../588ea77fffc6df4b37dfdfa4290cdc89.txt | 4 ++ .../f3713df3a586908a3a11f4c356153519.txt | 61 +++++++++++++++++++ cisco_aci/tests/fixtures/metadata.py | 60 ++++++++++++++++++ 8 files changed, 175 insertions(+), 24 deletions(-) create mode 100644 cisco_aci/tests/fixtures/fabric/588ea77fffc6df4b37dfdfa4290cdc89.txt create mode 100644 cisco_aci/tests/fixtures/fabric/f3713df3a586908a3a11f4c356153519.txt diff --git a/cisco_aci/datadog_checks/cisco_aci/api.py b/cisco_aci/datadog_checks/cisco_aci/api.py index 24b7d1e5f8e96..f2b7b0c92fd93 100644 --- a/cisco_aci/datadog_checks/cisco_aci/api.py +++ b/cisco_aci/datadog_checks/cisco_aci/api.py @@ -297,12 +297,12 @@ def get_eth_stats(self, pod, node, eth): response = self.make_request(path) return self._parse_response(response) - def get_lldp_adj_ep(self): + def get_lldp_adj_eps(self): path = '/api/node/class/lldpAdjEp.json' response = self.make_request(path) return self._parse_response(response) - def get_cdp_adj_ep(self): + def get_cdp_adj_eps(self): path = '/api/node/class/cdpAdjEp.json' response = self.make_request(path) return self._parse_response(response) diff --git a/cisco_aci/datadog_checks/cisco_aci/helpers.py b/cisco_aci/datadog_checks/cisco_aci/helpers.py index 62bd4e63bc4b8..7ebe981f7fbb1 100644 --- a/cisco_aci/datadog_checks/cisco_aci/helpers.py +++ b/cisco_aci/datadog_checks/cisco_aci/helpers.py @@ -11,7 +11,7 @@ EPG_REGEX = re.compile('/epg-([^/]+)/') IP_REGEX = re.compile('/ip-([^/]+)/') NODE_REGEX = re.compile('node-([0-9]+)') -ETH_REGEX = re.compile('\[([^]]*)\]') +ETH_REGEX = re.compile(r'\[([^]]*)\]') def parse_capacity_tags(dn): @@ -85,6 +85,7 @@ def get_node_from_dn(dn): """ return _get_value_from_dn(NODE_REGEX, dn) + def get_eth_id_from_dn(dn): """ This parses the interface ID (eth) from a dn designator. They look like this: @@ -93,6 +94,15 @@ def get_eth_id_from_dn(dn): return _get_value_from_dn(ETH_REGEX, dn) +def get_index_from_eth_id(eth_id): + """ + This parses the interface index (eth) from an interface's ID. They look like this: + eth1/49 + """ + split = re.split('eth|/', eth_id) + return int(split[-1]) + + def _get_value_from_dn(regex, dn): if not dn: return None diff --git a/cisco_aci/datadog_checks/cisco_aci/models.py b/cisco_aci/datadog_checks/cisco_aci/models.py index 078051a7ea78e..4a5ea4ac8976e 100644 --- a/cisco_aci/datadog_checks/cisco_aci/models.py +++ b/cisco_aci/datadog_checks/cisco_aci/models.py @@ -10,7 +10,6 @@ from . import helpers - """ Cisco ACI Response Models """ @@ -100,21 +99,31 @@ def local_device_dn(self) -> str: @computed_field @property - def remote_device_index(self) -> str: + def local_port_id(self) -> str: # example: topology/pod-1/paths-201/path-ep-[eth1/1] # use regex to extract port alias from square brackets - ex: eth1/1 - return helpers.get_eth_id_from_dn(self.port_desc) + return helpers.get_eth_id_from_dn(self.dn) @computed_field @property - def local_port_id(self) -> str: + def local_port_index(self) -> int: + return helpers.get_index_from_eth_id(self.local_port_id) + + @computed_field + @property + def remote_port_id(self) -> str: # example: topology/pod-1/paths-201/path-ep-[eth1/1] # use regex to extract port alias from square brackets - ex: eth1/1 - return helpers.get_eth_id_from_dn(self.dn) + return helpers.get_eth_id_from_dn(self.port_desc) + + @computed_field + @property + def remote_port_index(self) -> int: + return helpers.get_index_from_eth_id(self.remote_port_id) class LldpAdjEp(BaseModel): - attributes: dict = LldpAdjAttributes + attributes: LldpAdjAttributes """ @@ -267,9 +276,9 @@ class SourceType(StrEnum): class TopologyLinkMetadata(BaseModel): id: Optional[str] = None - source_type = Optional[SourceType] - local = Optional[TopologyLinkSide] - remote = Optional[TopologyLinkSide] + source_type: Optional[SourceType] = Field(default=None) + local: Optional[TopologyLinkSide] = Field(default=None) + remote: Optional[TopologyLinkSide] = Field(default=None) class NetworkDevicesMetadata(BaseModel): @@ -287,4 +296,6 @@ def append_metadata(self, metadata: DeviceMetadata | InterfaceMetadata): self.devices.append(metadata) if isinstance(metadata, InterfaceMetadata): self.interfaces.append(metadata) + if isinstance(metadata, TopologyLinkMetadata): + self.links.append(metadata) self.size += 1 diff --git a/cisco_aci/datadog_checks/cisco_aci/ndm.py b/cisco_aci/datadog_checks/cisco_aci/ndm.py index 76f57f1da5981..e19c87daa1bbc 100644 --- a/cisco_aci/datadog_checks/cisco_aci/ndm.py +++ b/cisco_aci/datadog_checks/cisco_aci/ndm.py @@ -74,32 +74,33 @@ def create_topology_link_metadata(lldp_adj_eps, cdp_adj_eps, device_map, namespa """ Create a TopologyLinkMetadata object from LLDP or CDP """ - lldp_adj_eps_list = LldpAdjEp() - for lldp_adj_ep in lldp_adj_eps_list: + for lldp_adj_ep in lldp_adj_eps: + lldp_adj_ep = LldpAdjEp(**lldp_adj_ep.get("lldpAdjEp", {})) + local_device_id = device_map.get(lldp_adj_ep.attributes.local_device_dn) - remote_entry_unique_id = lldp_adj_ep.attributes.local_port_id + "." + lldp_adj_ep.attributes.remote_port_id + local_interface_id = "{}:{}".format(local_device_id, lldp_adj_ep.attributes.local_port_index) + + remote_entry_unique_id = "{}.{}".format( + lldp_adj_ep.attributes.local_port_index, lldp_adj_ep.attributes.remote_port_index + ) local = TopologyLinkSide( - # TODO: need to grab the device id from mapping device=TopologyLinkDevice(dd_id=local_device_id), - # TODO: double check resolve the local interface id interface=TopologyLinkInterface( - dd_id='', id=lldp_adj_ep.attributes.local_port_id, id_type='interface_name' + dd_id=local_interface_id, id=lldp_adj_ep.attributes.local_port_id, id_type='interface_name' ), ) remote = TopologyLinkSide( - # this is all good afaik device=TopologyLinkDevice( - name=lldp_adj_ep.attributes.system_name, - description=lldp_adj_ep.attributes.system_desc, + name=lldp_adj_ep.attributes.sys_name, + description=lldp_adj_ep.attributes.sys_desc, id=lldp_adj_ep.attributes.chassis_id_v, id_type=lldp_adj_ep.attributes.chassis_id_t, ip_address=lldp_adj_ep.attributes.mgmt_ip, ), - # TODO: check on the interface alias/name for resolution vs. taken what's given to us interface=TopologyLinkInterface( id=lldp_adj_ep.attributes.remote_port_id, - id_type=lldp_adj_ep.attributes.port_id_t, + id_type="interface_name", description=lldp_adj_ep.attributes.port_desc, ), ) @@ -115,7 +116,7 @@ def get_device_ip_mapping(devices): devices_map = {} for device in devices: key = device.pod_node_id - devices_map[key] = device.ip_address + devices_map[key] = device.id return devices_map diff --git a/cisco_aci/tests/common.py b/cisco_aci/tests/common.py index 442cd69afecff..75df8154d3564 100644 --- a/cisco_aci/tests/common.py +++ b/cisco_aci/tests/common.py @@ -631,6 +631,10 @@ # 4efe80304d50330f5ed0f79252ef0a84 - Api.get_apps '_api_mo_uni_tn_DataDog_json_rsp_subtree_include_stats_no_scoped', # c8e9a0dbceac67fb1149684f7fc7772c - Api.get_tenant_stats + '_api_node_class_lldpAdjEp_json', + # f3713df3a586908a3a11f4c356153519 - Api.get_lldp_adj_eps + '_api_node_class_cdpAdjEp_json', + # 588ea77fffc6df4b37dfdfa4290cdc89 - Api.get_cdp_adj_eps ] # The map will contain the md5 hash to the fixture diff --git a/cisco_aci/tests/fixtures/fabric/588ea77fffc6df4b37dfdfa4290cdc89.txt b/cisco_aci/tests/fixtures/fabric/588ea77fffc6df4b37dfdfa4290cdc89.txt new file mode 100644 index 0000000000000..6cc0e51dea023 --- /dev/null +++ b/cisco_aci/tests/fixtures/fabric/588ea77fffc6df4b37dfdfa4290cdc89.txt @@ -0,0 +1,4 @@ +{ + "totalCount": "0", + "imdata": [] +} \ No newline at end of file diff --git a/cisco_aci/tests/fixtures/fabric/f3713df3a586908a3a11f4c356153519.txt b/cisco_aci/tests/fixtures/fabric/f3713df3a586908a3a11f4c356153519.txt new file mode 100644 index 0000000000000..6a11461f5096e --- /dev/null +++ b/cisco_aci/tests/fixtures/fabric/f3713df3a586908a3a11f4c356153519.txt @@ -0,0 +1,61 @@ +{ + "totalCount": "1", + "imdata": [ + { + "lldpAdjEp": { + "attributes": { + "capability": "router", + "chassisIdT": "mac", + "chassisIdV": "6a:00:21:1f:55:2a", + "childAction": "", + "dn": "topology/pod-1/node-101/sys/lldp/inst/if-[eth1/49]/adj-1", + "enCap": "", + "id": "1", + "mgmtId": "0", + "mgmtIp": "10.0.200.5", + "mgmtPortMac": "unspecified", + "modTs": "2024-09-12T06:51:52.580+00:00", + "monPolDn": "uni/fabric/monfab-default", + "name": "", + "portDesc": "topology/pod-1/paths-201/pathep-[eth5/1]", + "portIdT": "mac", + "portIdV": "6a:00:21:1f:55:2a", + "portVlan": "unspecified", + "stQual": "", + "status": "", + "sysDesc": "topology/pod-1/node-201", + "sysName": "SP201", + "ttl": "120" + } + } + }, + { + "lldpAdjEp": { + "attributes": { + "capability": "router", + "chassisIdT": "mac", + "chassisIdV": "6a:00:21:1f:55:2b", + "childAction": "", + "dn": "topology/pod-1/node-102/sys/lldp/inst/if-[eth1/49]/adj-1", + "enCap": "", + "id": "1", + "mgmtId": "0", + "mgmtIp": "10.0.200.5", + "mgmtPortMac": "unspecified", + "modTs": "2024-09-12T06:51:52.580+00:00", + "monPolDn": "uni/fabric/monfab-default", + "name": "", + "portDesc": "topology/pod-1/paths-201/pathep-[eth5/2]", + "portIdT": "mac", + "portIdV": "6a:00:21:1f:55:2b", + "portVlan": "unspecified", + "stQual": "", + "status": "", + "sysDesc": "topology/pod-1/node-201", + "sysName": "SP201", + "ttl": "120" + } + } + } + ] +} \ No newline at end of file diff --git a/cisco_aci/tests/fixtures/metadata.py b/cisco_aci/tests/fixtures/metadata.py index 0052b27c5448e..ac55169289d0e 100644 --- a/cisco_aci/tests/fixtures/metadata.py +++ b/cisco_aci/tests/fixtures/metadata.py @@ -3191,6 +3191,65 @@ }, ] +TOPOLOGY_LINK_METADATA = [ + { + 'id': 'default:10.0.200.0:49.1', + 'local': { + 'device': { + 'dd_id': 'default:10.0.200.0', + }, + 'interface': { + 'dd_id': 'default:10.0.200.0:49', + 'id': 'eth1/49', + 'id_type': 'interface_name', + }, + }, + 'remote': { + 'device': { + 'description': 'topology/pod-1/node-201', + 'id': '6a:00:21:1f:55:2a', + 'id_type': 'mac', + 'ip_address': '10.0.200.5', + 'name': 'SP201', + }, + 'interface': { + 'description': 'topology/pod-1/paths-201/pathep-[eth5/1]', + 'id': 'eth5/1', + 'id_type': 'interface_name', + }, + }, + 'source_type': 'lldp', + }, + { + 'id': 'default:10.0.200.1:49.2', + 'local': { + 'device': { + 'dd_id': 'default:10.0.200.1', + }, + 'interface': { + 'dd_id': 'default:10.0.200.1:49', + 'id': 'eth1/49', + 'id_type': 'interface_name', + }, + }, + 'remote': { + 'device': { + 'description': 'topology/pod-1/node-201', + 'id': '6a:00:21:1f:55:2b', + 'id_type': 'mac', + 'ip_address': '10.0.200.5', + 'name': 'SP201', + }, + 'interface': { + 'description': 'topology/pod-1/paths-201/pathep-[eth5/2]', + 'id': 'eth5/2', + 'id_type': 'interface_name', + }, + }, + 'source_type': 'lldp', + }, +] + EXPECTED_DEVICE_METADATA_RESULT = DeviceMetadataList(device_metadata=DEVICE_METADATA) # "2012-01-14 03:21:34" in seconds @@ -3211,6 +3270,7 @@ NetworkDevicesMetadata( namespace='default', interfaces=INTERFACE_METADATA[193::], + links=TOPOLOGY_LINK_METADATA, collect_timestamp=MOCK_TIME_EPOCH, ), ] From 7f711451c6bf9eab46f86b769e412aba45ad793b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?zoe=C2=A0=E2=9C=A8?= <9274242+zoedt@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:56:59 -0400 Subject: [PATCH 4/9] Amend getting the local interface DD ID for topology --- cisco_aci/datadog_checks/cisco_aci/ndm.py | 14 +++++++++++++- cisco_aci/tests/fixtures/metadata.py | 4 ++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/cisco_aci/datadog_checks/cisco_aci/ndm.py b/cisco_aci/datadog_checks/cisco_aci/ndm.py index 1e854ca826ce3..d15582a25673c 100644 --- a/cisco_aci/datadog_checks/cisco_aci/ndm.py +++ b/cisco_aci/datadog_checks/cisco_aci/ndm.py @@ -80,7 +80,7 @@ def create_topology_link_metadata(lldp_adj_eps, cdp_adj_eps, device_map, namespa lldp_adj_ep = LldpAdjEp(**lldp_adj_ep.get("lldpAdjEp", {})) local_device_id = device_map.get(lldp_adj_ep.attributes.local_device_dn) - local_interface_id = "{}:{}".format(local_device_id, lldp_adj_ep.attributes.local_port_index) + local_interface_id = "{}:{}".format(local_device_id, lldp_adj_ep.attributes.local_port_id) remote_entry_unique_id = "{}.{}".format( lldp_adj_ep.attributes.local_port_index, lldp_adj_ep.attributes.remote_port_index @@ -114,7 +114,19 @@ def create_topology_link_metadata(lldp_adj_eps, cdp_adj_eps, device_map, namespa ) +def get_interface_dd_id(device_id: str, port_id: str) -> str: + """ + Create the interface DD ID based off of the device DD ID and port ID + ex: default:10.0.200.1:cisco_aci-eth1/1 + """ + return '{}:cisco_aci-{}'.format(device_id, port_id) + + def get_device_ip_mapping(devices): + """ + Create a mapping of node ID to device ID + ex: pod-1-node-1 -> default:10.100.0.1 + """ devices_map = {} for device in devices: key = device.pod_node_id diff --git a/cisco_aci/tests/fixtures/metadata.py b/cisco_aci/tests/fixtures/metadata.py index ad3b37a86fdb4..46933da8e67da 100644 --- a/cisco_aci/tests/fixtures/metadata.py +++ b/cisco_aci/tests/fixtures/metadata.py @@ -280,7 +280,7 @@ 'dd_id': 'default:10.0.200.0', }, 'interface': { - 'dd_id': 'default:10.0.200.0:49', + 'dd_id': 'default:10.0.200.0:eth1/49', 'id': 'eth1/49', 'id_type': 'interface_name', }, @@ -308,7 +308,7 @@ 'dd_id': 'default:10.0.200.1', }, 'interface': { - 'dd_id': 'default:10.0.200.1:49', + 'dd_id': 'default:10.0.200.1:eth1/49', 'id': 'eth1/49', 'id_type': 'interface_name', }, From 9523c7330747668897bc3b7fad78d51362fd5afd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?zoe=C2=A0=E2=9C=A8?= <9274242+zoedt@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:57:02 -0500 Subject: [PATCH 5/9] Add local resolution, utilize interface raw ID --- cisco_aci/datadog_checks/cisco_aci/models.py | 24 ++++++ cisco_aci/datadog_checks/cisco_aci/ndm.py | 79 ++++++++++++++------ cisco_aci/tests/fixtures/metadata.py | 16 ++-- 3 files changed, 91 insertions(+), 28 deletions(-) diff --git a/cisco_aci/datadog_checks/cisco_aci/models.py b/cisco_aci/datadog_checks/cisco_aci/models.py index 8e2af99fdeb56..be1b029c3feb3 100644 --- a/cisco_aci/datadog_checks/cisco_aci/models.py +++ b/cisco_aci/datadog_checks/cisco_aci/models.py @@ -91,6 +91,23 @@ class LldpAdjAttributes(BaseModel): sys_desc: Optional[str] = Field(default=None, alias="sysDesc") sys_name: Optional[str] = Field(default=None, alias="sysName") + @computed_field + @property + def ndm_remote_interface_type(self) -> str: + # map the Cisco ACI port subtype to match what NDM (writer) expects + port_subtype_mapping = { + "if-alias": "interface_alias", + "port-name": "interface_name", + "mac": "mac_address", + "nw-addr": "network_address", + "if-name": "interface_name", + "agent-ckt-id": "agent_circuit_id", + "local": "local", + } + if self.port_id_t: + return port_subtype_mapping.get(self.port_id_t, "unknown") + return "unknown" + @computed_field @property def local_device_dn(self) -> str: @@ -109,6 +126,13 @@ def local_port_id(self) -> str: def local_port_index(self) -> int: return helpers.get_index_from_eth_id(self.local_port_id) + @computed_field + @property + def remote_device_dn(self) -> str: + # example: topology/pod-1/paths-201/path-ep-[eth1/1] + # use regex to extract the pod/node - ex: pod-1-node-201 + return helpers.get_hostname_from_dn(self.sys_desc) + @computed_field @property def remote_port_id(self) -> str: diff --git a/cisco_aci/datadog_checks/cisco_aci/ndm.py b/cisco_aci/datadog_checks/cisco_aci/ndm.py index d15582a25673c..5964f88c805a8 100644 --- a/cisco_aci/datadog_checks/cisco_aci/ndm.py +++ b/cisco_aci/datadog_checks/cisco_aci/ndm.py @@ -74,52 +74,87 @@ def create_interface_metadata(phys_if, address, namespace): def create_topology_link_metadata(lldp_adj_eps, cdp_adj_eps, device_map, namespace): """ - Create a TopologyLinkMetadata object from LLDP or CDP + Create a TopologyLinkMetadata object from LLDP or CDP (only LLDP is supported as of right now) """ for lldp_adj_ep in lldp_adj_eps: lldp_adj_ep = LldpAdjEp(**lldp_adj_ep.get("lldpAdjEp", {})) + lldp_attrs = lldp_adj_ep.attributes - local_device_id = device_map.get(lldp_adj_ep.attributes.local_device_dn) - local_interface_id = "{}:{}".format(local_device_id, lldp_adj_ep.attributes.local_port_id) - - remote_entry_unique_id = "{}.{}".format( - lldp_adj_ep.attributes.local_port_index, lldp_adj_ep.attributes.remote_port_index - ) + local_device_id = device_map.get(lldp_attrs.local_device_dn) + local_interface_id = "{}:{}".format(local_device_id, lldp_attrs.local_port_id) local = TopologyLinkSide( device=TopologyLinkDevice(dd_id=local_device_id), interface=TopologyLinkInterface( - dd_id=local_interface_id, id=lldp_adj_ep.attributes.local_port_id, id_type='interface_name' + dd_id=local_interface_id, id=lldp_attrs.local_port_id, id_type='interface_name' ), ) + + remote_device_dd_id = get_remote_device_dd_id(device_map, lldp_attrs.remote_device_dn, lldp_attrs.mgmt_ip) + remote_device = TopologyLinkDevice( + name=lldp_attrs.sys_name, + description=lldp_attrs.sys_desc, + id=lldp_attrs.chassis_id_v, + id_type=lldp_attrs.chassis_id_t, + ip_address=lldp_attrs.mgmt_ip, + ) + if remote_device_dd_id: + remote_device.dd_id = remote_device_dd_id + remote_interface = TopologyLinkInterface( + id=lldp_attrs.port_id_v, + id_type=lldp_attrs.ndm_remote_interface_type, + description=lldp_attrs.port_desc, + ) + if remote_device_dd_id: + remote_interface.dd_id = get_interface_dd_id(remote_device_dd_id, lldp_attrs.remote_port_id) + remote = TopologyLinkSide( - device=TopologyLinkDevice( - name=lldp_adj_ep.attributes.sys_name, - description=lldp_adj_ep.attributes.sys_desc, - id=lldp_adj_ep.attributes.chassis_id_v, - id_type=lldp_adj_ep.attributes.chassis_id_t, - ip_address=lldp_adj_ep.attributes.mgmt_ip, - ), - interface=TopologyLinkInterface( - id=lldp_adj_ep.attributes.remote_port_id, - id_type="interface_name", - description=lldp_adj_ep.attributes.port_desc, - ), + device=remote_device, + interface=remote_interface, ) + + if remote_device_dd_id: + link_id = ( + f"{local_device_id}:{get_raw_id(lldp_attrs.local_port_id)}.{get_raw_id(lldp_attrs.remote_port_id)}" + ) + else: + link_id = f"{local_device_id}:{get_raw_id(lldp_attrs.local_port_id)}.{lldp_attrs.remote_port_index}" + yield TopologyLinkMetadata( - id='{}:{}'.format(local_device_id, remote_entry_unique_id), + id=link_id, source_type=SourceType.LLDP, local=local, remote=remote, ) +def get_remote_device_dd_id(device_map, remote_device_dn, mgmt_ip) -> str | None: + """ + Get the Cisco DN for a remote device, if the device is in the device map then + check that it matches the management IP of the LLDP neighbor, then return it + """ + device_id = device_map.get(remote_device_dn, "") + if device_id: + if device_id.endswith(mgmt_ip): + return device_id + return None + + def get_interface_dd_id(device_id: str, port_id: str) -> str: """ Create the interface DD ID based off of the device DD ID and port ID ex: default:10.0.200.1:cisco_aci-eth1/1 """ - return '{}:cisco_aci-{}'.format(device_id, port_id) + raw_id = get_raw_id(port_id) + return f"{device_id}:{raw_id}" + + +def get_raw_id(raw_id, raw_id_type="cisco-aci") -> str: + """ + Create the interface raw ID, based on the type (cisco-aci) and the interface's identifier + separated by a hyphen - ex: cisco-aci-eth1/1 + """ + return f"{raw_id_type}-{raw_id}" def get_device_ip_mapping(devices): diff --git a/cisco_aci/tests/fixtures/metadata.py b/cisco_aci/tests/fixtures/metadata.py index 46933da8e67da..d150e96216019 100644 --- a/cisco_aci/tests/fixtures/metadata.py +++ b/cisco_aci/tests/fixtures/metadata.py @@ -274,7 +274,7 @@ TOPOLOGY_LINK_METADATA = [ { - 'id': 'default:10.0.200.0:49.1', + 'id': 'default:10.0.200.0:cisco-aci-eth1/49.cisco-aci-eth5/1', 'local': { 'device': { 'dd_id': 'default:10.0.200.0', @@ -287,6 +287,7 @@ }, 'remote': { 'device': { + 'dd_id': 'default:10.0.200.5', 'description': 'topology/pod-1/node-201', 'id': '6a:00:21:1f:55:2a', 'id_type': 'mac', @@ -294,15 +295,16 @@ 'name': 'SP201', }, 'interface': { + 'dd_id': 'default:10.0.200.5:cisco-aci-eth5/1', 'description': 'topology/pod-1/paths-201/pathep-[eth5/1]', - 'id': 'eth5/1', - 'id_type': 'interface_name', + 'id': '6a:00:21:1f:55:2a', + 'id_type': 'mac_address', }, }, 'source_type': 'lldp', }, { - 'id': 'default:10.0.200.1:49.2', + 'id': 'default:10.0.200.1:cisco-aci-eth1/49.cisco-aci-eth5/2', 'local': { 'device': { 'dd_id': 'default:10.0.200.1', @@ -315,6 +317,7 @@ }, 'remote': { 'device': { + 'dd_id': 'default:10.0.200.5', 'description': 'topology/pod-1/node-201', 'id': '6a:00:21:1f:55:2b', 'id_type': 'mac', @@ -322,9 +325,10 @@ 'name': 'SP201', }, 'interface': { + 'dd_id': 'default:10.0.200.5:cisco-aci-eth5/2', 'description': 'topology/pod-1/paths-201/pathep-[eth5/2]', - 'id': 'eth5/2', - 'id_type': 'interface_name', + 'id': '6a:00:21:1f:55:2b', + 'id_type': 'mac_address', }, }, 'source_type': 'lldp', From 8aeba789b74e24bc1e7fccefac35cd818e1e0902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?zoe=C2=A0=E2=9C=A8?= <9274242+zoedt@users.noreply.github.com> Date: Wed, 4 Dec 2024 13:25:14 -0500 Subject: [PATCH 6/9] Add integration field for TopologyLinkMetadata --- cisco_aci/datadog_checks/cisco_aci/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cisco_aci/datadog_checks/cisco_aci/models.py b/cisco_aci/datadog_checks/cisco_aci/models.py index be1b029c3feb3..c9b733fdd3708 100644 --- a/cisco_aci/datadog_checks/cisco_aci/models.py +++ b/cisco_aci/datadog_checks/cisco_aci/models.py @@ -306,6 +306,7 @@ class TopologyLinkMetadata(BaseModel): source_type: Optional[SourceType] = Field(default=None) local: Optional[TopologyLinkSide] = Field(default=None) remote: Optional[TopologyLinkSide] = Field(default=None) + integration: Optional[str] = Field(default='cisco-aci') class NetworkDevicesMetadata(BaseModel): From 5f06baf9fcecd9d2da6367e1e0949683df98bbf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?zoe=C2=A0=E2=9C=A8?= <9274242+zoedt@users.noreply.github.com> Date: Wed, 4 Dec 2024 13:56:52 -0500 Subject: [PATCH 7/9] Fix interface raw ID type, local interface ID --- cisco_aci/datadog_checks/cisco_aci/models.py | 2 +- cisco_aci/datadog_checks/cisco_aci/ndm.py | 5 +++-- cisco_aci/tests/fixtures/metadata.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cisco_aci/datadog_checks/cisco_aci/models.py b/cisco_aci/datadog_checks/cisco_aci/models.py index c9b733fdd3708..8505fdcb6efa3 100644 --- a/cisco_aci/datadog_checks/cisco_aci/models.py +++ b/cisco_aci/datadog_checks/cisco_aci/models.py @@ -212,7 +212,7 @@ class InterfaceMetadata(BaseModel): device_id: Optional[str] = Field(default=None) id_tags: list = Field(default_factory=list) raw_id: Optional[str] = Field(default=None) - raw_id_type: Optional[str] = Field(default='cisco_aci') + raw_id_type: Optional[str] = Field(default='cisco-aci') index: Optional[int] = Field(default=None) name: Optional[str] = Field(default=None) alias: Optional[str] = Field(default=None) diff --git a/cisco_aci/datadog_checks/cisco_aci/ndm.py b/cisco_aci/datadog_checks/cisco_aci/ndm.py index 5964f88c805a8..1eb399584967a 100644 --- a/cisco_aci/datadog_checks/cisco_aci/ndm.py +++ b/cisco_aci/datadog_checks/cisco_aci/ndm.py @@ -81,12 +81,13 @@ def create_topology_link_metadata(lldp_adj_eps, cdp_adj_eps, device_map, namespa lldp_attrs = lldp_adj_ep.attributes local_device_id = device_map.get(lldp_attrs.local_device_dn) - local_interface_id = "{}:{}".format(local_device_id, lldp_attrs.local_port_id) local = TopologyLinkSide( device=TopologyLinkDevice(dd_id=local_device_id), interface=TopologyLinkInterface( - dd_id=local_interface_id, id=lldp_attrs.local_port_id, id_type='interface_name' + dd_id=get_interface_dd_id(local_device_id, lldp_attrs.local_port_id), + id=lldp_attrs.local_port_id, + id_type='interface_name', ), ) diff --git a/cisco_aci/tests/fixtures/metadata.py b/cisco_aci/tests/fixtures/metadata.py index d150e96216019..5a19d2deee13d 100644 --- a/cisco_aci/tests/fixtures/metadata.py +++ b/cisco_aci/tests/fixtures/metadata.py @@ -280,7 +280,7 @@ 'dd_id': 'default:10.0.200.0', }, 'interface': { - 'dd_id': 'default:10.0.200.0:eth1/49', + 'dd_id': 'default:10.0.200.0:cisco-aci-eth1/49', 'id': 'eth1/49', 'id_type': 'interface_name', }, @@ -310,7 +310,7 @@ 'dd_id': 'default:10.0.200.1', }, 'interface': { - 'dd_id': 'default:10.0.200.1:eth1/49', + 'dd_id': 'default:10.0.200.1:cisco-aci-eth1/49', 'id': 'eth1/49', 'id_type': 'interface_name', }, From 300aa3ab56bf44fb5701f62b02fa7189ee154b5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?zoe=C2=A0=E2=9C=A8?= <9274242+zoedt@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:38:16 -0500 Subject: [PATCH 8/9] Reference to local interface ID --- cisco_aci/datadog_checks/cisco_aci/ndm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cisco_aci/datadog_checks/cisco_aci/ndm.py b/cisco_aci/datadog_checks/cisco_aci/ndm.py index 1eb399584967a..260e4ad5ea45c 100644 --- a/cisco_aci/datadog_checks/cisco_aci/ndm.py +++ b/cisco_aci/datadog_checks/cisco_aci/ndm.py @@ -81,11 +81,12 @@ def create_topology_link_metadata(lldp_adj_eps, cdp_adj_eps, device_map, namespa lldp_attrs = lldp_adj_ep.attributes local_device_id = device_map.get(lldp_attrs.local_device_dn) + local_interface_id = get_interface_dd_id(local_device_id, lldp_attrs.local_port_id) local = TopologyLinkSide( device=TopologyLinkDevice(dd_id=local_device_id), interface=TopologyLinkInterface( - dd_id=get_interface_dd_id(local_device_id, lldp_attrs.local_port_id), + dd_id=local_interface_id, id=lldp_attrs.local_port_id, id_type='interface_name', ), From 59cfd9754b5bacb08724efa00ddd8c56b54c49e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?zoe=C2=A0=E2=9C=A8?= <9274242+zoedt@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:47:50 -0500 Subject: [PATCH 9/9] Update changelog to be explicit about LLDP support --- cisco_aci/changelog.d/18675.added | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cisco_aci/changelog.d/18675.added b/cisco_aci/changelog.d/18675.added index 027112714322f..72ee9491e1b34 100644 --- a/cisco_aci/changelog.d/18675.added +++ b/cisco_aci/changelog.d/18675.added @@ -1 +1 @@ -[NDM] [Cisco ACI] Support submitting topology metadata +[NDM] [Cisco ACI] Support submitting topology metadata (utilizing LLDP neighbor information)