From 2ddaeb76a31ced15cc55e1eece8e3dc638451ed4 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 24 Nov 2020 19:52:28 -0500 Subject: [PATCH] Add system test for Cisco IOS A system test was required because all of the parsing logic lives on the agent side rather than in an ingest pipeline. The tests detected several missing fields. In order to test the logfile input (instead of the syslog input) a solution to elastic/elastic-package#186 is required. --- packages/cisco/_dev/deploy/docker/Dockerfile | 5 ++ .../cisco/_dev/deploy/docker/cisco-ios.log | 13 ++++ .../_dev/deploy/docker/docker-compose.yml | 8 +++ .../ios/_dev/test/system/config.yml | 6 ++ .../ios/agent/stream/stream.yml.hbs | 3 - .../ios/agent/stream/syslog.yml.hbs | 3 - .../cisco/data_stream/ios/fields/agent.yml | 12 ++++ packages/cisco/data_stream/ios/fields/ecs.yml | 72 +++++++++++++++++++ .../cisco/data_stream/ios/fields/fields.yml | 9 +++ packages/cisco/data_stream/ios/manifest.yml | 8 --- packages/cisco/docs/README.md | 35 ++++++++- packages/cisco/manifest.yml | 2 +- 12 files changed, 160 insertions(+), 16 deletions(-) create mode 100644 packages/cisco/_dev/deploy/docker/Dockerfile create mode 100644 packages/cisco/_dev/deploy/docker/cisco-ios.log create mode 100644 packages/cisco/_dev/deploy/docker/docker-compose.yml create mode 100644 packages/cisco/data_stream/ios/_dev/test/system/config.yml diff --git a/packages/cisco/_dev/deploy/docker/Dockerfile b/packages/cisco/_dev/deploy/docker/Dockerfile new file mode 100644 index 00000000000..dd3d357204d --- /dev/null +++ b/packages/cisco/_dev/deploy/docker/Dockerfile @@ -0,0 +1,5 @@ +FROM alpine + +COPY ./cisco-ios.log /sample_logs/ + +ENTRYPOINT [ "/bin/sh" ] \ No newline at end of file diff --git a/packages/cisco/_dev/deploy/docker/cisco-ios.log b/packages/cisco/_dev/deploy/docker/cisco-ios.log new file mode 100644 index 00000000000..2b366c25af3 --- /dev/null +++ b/packages/cisco/_dev/deploy/docker/cisco-ios.log @@ -0,0 +1,13 @@ +Feb 8 04:00:48 198.51.100.2 585917: Feb 8 04:00:47.272: %SEC-6-IPACCESSLOGRP: list 177 denied igmp 198.51.100.197 -> 224.0.0.22, 1 packet +Feb 9 04:00:48 198.51.100.2 585918: Feb 9 04:00:47.272: %SEC-6-IPACCESSLOGSP: list INBOUND-ON-F11 denied igmp 198.51.100.2 -> 224.0.0.2 (20), 1 packet +Feb 10 04:00:48 198.51.100.2 585919: Feb 10 04:00:47.272: %SEC-6-IPACCESSLOGNP: list 171 denied 0 198.51.100.1 -> 255.255.255.255, 1 packet +May 3 19:11:33 198.51.100.2 585920: May 3 19:11:32.619: %IPV6-6-ACCESSLOGP: list ACL-IPv6-E0/0-IN/10 permitted tcp 2001:DB8::3(1027) -> 2001:DB8:1000::1(22), 9 packets +Jun 20 02:41:40 198.51.100.2 1663303: Jun 20 02:41:39.326: %SEC-6-IPACCESSLOGP: list 177 denied udp 198.51.100.195(55250) -> 198.51.100.255(15600), 1 packet +Jun 20 02:41:45 198.51.100.2 1663304: Jun 20 02:41:44.921: %SEC-6-IPACCESSLOGDP: list 151 denied icmp 198.51.100.1 -> 198.51.100.2 (3/4), 1 packet +Jun 20 02:42:28 198.51.100.2 1663312: Jun 20 02:42:27.342: %SEC-6-IPACCESSLOGP: list 177 denied udp 198.51.100.195(54309) -> 198.51.100.255(15600), 1 packet +Jun 20 02:42:28 198.51.100.2 1663313: Jun 20 02:42:28.374: %SEC-6-IPACCESSLOGRL: access-list logging rate-limited or missed 18 packets +Jun 20 02:42:34 198.51.100.2 1663314: Jun 20 02:42:33.340: %SEC-6-IPACCESSLOGP: list 177 denied udp 198.51.100.195(43989) -> 198.51.100.255(15600), 1 packet +Jun 20 02:43:09 198.51.100.2 1663321: Jun 20 02:43:08.454: %SEC-6-IPACCESSLOGP: list 150 denied tcp 198.51.100.12(59832) -> 172.217.10.46(80), 1 packet +Jun 20 02:43:29 198.51.100.2 1663325: Jun 20 02:43:28.403: %SEC-6-IPACCESSLOGRL: access-list logging rate-limited or missed 23 packets +Jun 20 02:43:29 198.51.100.2 1663326: Jun 20 02:43:28.403: %SEC-6-IPACCESSLOGDP: list 150 denied icmp 198.51.100.12 -> 198.51.100.1 (3/3), 32 packets +Jun 20 02:43:30 198.51.100.2 1663327: Jun 20 02:43:29.451: %SEC-6-IPACCESSLOGP: list 150 denied tcp 198.51.100.12(59834) -> 172.217.10.46(80), 1 packet diff --git a/packages/cisco/_dev/deploy/docker/docker-compose.yml b/packages/cisco/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..3e68f8a0c25 --- /dev/null +++ b/packages/cisco/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,8 @@ +version: '2.3' +services: + cisco: + tty: true + build: . + volumes: + - ${SERVICE_LOGS_DIR}:/logs + command: -c "cp /sample_logs/*.log /logs/" diff --git a/packages/cisco/data_stream/ios/_dev/test/system/config.yml b/packages/cisco/data_stream/ios/_dev/test/system/config.yml new file mode 100644 index 00000000000..103a9feead8 --- /dev/null +++ b/packages/cisco/data_stream/ios/_dev/test/system/config.yml @@ -0,0 +1,6 @@ +input: logfile +vars: ~ +data_stream: + vars: + paths: + - "{{SERVICE_LOGS_DIR}}/*.log" diff --git a/packages/cisco/data_stream/ios/agent/stream/stream.yml.hbs b/packages/cisco/data_stream/ios/agent/stream/stream.yml.hbs index 63b6038e556..b82f9f1a437 100644 --- a/packages/cisco/data_stream/ios/agent/stream/stream.yml.hbs +++ b/packages/cisco/data_stream/ios/agent/stream/stream.yml.hbs @@ -179,9 +179,6 @@ processors: {from: "source.port", type: "long"}, {from: "source.packets", type: "long"}, {from: "source.packets", to: "network.packets", type: "long"}, - {from: "icmp.type", type: "long"}, - {from: "icmp.code", type: "long"}, - {from: "igmp.type", type: "long"}, ], ignore_missing: true, }).Run; diff --git a/packages/cisco/data_stream/ios/agent/stream/syslog.yml.hbs b/packages/cisco/data_stream/ios/agent/stream/syslog.yml.hbs index 8d65c80610a..ca1920ca27c 100644 --- a/packages/cisco/data_stream/ios/agent/stream/syslog.yml.hbs +++ b/packages/cisco/data_stream/ios/agent/stream/syslog.yml.hbs @@ -176,9 +176,6 @@ processors: {from: "source.port", type: "long"}, {from: "source.packets", type: "long"}, {from: "source.packets", to: "network.packets", type: "long"}, - {from: "icmp.type", type: "long"}, - {from: "icmp.code", type: "long"}, - {from: "igmp.type", type: "long"}, ], ignore_missing: true, }).Run; diff --git a/packages/cisco/data_stream/ios/fields/agent.yml b/packages/cisco/data_stream/ios/fields/agent.yml index da4e652c53b..ba04c2fa7d8 100644 --- a/packages/cisco/data_stream/ios/fields/agent.yml +++ b/packages/cisco/data_stream/ios/fields/agent.yml @@ -196,3 +196,15 @@ description: > OS codename, if any. +- name: elastic.agent.id + type: keyword +- name: elastic.agent.snapshot + type: boolean +- name: elastic.agent.version + type: keyword +- name: input.type + type: keyword +- name: log.offset + type: long +- name: log.source.address + type: keyword diff --git a/packages/cisco/data_stream/ios/fields/ecs.yml b/packages/cisco/data_stream/ios/fields/ecs.yml index 0be29b4bfc7..b785e853fbf 100644 --- a/packages/cisco/data_stream/ios/fields/ecs.yml +++ b/packages/cisco/data_stream/ios/fields/ecs.yml @@ -51,3 +51,75 @@ - name: event.duration type: long description: Duration of the event in nanoseconds. +- name: destination.address + type: keyword + description: Destination network address. +- name: destination.as.number + type: long + description: Unique number allocated to the autonomous system. +- name: destination.as.organization.name + type: keyword + description: Organization name. +- name: destination.geo.continent_name + type: keyword + description: Name of the continent. +- name: destination.geo.country_iso_code + type: keyword + description: Country ISO code. +- name: destination.geo.country_name + type: keyword + description: Country name. +- name: destination.geo.location + type: geo_point + description: Longitude and latitude. +- name: destination.ip + type: ip + description: IP address of the destination. +- name: destination.port + type: long + description: Port of the destination. +- name: ecs.version + type: keyword + description: ECS version this event conforms to. +- name: log.file.path + type: keyword + description: Full path to the log file this event came from. +- name: log.level + type: keyword + description: Log level of the log event. +- name: log.original + type: keyword + description: Original log message with light interpretation only (encoding, newlines). +- name: network.community_id + type: keyword + description: A hash of source and destination IPs and ports. +- name: network.iana_number + type: keyword + description: IANA Protocol Number. +- name: network.packets + type: long + description: Total packets transferred in both directions. +- name: network.transport + type: keyword + description: Protocol Name corresponding to the field `iana_number`. +- name: network.type + type: keyword + description: In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc +- name: related.ip + type: ip + description: All of the IPs seen on your event. +- name: source.address + type: keyword + description: Source network address. +- name: source.ip + type: ip + description: IP address of the source. +- name: source.packets + type: long + description: Packets sent from the source to the destination. +- name: source.port + type: long + description: Port of the source. +- name: tags + type: keyword + description: List of keywords used to tag each event. diff --git a/packages/cisco/data_stream/ios/fields/fields.yml b/packages/cisco/data_stream/ios/fields/fields.yml index 07bea124b14..0b5ec6d7ee7 100644 --- a/packages/cisco/data_stream/ios/fields/fields.yml +++ b/packages/cisco/data_stream/ios/fields/fields.yml @@ -9,3 +9,12 @@ type: keyword description: | The facility to which the message refers (for example, SNMP, SYS, and so forth). A facility can be a hardware device, a protocol, or a module of the system software. It denotes the source or the cause of the system message. +- name: icmp.code + type: keyword + description: ICMP code. +- name: icmp.type + type: keyword + description: ICMP type. +- name: igmp.type + type: keyword + description: IGMP type. diff --git a/packages/cisco/data_stream/ios/manifest.yml b/packages/cisco/data_stream/ios/manifest.yml index bf491442dcd..40df260fade 100644 --- a/packages/cisco/data_stream/ios/manifest.yml +++ b/packages/cisco/data_stream/ios/manifest.yml @@ -7,14 +7,6 @@ streams: description: Collect Cisco IOS logs template_path: syslog.yml.hbs vars: - - name: paths - type: text - title: Paths - multi: true - required: true - show_user: true - default: - - /var/log/cisco-ios.log - name: tags type: text title: Tags diff --git a/packages/cisco/docs/README.md b/packages/cisco/docs/README.md index aa3ad5dcef3..4301c9dfa2f 100644 --- a/packages/cisco/docs/README.md +++ b/packages/cisco/docs/README.md @@ -187,9 +187,22 @@ The `ios` dataset collects the Cisco IOS router and switch logs. | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| destination.address | Destination network address. | keyword | +| destination.as.number | Unique number allocated to the autonomous system. | long | +| destination.as.organization.name | Organization name. | keyword | +| destination.geo.continent_name | Name of the continent. | keyword | +| destination.geo.country_iso_code | Country ISO code. | keyword | +| destination.geo.country_name | Country name. | keyword | +| destination.geo.location | Longitude and latitude. | geo_point | +| destination.ip | IP address of the destination. | ip | +| destination.port | Port of the destination. | long | +| ecs.version | ECS version this event conforms to. | keyword | +| elastic.agent.id | | keyword | +| elastic.agent.snapshot | | boolean | +| elastic.agent.version | | keyword | | event.category | Event category (e.g. database) | keyword | | event.code | Identification code for this event | keyword | -| event.created | The date/time when the event was first read by an agent, or by your pipeline. | date | +| event.created | Date/time when the event was first read by an agent, or by your pipeline. | date | | event.duration | Duration of the event in nanoseconds. | long | | event.end | The date when the event ended or when the activity was last observed. | keyword | | event.kind | Event kind (e.g. event) | keyword | @@ -213,8 +226,28 @@ The `ios` dataset collects the Cisco IOS router and switch logs. | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| icmp.code | ICMP code. | keyword | +| icmp.type | ICMP type. | keyword | +| igmp.type | IGMP type. | keyword | +| input.type | | keyword | | labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. | object | +| log.file.path | Full path to the log file this event came from. | keyword | +| log.level | Log level of the log event. | keyword | +| log.offset | | long | +| log.original | Original log message with light interpretation only (encoding, newlines). | keyword | +| log.source.address | | keyword | | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. | text | +| network.community_id | A hash of source and destination IPs and ports. | keyword | +| network.iana_number | IANA Protocol Number. | keyword | +| network.packets | Total packets transferred in both directions. | long | +| network.transport | Protocol Name corresponding to the field `iana_number`. | keyword | +| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| source.address | Source network address. | keyword | +| source.ip | IP address of the source. | ip | +| source.packets | Packets sent from the source to the destination. | long | +| source.port | Port of the source. | long | +| tags | List of keywords used to tag each event. | keyword | ### Nexus diff --git a/packages/cisco/manifest.yml b/packages/cisco/manifest.yml index acc8b6732d6..523d82bc664 100644 --- a/packages/cisco/manifest.yml +++ b/packages/cisco/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: cisco title: Cisco -version: 0.7.2 +version: 0.7.3 license: basic description: Cisco Integration type: integration