Modified auditd ingest pipeline to handle node=hostname (elastic#19659)

This change permits auditd logs to be ingested whether or not they begin with a node=. Previous to this change a type= was expected. It handles the case where name_format (man 5 auditd.conf) is a value other than none. Also adding 10 lines of resulting audit log with name_format = hostname parameter on CentOS 7. Co-authored-by: Marc Guasch <marc.guasch@elastic.co>
melchiormoulin · Oct 14, 2020 · 83b6521 · 83b6521
1 parent 9f0697a
commit 83b6521
Show file tree

Hide file tree

Showing 4 changed files with 233 additions and 2 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -511,13 +511,15 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add experimental dataset squid/log for Squid Proxy Server logs {pull}19713[19713]
 - Add experimental dataset zscaler/zia for Zscaler Internet Access logs {pull}19713[19713]
 - Add initial support for configurable file identity tracking. {pull}18748[18748]
+- Add support for reading auditd logs that are prefixed with `node=`. {pull}19659[19659]
 - Add event.ingested for CrowdStrike module {pull}20138[20138]
 - Add support for additional fields and FirewallMatchEvent type events in CrowdStrike module {pull}20138[20138]
 - Add event.ingested for Suricata module {pull}20220[20220]
 - Add support for custom header and headersecret for filebeat http_endpoint input {pull}20435[20435]
 - Add event.ingested to all Filebeat modules. {pull}20386[20386]
 - Return error when log harvester tries to open a named pipe. {issue}18682[18682] {pull}20450[20450]
 
+
 *Heartbeat*
 
 - Allow a list of status codes for HTTP checks. {pull}15587[15587]

diff --git a/filebeat/module/auditd/log/ingest/pipeline.yml b/filebeat/module/auditd/log/ingest/pipeline.yml
@@ -7,8 +7,9 @@ processors:
 - grok:
     field: message
     pattern_definitions:
-      AUDIT_TYPE: "^type=%{NOTSPACE:auditd.log.record_type}"
-      AUDIT_PREFIX: "%{AUDIT_TYPE} msg=audit\\(%{NUMBER:auditd.log.epoch}:%{NUMBER:auditd.log.sequence}\\):(%{DATA})?"
+      AUDIT_TYPE: "type=%{NOTSPACE:auditd.log.record_type}"
+      AUDIT_NODE: "node=%{IPORHOST:auditd.log.node} "
+      AUDIT_PREFIX: "^(?:%{AUDIT_NODE})?%{AUDIT_TYPE} msg=audit\\(%{NUMBER:auditd.log.epoch}:%{NUMBER:auditd.log.sequence}\\):(%{DATA})?"
       AUDIT_KEY_VALUES: "%{WORD}=%{GREEDYDATA}"
     patterns:
     - "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv} old auid=%{NUMBER:auditd.log.old_auid}

diff --git a/filebeat/module/auditd/log/test/audit-cent7-node.log b/filebeat/module/auditd/log/test/audit-cent7-node.log
@@ -0,0 +1,10 @@
+node=localhost.localdomain type=DAEMON_START msg=audit(1594053514.588:4686): op=start ver=2.8.5 format=raw kernel=3.10.0-1062.9.1.el7.x86_64 auid=4294967295 pid=1643 uid=0 ses=4294967295 subj=system_u:system_r:auditd_t:s0 res=success
+node=localhost.localdomain type=CONFIG_CHANGE msg=audit(1594053514.707:4): audit_backlog_limit=8192 old=64 auid=4294967295 ses=4294967295 subj=system_u:system_r:unconfined_service_t:s0 res=1
+node=localhost.localdomain type=CONFIG_CHANGE msg=audit(1594053514.707:5): audit_failure=1 old=1 auid=4294967295 ses=4294967295 subj=system_u:system_r:unconfined_service_t:s0 res=1
+node=localhost.localdomain type=SERVICE_START msg=audit(1594053514.709:6): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=auditd comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success'
+node=localhost.localdomain type=SYSTEM_BOOT msg=audit(1594053514.725:7): pid=1667 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg=' comm="systemd-update-utmp" exe="/usr/lib/systemd/systemd-update-utmp" hostname=? addr=? terminal=? res=success'
+node=localhost.localdomain type=SERVICE_START msg=audit(1594053514.739:8): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=systemd-update-utmp comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success'
+node=localhost.localdomain type=SERVICE_START msg=audit(1594053514.807:9): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=rngd comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success'
+node=localhost.localdomain type=SERVICE_START msg=audit(1594053514.843:10): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=irqbalance comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success'
+node=localhost.localdomain type=SERVICE_START msg=audit(1594053514.850:11): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=abrtd comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success'
+node=localhost.localdomain type=SERVICE_START msg=audit(1594053514.857:12): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='unit=abrt-xorg comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success'
diff --git a/filebeat/module/auditd/log/test/audit-cent7-node.log-expected.json b/filebeat/module/auditd/log/test/audit-cent7-node.log-expected.json
@@ -0,0 +1,218 @@
+[
+    {
+        "@timestamp": "2020-07-06T16:38:34.588Z",
+        "auditd.log.format": "raw",
+        "auditd.log.kernel": "3.10.0-1062.9.1.el7.x86_64",
+        "auditd.log.node": "localhost.localdomain",
+        "auditd.log.op": "start",
+        "auditd.log.sequence": 4686,
+        "auditd.log.ses": "4294967295",
+        "auditd.log.subj": "system_u:system_r:auditd_t:s0",
+        "auditd.log.ver": "2.8.5",
+        "event.action": "daemon_start",
+        "event.dataset": "auditd.log",
+        "event.kind": "event",
+        "event.module": "auditd",
+        "event.outcome": "success",
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.offset": 0,
+        "process.pid": 1643,
+        "service.type": "auditd",
+        "user.audit.id": "4294967295",
+        "user.id": "0"
+    },
+    {
+        "@timestamp": "2020-07-06T16:38:34.707Z",
+        "auditd.log.audit_backlog_limit": "8192",
+        "auditd.log.node": "localhost.localdomain",
+        "auditd.log.old": "64",
+        "auditd.log.sequence": 4,
+        "auditd.log.ses": "4294967295",
+        "auditd.log.subj": "system_u:system_r:unconfined_service_t:s0",
+        "event.action": "config_change",
+        "event.dataset": "auditd.log",
+        "event.kind": "event",
+        "event.module": "auditd",
+        "event.outcome": "1",
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.offset": 234,
+        "service.type": "auditd",
+        "user.audit.id": "4294967295"
+    },
+    {
+        "@timestamp": "2020-07-06T16:38:34.707Z",
+        "auditd.log.audit_failure": "1",
+        "auditd.log.node": "localhost.localdomain",
+        "auditd.log.old": "1",
+        "auditd.log.sequence": 5,
+        "auditd.log.ses": "4294967295",
+        "auditd.log.subj": "system_u:system_r:unconfined_service_t:s0",
+        "event.action": "config_change",
+        "event.dataset": "auditd.log",
+        "event.kind": "event",
+        "event.module": "auditd",
+        "event.outcome": "1",
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.offset": 425,
+        "service.type": "auditd",
+        "user.audit.id": "4294967295"
+    },
+    {
+        "@timestamp": "2020-07-06T16:38:34.709Z",
+        "auditd.log.node": "localhost.localdomain",
+        "auditd.log.sequence": 6,
+        "auditd.log.ses": "4294967295",
+        "auditd.log.subj": "system_u:system_r:init_t:s0",
+        "auditd.log.unit": "auditd",
+        "event.action": "service_start",
+        "event.dataset": "auditd.log",
+        "event.kind": "event",
+        "event.module": "auditd",
+        "event.outcome": "success",
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.offset": 606,
+        "process.executable": "/usr/lib/systemd/systemd",
+        "process.name": "systemd",
+        "process.pid": 1,
+        "service.type": "auditd",
+        "user.audit.id": "4294967295",
+        "user.id": "0"
+    },
+    {
+        "@timestamp": "2020-07-06T16:38:34.725Z",
+        "auditd.log.node": "localhost.localdomain",
+        "auditd.log.sequence": 7,
+        "auditd.log.ses": "4294967295",
+        "auditd.log.subj": "system_u:system_r:init_t:s0",
+        "event.action": "system_boot",
+        "event.dataset": "auditd.log",
+        "event.kind": "event",
+        "event.module": "auditd",
+        "event.outcome": "success",
+        "event.type": "host",
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.offset": 862,
+        "message": "",
+        "process.executable": "/usr/lib/systemd/systemd-update-utmp",
+        "process.name": "systemd-update-utmp",
+        "process.pid": 1667,
+        "service.type": "auditd",
+        "user.audit.id": "4294967295",
+        "user.id": "0"
+    },
+    {
+        "@timestamp": "2020-07-06T16:38:34.739Z",
+        "auditd.log.node": "localhost.localdomain",
+        "auditd.log.sequence": 8,
+        "auditd.log.ses": "4294967295",
+        "auditd.log.subj": "system_u:system_r:init_t:s0",
+        "auditd.log.unit": "systemd-update-utmp",
+        "event.action": "service_start",
+        "event.dataset": "auditd.log",
+        "event.kind": "event",
+        "event.module": "auditd",
+        "event.outcome": "success",
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.offset": 1132,
+        "process.executable": "/usr/lib/systemd/systemd",
+        "process.name": "systemd",
+        "process.pid": 1,
+        "service.type": "auditd",
+        "user.audit.id": "4294967295",
+        "user.id": "0"
+    },
+    {
+        "@timestamp": "2020-07-06T16:38:34.807Z",
+        "auditd.log.node": "localhost.localdomain",
+        "auditd.log.sequence": 9,
+        "auditd.log.ses": "4294967295",
+        "auditd.log.subj": "system_u:system_r:init_t:s0",
+        "auditd.log.unit": "rngd",
+        "event.action": "service_start",
+        "event.dataset": "auditd.log",
+        "event.kind": "event",
+        "event.module": "auditd",
+        "event.outcome": "success",
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.offset": 1401,
+        "process.executable": "/usr/lib/systemd/systemd",
+        "process.name": "systemd",
+        "process.pid": 1,
+        "service.type": "auditd",
+        "user.audit.id": "4294967295",
+        "user.id": "0"
+    },
+    {
+        "@timestamp": "2020-07-06T16:38:34.843Z",
+        "auditd.log.node": "localhost.localdomain",
+        "auditd.log.sequence": 10,
+        "auditd.log.ses": "4294967295",
+        "auditd.log.subj": "system_u:system_r:init_t:s0",
+        "auditd.log.unit": "irqbalance",
+        "event.action": "service_start",
+        "event.dataset": "auditd.log",
+        "event.kind": "event",
+        "event.module": "auditd",
+        "event.outcome": "success",
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.offset": 1655,
+        "process.executable": "/usr/lib/systemd/systemd",
+        "process.name": "systemd",
+        "process.pid": 1,
+        "service.type": "auditd",
+        "user.audit.id": "4294967295",
+        "user.id": "0"
+    },
+    {
+        "@timestamp": "2020-07-06T16:38:34.850Z",
+        "auditd.log.node": "localhost.localdomain",
+        "auditd.log.sequence": 11,
+        "auditd.log.ses": "4294967295",
+        "auditd.log.subj": "system_u:system_r:init_t:s0",
+        "auditd.log.unit": "abrtd",
+        "event.action": "service_start",
+        "event.dataset": "auditd.log",
+        "event.kind": "event",
+        "event.module": "auditd",
+        "event.outcome": "success",
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.offset": 1916,
+        "process.executable": "/usr/lib/systemd/systemd",
+        "process.name": "systemd",
+        "process.pid": 1,
+        "service.type": "auditd",
+        "user.audit.id": "4294967295",
+        "user.id": "0"
+    },
+    {
+        "@timestamp": "2020-07-06T16:38:34.857Z",
+        "auditd.log.node": "localhost.localdomain",
+        "auditd.log.sequence": 12,
+        "auditd.log.ses": "4294967295",
+        "auditd.log.subj": "system_u:system_r:init_t:s0",
+        "auditd.log.unit": "abrt-xorg",
+        "event.action": "service_start",
+        "event.dataset": "auditd.log",
+        "event.kind": "event",
+        "event.module": "auditd",
+        "event.outcome": "success",
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.offset": 2172,
+        "process.executable": "/usr/lib/systemd/systemd",
+        "process.name": "systemd",
+        "process.pid": 1,
+        "service.type": "auditd",
+        "user.audit.id": "4294967295",
+        "user.id": "0"
+    }
+]