[8.8](backport #35581) sync filebeat system auth dataset with system …

…integration (#35597) * sync filebeat system auth dataset with system integration (#35581) - sync with version 1.29.0 of system integration (cherry picked from commit e870503) * fix changelog merge --------- Co-authored-by: Lee E Hinman <57081003+leehinman@users.noreply.github.com> Co-authored-by: Lee E. Hinman <lee.e.hinman@elastic.co>
elastic · May 30, 2023 · 1c4c61f · 1c4c61f
1 parent d5d2a8f
commit 1c4c61f
Show file tree

Hide file tree

Showing 5 changed files with 94 additions and 12 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -99,12 +99,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Fix crash when processing forwarded logs missing a message. {issue}34705[34705] {pull}34865[34865]
 - Fix crash when loading azurewebstorage cursor with no partially processed data. {pull}35433[35433]
 - Fix crash when processing forwarded logs missing a message. {issue}34705[34705] {pull}34865[34865]
-
-*Heartbeat*
-
-
-*Heartbeat*
-
+- [system] sync system/auth dataset with system integration 1.29.0. {pull}35581[35581]
 
 *Heartbeat*
 

diff --git a/filebeat/module/system/auth/ingest/pipeline.yml b/filebeat/module/system/auth/ingest/pipeline.yml
@@ -24,8 +24,8 @@ processors:
       field: _temp.message
       ignore_missing: true
       patterns:
-        - '^%{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user)?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?'
-        - '^%{DATA:system.auth.ssh.event} user %{DATA:user.name} from %{IPORHOST:source.ip}'
+        - '^%{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user)?%{DATA:user.name} from %{IPORHOST:source.address} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?'
+        - '^%{DATA:system.auth.ssh.event} user %{DATA:user.name} from %{IPORHOST:source.address}'
         - '^Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}'
         - '^%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}'
         - '^new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}'
@@ -44,9 +44,13 @@ processors:
       ignore_missing: true
       ignore_failure: true
       patterns:
-        - 'for user \"?%{DATA:_temp.foruser}\"? by \"?%{DATA:_temp.byuser}\"?(?:\(uid=%{NUMBER:_temp.byuid}\))?$'
-        - 'for user \"?%{DATA:_temp.foruser}\"?$'
-        - 'by user \"?%{DATA:_temp.byuser}\"?$'
+        - 'for user %{QUOTE}?%{DATA:_temp.foruser}%{QUOTE}? by %{QUOTE}?%{DATA:_temp.byuser}%{QUOTE}?(?:\(uid=%{NUMBER:_temp.byuid}\))?$'
+        - 'for user %{QUOTE}?%{DATA:_temp.foruser}%{QUOTE}?$'
+        - 'by user %{QUOTE}?%{DATA:_temp.byuser}%{QUOTE}?$'
+        - '%{BOUNDARY} user %{QUOTE}%{DATA:_temp.user}%{QUOTE}'
+      pattern_definitions:
+        QUOTE: "['\"]"
+        BOUNDARY: "(?<! )"
       if: ctx.message != null && ctx.message != ""
   - rename:
       field: _temp.byuser
@@ -64,6 +68,12 @@ processors:
       ignore_missing: true
       ignore_failure: true
       if: ctx.user?.name == null || ctx.user?.name == ""
+  - rename:
+      field: _temp.user
+      target_field: user.name
+      ignore_missing: true
+      ignore_failure: true
+      if: ctx.user?.name == null || ctx.user?.name == ""
   - rename:
       field: _temp.foruser
       target_field: user.effective.name
@@ -73,6 +83,16 @@ processors:
   - remove:
       field: _temp
       ignore_missing: true
+  - convert:
+      field: source.address
+      target_field: source.ip
+      type: ip
+      ignore_missing: true
+      on_failure:
+        - set:
+            field: source.domain
+            copy_from: source.address
+            ignore_failure: true
   - convert:
       field: system.auth.sudo.user
       target_field: user.effective.name
@@ -161,7 +181,11 @@ processors:
   - set:
       field: event.outcome
       value: success
-      if: ctx.process?.name != null && ['groupadd', 'groupdel', 'groupmod', 'useradd', 'userdel', 'usermod'].contains(ctx.process.name)
+      if: ctx.process?.name != null && (ctx.message == null || !ctx.message.contains("fail")) && ['groupadd', 'groupdel', 'groupmod', 'useradd', 'userdel', 'usermod'].contains(ctx.process.name)
+  - set:
+      field: event.outcome
+      value: failure
+      if: ctx.process?.name != null && (ctx.message != null && ctx.message.contains("fail")) && ['groupadd', 'groupdel', 'groupmod', 'useradd', 'userdel', 'usermod'].contains(ctx.process.name)
   - append:
       field: event.type
       value: user

diff --git a/filebeat/module/system/auth/test/secure-rhel7.log-expected.json b/filebeat/module/system/auth/test/secure-rhel7.log-expected.json
@@ -28,6 +28,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -89,6 +90,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -150,6 +152,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -279,6 +282,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -340,6 +344,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -401,6 +406,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -462,6 +468,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -523,6 +530,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -690,6 +698,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -751,6 +760,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "216.160.83.58",
         "source.as.number": 209,
         "source.geo.city_name": "Milton",
         "source.geo.continent_name": "North America",
@@ -816,6 +826,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -877,6 +888,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "216.160.83.58",
         "source.as.number": 209,
         "source.geo.city_name": "Milton",
         "source.geo.continent_name": "North America",
@@ -942,6 +954,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -1003,6 +1016,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "216.160.83.58",
         "source.as.number": 209,
         "source.geo.city_name": "Milton",
         "source.geo.continent_name": "North America",
@@ -1081,6 +1095,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -1142,6 +1157,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -1271,6 +1287,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -1332,6 +1349,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -1393,6 +1411,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -1454,6 +1473,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -1515,6 +1535,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -1644,6 +1665,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -1705,6 +1727,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -1766,6 +1789,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -1827,6 +1851,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -1888,6 +1913,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -2017,6 +2043,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "216.160.83.58",
         "source.as.number": 209,
         "source.geo.city_name": "Milton",
         "source.geo.continent_name": "North America",
@@ -2082,6 +2109,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "216.160.83.58",
         "source.as.number": 209,
         "source.geo.city_name": "Milton",
         "source.geo.continent_name": "North America",
@@ -2147,6 +2175,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "216.160.83.58",
         "source.as.number": 209,
         "source.geo.city_name": "Milton",
         "source.geo.continent_name": "North America",
@@ -2263,6 +2292,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -2324,6 +2354,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -2385,6 +2416,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -2446,6 +2478,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -2507,6 +2540,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",
@@ -2636,6 +2670,7 @@
             "root"
         ],
         "service.type": "system",
+        "source.address": "202.196.224.106",
         "source.geo.continent_name": "Asia",
         "source.geo.country_iso_code": "PH",
         "source.geo.country_name": "Philippines",

diff --git a/filebeat/module/system/auth/test/test.log b/filebeat/module/system/auth/test/test.log
@@ -8,3 +8,4 @@ Feb 23 00:08:48 localhost sudo: vagrant : TTY=pts/1 ; PWD=/home/vagrant ; USER=r
 Feb 24 00:13:02 precise32 sudo:      tsg : user NOT in sudoers ; TTY=pts/1 ; PWD=/home/vagrant ; USER=root ; COMMAND=/bin/ls
 Feb 22 11:47:05 localhost groupadd[6991]: new group: name=apache, GID=48
 Feb 22 11:47:05 localhost useradd[6995]: new user: name=apache, UID=48, GID=48, home=/usr/share/httpd, shell=/sbin/nologin
+Feb 22 12:53:50 localhost sshd[10161]: error: PAM: User not known to the underlying authentication module for illegal user test from test.example.com