Test packages mappings - DO NOT MERGE

.buildkite/pipeline.yml

@@ -8,6 +8,7 @@ env:
   YQ_VERSION: 'v4.35.2'
   JQ_VERSION: '1.7'
   GH_CLI_VERSION: "2.29.0"
+  STACK_VERSION: "8.18.0-SNAPSHOT"
 
   # Agent images used in pipeline steps
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
@@ -30,6 +31,8 @@ env:
   ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}"
   # Disable checking for newer versions
   ELASTIC_PACKAGE_CHECK_UPDATE_DISABLED: "true"
+  # Select method to validate fields are documented
+  ELASTIC_PACKAGE_FIELD_VALIDATION_TEST_METHOD: "mappings"
 
 steps:
   - label: "Get reference from target branch"

.buildkite/scripts/common.sh

@@ -757,7 +757,7 @@ teardown_test_package() {
 }
 
 list_all_directories() {
-    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort
+    find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |grep -E '^(teleport|cef|mimecast|box_events|ti_anomali|claroty_ctd|sublime_security|crowdstrike|auditd_manager|mongodb_atlas|awsfirehose|linux|envoyproxy|ti_custom|ti_abusech|github|tychon|wiz)$'
 }
 
 check_package() {

go.mod

@@ -231,3 +231,5 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+replace github.com/elastic/elastic-package => github.com/mrodm/elastic-package v0.53.1-0.20250205092747-a82e4e12f12a

go.sum

@@ -125,8 +125,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0 h1:sx1lpZuTG5suJuvgix4FWQFCLFFbzkoOmPoHWYOPLCY=
 github.com/elastic/elastic-integration-corpus-generator-tool v0.10.0/go.mod h1:2/30n+2QRzRzus4TPVUV1T3U/j8g2ItUgvP0pcpjLGk=
-github.com/elastic/elastic-package v0.109.1 h1:ATZVgYOCI6L5Yr0NxjSX+MsuK4UvXkpu9tDkO4K2vgo=
-github.com/elastic/elastic-package v0.109.1/go.mod h1:vmVYISfxBrl0ejjKbm/AG0drjrmevysVg2ZIP7yewLo=
 github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
 github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
 github.com/elastic/go-licenser v0.4.2 h1:bPbGm8bUd8rxzSswFOqvQh1dAkKGkgAmrPxbUi+Y9+A=
@@ -372,6 +370,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
+github.com/mrodm/elastic-package v0.53.1-0.20250205092747-a82e4e12f12a h1:JtX6aMz9BtUskoPLUW6LF1lGxzPZ4kEEzXE2KJneSFg=
+github.com/mrodm/elastic-package v0.53.1-0.20250205092747-a82e4e12f12a/go.mod h1:vmVYISfxBrl0ejjKbm/AG0drjrmevysVg2ZIP7yewLo=
 github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
 github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
 github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo=

packages/auditd_manager/data_stream/auditd/fields/fields.yml

@@ -623,9 +623,11 @@
 - name: auditd.data.perm_mask
   description: file permission mask that triggered a watch event
   type: keyword
-- name: auditd.data.a0-N
-  description: the arguments to a syscall
-  type: keyword
+# this mapping does not generate a dynamic template, and the expected fields do not match
+# should it be kept for documentation purposes?
+# - name: auditd.data.a0-N
+#   description: the arguments to a syscall
+#   type: keyword
 - name: auditd.data.ses
   description: login session ID
   type: keyword
@@ -737,6 +739,6 @@
   type: keyword
 - name: auditd.data.result
   type: keyword
-- name: auditd.data
+- name: auditd.data.*
   description: Auditd related data
-  type: flattened
+  type: keyword

packages/auditd_manager/data_stream/auditd/sample_event.json

@@ -1,22 +1,22 @@
 {
-    "@timestamp": "2022-05-12T13:10:13.230Z",
+    "@timestamp": "2025-01-14T18:00:56.117Z",
     "agent": {
-        "ephemeral_id": "cfe4170e-f9b4-435f-b19c-a0e75b573b3a",
-        "id": "753ce520-4f32-45b1-9212-c4dcc9d575a1",
-        "name": "custom-agent",
+        "ephemeral_id": "df24f825-df17-4f54-b3c4-3048edbdf123",
+        "id": "da084743-3b4d-43eb-a6c9-f26c44204375",
+        "name": "elastic-agent-90019",
         "type": "auditbeat",
-        "version": "8.2.0"
+        "version": "8.16.0"
     },
     "auditd": {
         "data": {
-            "a0": "a",
-            "a1": "c00024e8c0",
-            "a2": "38",
+            "a0": "10",
+            "a1": "c001144140",
+            "a2": "3c",
             "a3": "0",
             "arch": "x86_64",
-            "audit_pid": "22501",
+            "audit_pid": 2532842,
             "auid": "unset",
-            "exit": "56",
+            "exit": "60",
             "old": "0",
             "op": "set",
             "result": "success",
@@ -25,23 +25,24 @@
                 "family": "netlink",
                 "saddr": "100000000000000000000000"
             },
+            "subj_user": "docker-default",
             "syscall": "sendto",
             "tty": "(none)"
         },
         "message_type": "config_change",
         "messages": [
-            "type=CONFIG_CHANGE msg=audit(1652361013.230:94471): op=set audit_pid=22501 old=0 auid=4294967295 ses=4294967295 res=1",
-            "type=SYSCALL msg=audit(1652361013.230:94471): arch=c000003e syscall=44 success=yes exit=56 a0=a a1=c00024e8c0 a2=38 a3=0 items=0 ppid=9509 pid=22501 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"auditbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat\" key=(null)",
-            "type=SOCKADDR msg=audit(1652361013.230:94471): saddr=100000000000000000000000",
-            "type=PROCTITLE msg=audit(1652361013.230:94471): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D6239613238612F696E7374616C6C2F6175646974626561742D382E322E302D6C696E75782D7838365F36342F617564697462656174002D63006175646974626561742E656C61737469632D6167656E742E796D6C"
+            "type=CONFIG_CHANGE msg=audit(1736877656.117:1197107): op=set audit_pid=2532842 old=0 auid=4294967295 ses=4294967295 subj=docker-default res=1",
+            "type=SYSCALL msg=audit(1736877656.117:1197107): arch=c000003e syscall=44 success=yes exit=60 a0=10 a1=c001144140 a2=3c a3=0 items=0 ppid=2531521 pid=2532842 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"agentbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat\" subj=docker-default key=(null)",
+            "type=SOCKADDR msg=audit(1736877656.117:1197107): saddr=100000000000000000000000",
+            "type=PROCTITLE msg=audit(1736877656.117:1197107): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D3366303766322F636F6D706F6E656E74732F6167656E746265617400617564697462656174002D450073657475702E696C6D2E656E61626C65643D66616C7365002D450073657475702E74656D706C6174652E65"
         ],
         "result": "success",
         "summary": {
             "actor": {
                 "primary": "unset",
                 "secondary": "root"
             },
-            "how": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat",
+            "how": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat",
             "object": {
                 "primary": "set",
                 "type": "audit-config"
@@ -63,21 +64,24 @@
                 },
                 "id": "0",
                 "name": "root"
+            },
+            "selinux": {
+                "user": "docker-default"
             }
         }
     },
     "data_stream": {
         "dataset": "auditd_manager.auditd",
-        "namespace": "ep",
+        "namespace": "73800",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "753ce520-4f32-45b1-9212-c4dcc9d575a1",
+        "id": "da084743-3b4d-43eb-a6c9-f26c44204375",
         "snapshot": false,
-        "version": "8.2.0"
+        "version": "8.16.0"
     },
     "event": {
         "action": "changed-audit-configuration",
@@ -88,32 +92,50 @@
             "network"
         ],
         "dataset": "auditd_manager.auditd",
-        "ingested": "2022-05-12T13:10:16Z",
+        "ingested": "2025-01-14T18:00:59Z",
         "kind": "event",
         "module": "auditd",
-        "original": "type=CONFIG_CHANGE msg=audit(1652361013.230:94471): op=set audit_pid=22501 old=0 auid=4294967295 ses=4294967295 res=1\ntype=SYSCALL msg=audit(1652361013.230:94471): arch=c000003e syscall=44 success=yes exit=56 a0=a a1=c00024e8c0 a2=38 a3=0 items=0 ppid=9509 pid=22501 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"auditbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat\" key=(null)\ntype=SOCKADDR msg=audit(1652361013.230:94471): saddr=100000000000000000000000\ntype=PROCTITLE msg=audit(1652361013.230:94471): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D6239613238612F696E7374616C6C2F6175646974626561742D382E322E302D6C696E75782D7838365F36342F617564697462656174002D63006175646974626561742E656C61737469632D6167656E742E796D6C",
+        "original": "type=CONFIG_CHANGE msg=audit(1736877656.117:1197107): op=set audit_pid=2532842 old=0 auid=4294967295 ses=4294967295 subj=docker-default res=1\ntype=SYSCALL msg=audit(1736877656.117:1197107): arch=c000003e syscall=44 success=yes exit=60 a0=10 a1=c001144140 a2=3c a3=0 items=0 ppid=2531521 pid=2532842 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"agentbeat\" exe=\"/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat\" subj=docker-default key=(null)\ntype=SOCKADDR msg=audit(1736877656.117:1197107): saddr=100000000000000000000000\ntype=PROCTITLE msg=audit(1736877656.117:1197107): proctitle=2F7573722F73686172652F656C61737469632D6167656E742F646174612F656C61737469632D6167656E742D3366303766322F636F6D706F6E656E74732F6167656E746265617400617564697462656174002D450073657475702E696C6D2E656E61626C65643D66616C7365002D450073657475702E74656D706C6174652E65",
         "outcome": "success",
-        "sequence": 94471,
+        "sequence": 1197107,
         "type": [
             "change",
             "connection",
             "info"
         ]
     },
     "host": {
-        "name": "custom-agent"
+        "architecture": "x86_64",
+        "containerized": false,
+        "hostname": "elastic-agent-90019",
+        "ip": [
+            "192.168.176.2",
+            "192.168.144.5"
+        ],
+        "mac": [
+            "02-42-C0-A8-90-05",
+            "02-42-C0-A8-B0-02"
+        ],
+        "name": "elastic-agent-90019",
+        "os": {
+            "kernel": "6.8.0-51-generic",
+            "name": "Wolfi",
+            "platform": "wolfi",
+            "type": "linux",
+            "version": "20230201"
+        }
     },
     "network": {
         "direction": "egress"
     },
     "process": {
-        "executable": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat",
-        "name": "auditbeat",
+        "executable": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat",
+        "name": "agentbeat",
         "parent": {
-            "pid": 9509
+            "pid": 2531521
         },
-        "pid": 22501,
-        "title": "/usr/share/elastic-agent/data/elastic-agent-b9a28a/install/auditbeat-8.2.0-linux-x86_64/auditbeat -c auditbeat.elastic-agent.yml"
+        "pid": 2532842,
+        "title": "/usr/share/elastic-agent/data/elastic-agent-3f07f2/components/agentbeat auditbeat -E setup.ilm.enabled=false -E setup.template.e"
     },
     "service": {
         "type": "auditd"
@@ -130,4 +152,4 @@
         "id": "0",
         "name": "root"
     }
-}
+}