diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 78a1385e589..3fb367f063a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -212,6 +212,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add Kerberos support to Elasticsearch output. {pull}17927[17927] - Add support for fixed length extraction in `dissect` processor. {pull}17191[17191] - Set `agent.name` to the hostname by default. {issue}16377[16377] {pull}18000[18000] +- Add config example of how to skip the `add_host_metadata` processor when forwarding logs. {issue}13920[13920] {pull}18153[18153] *Auditbeat* diff --git a/dev-tools/mage/config.go b/dev-tools/mage/config.go index 668c71e1074..6cabac5c9b6 100644 --- a/dev-tools/mage/config.go +++ b/dev-tools/mage/config.go @@ -116,6 +116,7 @@ func Config(types ConfigFileType, args ConfigFileParams, targetDir string) error "UseDockerMetadataProcessor": true, "UseKubernetesMetadataProcessor": false, "ExcludeDashboards": false, + "UseProcessorsTemplate": false, } for k, v := range args.ExtraVars { params[k] = v diff --git a/libbeat/_meta/config.yml.tmpl b/libbeat/_meta/config.yml.tmpl index 2d5e510e33f..3cf0e3e00af 100644 --- a/libbeat/_meta/config.yml.tmpl +++ b/libbeat/_meta/config.yml.tmpl @@ -90,6 +90,7 @@ output.elasticsearch: #ssl.key: "/etc/pki/client/cert.key" {{end}} #================================ Processors ===================================== +{{if .UseProcessorsTemplate}}{{template "processors.yml.tmpl" .}}{{else -}} {{if not .UseObserverProcessor}} # Configure processors to enhance or manipulate events generated by the beat. @@ -112,7 +113,7 @@ processors: #name: us-east-1a # Lat, Lon " #location: "37.926868, -78.024902" -{{end}} +{{end}}{{end}} #================================ Logging ===================================== # Sets log level. The default log level is info. diff --git a/winlogbeat/_meta/beat.yml.tmpl b/winlogbeat/_meta/beat.yml.tmpl index 24b27d36648..093c6f69c04 100644 --- a/winlogbeat/_meta/beat.yml.tmpl +++ b/winlogbeat/_meta/beat.yml.tmpl @@ -2,18 +2,11 @@ winlogbeat.event_logs: - name: Application ignore_older: 72h -{{if .Reference}} - # Set to true to publish fields with null values in events. - #keep_null: false -{{end}} + - name: System -{{if .Reference}} - # Set to true to publish fields with null values in events. - #keep_null: false -{{end}} + - name: Security -{{if .Reference}} - # Set to true to publish fields with null values in events. - #keep_null: false -{{end}} + + - name: ForwardedEvents + tags: [forwarded] {{if not .Reference}}{{ template "elasticsearch_settings" . }}{{end}} diff --git a/winlogbeat/_meta/common.yml.tmpl b/winlogbeat/_meta/common.yml.tmpl index 7a7feaddf5a..63aa30fa0b5 100644 --- a/winlogbeat/_meta/common.yml.tmpl +++ b/winlogbeat/_meta/common.yml.tmpl @@ -34,3 +34,9 @@ setup.template.settings: #index.codec: best_compression #_source.enabled: false {{end -}} +{{define "processors.yml.tmpl"}} +processors: + - add_host_metadata: + when.not.contains.tags: forwarded + - add_cloud_metadata: ~ +{{end -}} diff --git a/winlogbeat/cmd/root.go b/winlogbeat/cmd/root.go index 7075a51aeb0..ecc3aa8e38f 100644 --- a/winlogbeat/cmd/root.go +++ b/winlogbeat/cmd/root.go @@ -20,6 +20,7 @@ package cmd import ( "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" + "github.com/elastic/beats/v7/libbeat/publisher/processing" "github.com/elastic/beats/v7/winlogbeat/beater" // Register fields. @@ -35,4 +36,8 @@ import ( var Name = "winlogbeat" // RootCmd to handle beats cli -var RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{Name: Name, HasDashboards: true}) +var RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{ + Name: Name, + HasDashboards: true, + Processing: processing.MakeDefaultSupport(true, processing.WithECS, processing.WithAgentMeta()), +}) diff --git a/winlogbeat/scripts/mage/config.go b/winlogbeat/scripts/mage/config.go index 1dae96510dd..70cc8cb43a6 100644 --- a/winlogbeat/scripts/mage/config.go +++ b/winlogbeat/scripts/mage/config.go @@ -54,7 +54,8 @@ func configFileParams() devtools.ConfigFileParams { devtools.LibbeatDir("_meta/config.docker.yml"), }, ExtraVars: map[string]interface{}{ - "GOOS": "windows", + "GOOS": "windows", + "UseProcessorsTemplate": true, }, } } diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 471b6c4e7fc..39e22d25e34 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -26,19 +26,12 @@ winlogbeat.event_logs: - name: Application ignore_older: 72h - # Set to true to publish fields with null values in events. - #keep_null: false - - name: System - # Set to true to publish fields with null values in events. - #keep_null: false - - name: Security - # Set to true to publish fields with null values in events. - #keep_null: false - + - name: ForwardedEvents + tags: [forwarded] #================================ General ====================================== diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml index 8887e8d75c7..d816327de99 100644 --- a/winlogbeat/winlogbeat.yml +++ b/winlogbeat/winlogbeat.yml @@ -25,6 +25,8 @@ winlogbeat.event_logs: - name: Security + - name: ForwardedEvents + tags: [forwarded] #==================== Elasticsearch template settings ========================== setup.template.settings: @@ -125,12 +127,10 @@ output.elasticsearch: #================================ Processors ===================================== -# Configure processors to enhance or manipulate events generated by the beat. - processors: - - add_host_metadata: ~ + - add_host_metadata: + when.not.contains.tags: forwarded - add_cloud_metadata: ~ - - add_docker_metadata: ~ #================================ Logging ===================================== diff --git a/x-pack/winlogbeat/_meta/beat.yml.tmpl b/x-pack/winlogbeat/_meta/beat.yml.tmpl index f2660df68bd..1ea8cdcc879 100644 --- a/x-pack/winlogbeat/_meta/beat.yml.tmpl +++ b/x-pack/winlogbeat/_meta/beat.yml.tmpl @@ -19,4 +19,18 @@ winlogbeat.event_logs: id: sysmon file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + - name: ForwardedEvents + tags: [forwarded] + processors: + - script: + when.equals.winlog.channel: Security + lang: javascript + id: security + file: ${path.home}/module/security/config/winlogbeat-security.js + - script: + when.equals.winlog.channel: Microsoft-Windows-Sysmon/Operational + lang: javascript + id: sysmon + file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + {{if not .Reference}}{{ template "elasticsearch_settings" . }}{{end}} diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index c8643d904ab..3fd2ffcba4f 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -42,6 +42,20 @@ winlogbeat.event_logs: id: sysmon file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + - name: ForwardedEvents + tags: [forwarded] + processors: + - script: + when.equals.winlog.channel: Security + lang: javascript + id: security + file: ${path.home}/module/security/config/winlogbeat-security.js + - script: + when.equals.winlog.channel: Microsoft-Windows-Sysmon/Operational + lang: javascript + id: sysmon + file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + #================================ General ====================================== diff --git a/x-pack/winlogbeat/winlogbeat.yml b/x-pack/winlogbeat/winlogbeat.yml index bc5dbc294d2..e718fb91d41 100644 --- a/x-pack/winlogbeat/winlogbeat.yml +++ b/x-pack/winlogbeat/winlogbeat.yml @@ -37,6 +37,20 @@ winlogbeat.event_logs: id: sysmon file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + - name: ForwardedEvents + tags: [forwarded] + processors: + - script: + when.equals.winlog.channel: Security + lang: javascript + id: security + file: ${path.home}/module/security/config/winlogbeat-security.js + - script: + when.equals.winlog.channel: Microsoft-Windows-Sysmon/Operational + lang: javascript + id: sysmon + file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + #==================== Elasticsearch template settings ========================== setup.template.settings: @@ -137,12 +151,10 @@ output.elasticsearch: #================================ Processors ===================================== -# Configure processors to enhance or manipulate events generated by the beat. - processors: - - add_host_metadata: ~ + - add_host_metadata: + when.not.contains.tags: forwarded - add_cloud_metadata: ~ - - add_docker_metadata: ~ #================================ Logging =====================================