From 5ff223350266fcf4740ab1696d1f28e417af8488 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 30 Apr 2020 12:59:56 -0400 Subject: [PATCH] Skip add_host_metadata for forwarded event logs Update config examples to use the "forwarded" tag to skip adding host metadata. Also disable host.name being added by libbeat. This field was overwritten by the winlog.computer_name so it didn't serve any purpose to have libbeat set it. Relates #13920 --- CHANGELOG.next.asciidoc | 1 + dev-tools/mage/config.go | 1 + libbeat/_meta/config.yml.tmpl | 3 ++- winlogbeat/_meta/beat.yml.tmpl | 17 +++++------------ winlogbeat/_meta/common.yml.tmpl | 6 ++++++ winlogbeat/cmd/root.go | 7 ++++++- winlogbeat/scripts/mage/config.go | 3 ++- winlogbeat/winlogbeat.reference.yml | 11 ++--------- winlogbeat/winlogbeat.yml | 8 ++++---- x-pack/winlogbeat/_meta/beat.yml.tmpl | 14 ++++++++++++++ x-pack/winlogbeat/winlogbeat.reference.yml | 14 ++++++++++++++ x-pack/winlogbeat/winlogbeat.yml | 20 ++++++++++++++++---- 12 files changed, 73 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 78a1385e589..3fb367f063a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -212,6 +212,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add Kerberos support to Elasticsearch output. {pull}17927[17927] - Add support for fixed length extraction in `dissect` processor. {pull}17191[17191] - Set `agent.name` to the hostname by default. {issue}16377[16377] {pull}18000[18000] +- Add config example of how to skip the `add_host_metadata` processor when forwarding logs. {issue}13920[13920] {pull}18153[18153] *Auditbeat* diff --git a/dev-tools/mage/config.go b/dev-tools/mage/config.go index 668c71e1074..6cabac5c9b6 100644 --- a/dev-tools/mage/config.go +++ b/dev-tools/mage/config.go @@ -116,6 +116,7 @@ func Config(types ConfigFileType, args ConfigFileParams, targetDir string) error "UseDockerMetadataProcessor": true, "UseKubernetesMetadataProcessor": false, "ExcludeDashboards": false, + "UseProcessorsTemplate": false, } for k, v := range args.ExtraVars { params[k] = v diff --git a/libbeat/_meta/config.yml.tmpl b/libbeat/_meta/config.yml.tmpl index 2d5e510e33f..3cf0e3e00af 100644 --- a/libbeat/_meta/config.yml.tmpl +++ b/libbeat/_meta/config.yml.tmpl @@ -90,6 +90,7 @@ output.elasticsearch: #ssl.key: "/etc/pki/client/cert.key" {{end}} #================================ Processors ===================================== +{{if .UseProcessorsTemplate}}{{template "processors.yml.tmpl" .}}{{else -}} {{if not .UseObserverProcessor}} # Configure processors to enhance or manipulate events generated by the beat. @@ -112,7 +113,7 @@ processors: #name: us-east-1a # Lat, Lon " #location: "37.926868, -78.024902" -{{end}} +{{end}}{{end}} #================================ Logging ===================================== # Sets log level. The default log level is info. diff --git a/winlogbeat/_meta/beat.yml.tmpl b/winlogbeat/_meta/beat.yml.tmpl index 24b27d36648..093c6f69c04 100644 --- a/winlogbeat/_meta/beat.yml.tmpl +++ b/winlogbeat/_meta/beat.yml.tmpl @@ -2,18 +2,11 @@ winlogbeat.event_logs: - name: Application ignore_older: 72h -{{if .Reference}} - # Set to true to publish fields with null values in events. - #keep_null: false -{{end}} + - name: System -{{if .Reference}} - # Set to true to publish fields with null values in events. - #keep_null: false -{{end}} + - name: Security -{{if .Reference}} - # Set to true to publish fields with null values in events. - #keep_null: false -{{end}} + + - name: ForwardedEvents + tags: [forwarded] {{if not .Reference}}{{ template "elasticsearch_settings" . }}{{end}} diff --git a/winlogbeat/_meta/common.yml.tmpl b/winlogbeat/_meta/common.yml.tmpl index 7a7feaddf5a..63aa30fa0b5 100644 --- a/winlogbeat/_meta/common.yml.tmpl +++ b/winlogbeat/_meta/common.yml.tmpl @@ -34,3 +34,9 @@ setup.template.settings: #index.codec: best_compression #_source.enabled: false {{end -}} +{{define "processors.yml.tmpl"}} +processors: + - add_host_metadata: + when.not.contains.tags: forwarded + - add_cloud_metadata: ~ +{{end -}} diff --git a/winlogbeat/cmd/root.go b/winlogbeat/cmd/root.go index 7075a51aeb0..ecc3aa8e38f 100644 --- a/winlogbeat/cmd/root.go +++ b/winlogbeat/cmd/root.go @@ -20,6 +20,7 @@ package cmd import ( "github.com/elastic/beats/v7/libbeat/cmd" "github.com/elastic/beats/v7/libbeat/cmd/instance" + "github.com/elastic/beats/v7/libbeat/publisher/processing" "github.com/elastic/beats/v7/winlogbeat/beater" // Register fields. @@ -35,4 +36,8 @@ import ( var Name = "winlogbeat" // RootCmd to handle beats cli -var RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{Name: Name, HasDashboards: true}) +var RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{ + Name: Name, + HasDashboards: true, + Processing: processing.MakeDefaultSupport(true, processing.WithECS, processing.WithAgentMeta()), +}) diff --git a/winlogbeat/scripts/mage/config.go b/winlogbeat/scripts/mage/config.go index 1dae96510dd..70cc8cb43a6 100644 --- a/winlogbeat/scripts/mage/config.go +++ b/winlogbeat/scripts/mage/config.go @@ -54,7 +54,8 @@ func configFileParams() devtools.ConfigFileParams { devtools.LibbeatDir("_meta/config.docker.yml"), }, ExtraVars: map[string]interface{}{ - "GOOS": "windows", + "GOOS": "windows", + "UseProcessorsTemplate": true, }, } } diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 471b6c4e7fc..39e22d25e34 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -26,19 +26,12 @@ winlogbeat.event_logs: - name: Application ignore_older: 72h - # Set to true to publish fields with null values in events. - #keep_null: false - - name: System - # Set to true to publish fields with null values in events. - #keep_null: false - - name: Security - # Set to true to publish fields with null values in events. - #keep_null: false - + - name: ForwardedEvents + tags: [forwarded] #================================ General ====================================== diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml index 8887e8d75c7..d816327de99 100644 --- a/winlogbeat/winlogbeat.yml +++ b/winlogbeat/winlogbeat.yml @@ -25,6 +25,8 @@ winlogbeat.event_logs: - name: Security + - name: ForwardedEvents + tags: [forwarded] #==================== Elasticsearch template settings ========================== setup.template.settings: @@ -125,12 +127,10 @@ output.elasticsearch: #================================ Processors ===================================== -# Configure processors to enhance or manipulate events generated by the beat. - processors: - - add_host_metadata: ~ + - add_host_metadata: + when.not.contains.tags: forwarded - add_cloud_metadata: ~ - - add_docker_metadata: ~ #================================ Logging ===================================== diff --git a/x-pack/winlogbeat/_meta/beat.yml.tmpl b/x-pack/winlogbeat/_meta/beat.yml.tmpl index f2660df68bd..1ea8cdcc879 100644 --- a/x-pack/winlogbeat/_meta/beat.yml.tmpl +++ b/x-pack/winlogbeat/_meta/beat.yml.tmpl @@ -19,4 +19,18 @@ winlogbeat.event_logs: id: sysmon file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + - name: ForwardedEvents + tags: [forwarded] + processors: + - script: + when.equals.winlog.channel: Security + lang: javascript + id: security + file: ${path.home}/module/security/config/winlogbeat-security.js + - script: + when.equals.winlog.channel: Microsoft-Windows-Sysmon/Operational + lang: javascript + id: sysmon + file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + {{if not .Reference}}{{ template "elasticsearch_settings" . }}{{end}} diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index c8643d904ab..3fd2ffcba4f 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -42,6 +42,20 @@ winlogbeat.event_logs: id: sysmon file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + - name: ForwardedEvents + tags: [forwarded] + processors: + - script: + when.equals.winlog.channel: Security + lang: javascript + id: security + file: ${path.home}/module/security/config/winlogbeat-security.js + - script: + when.equals.winlog.channel: Microsoft-Windows-Sysmon/Operational + lang: javascript + id: sysmon + file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + #================================ General ====================================== diff --git a/x-pack/winlogbeat/winlogbeat.yml b/x-pack/winlogbeat/winlogbeat.yml index bc5dbc294d2..e718fb91d41 100644 --- a/x-pack/winlogbeat/winlogbeat.yml +++ b/x-pack/winlogbeat/winlogbeat.yml @@ -37,6 +37,20 @@ winlogbeat.event_logs: id: sysmon file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + - name: ForwardedEvents + tags: [forwarded] + processors: + - script: + when.equals.winlog.channel: Security + lang: javascript + id: security + file: ${path.home}/module/security/config/winlogbeat-security.js + - script: + when.equals.winlog.channel: Microsoft-Windows-Sysmon/Operational + lang: javascript + id: sysmon + file: ${path.home}/module/sysmon/config/winlogbeat-sysmon.js + #==================== Elasticsearch template settings ========================== setup.template.settings: @@ -137,12 +151,10 @@ output.elasticsearch: #================================ Processors ===================================== -# Configure processors to enhance or manipulate events generated by the beat. - processors: - - add_host_metadata: ~ + - add_host_metadata: + when.not.contains.tags: forwarded - add_cloud_metadata: ~ - - add_docker_metadata: ~ #================================ Logging =====================================