From 1f1dd81555847134a5e923a29a44032eb1f296e0 Mon Sep 17 00:00:00 2001 From: MarNicGit <47538428+MarNicGit@users.noreply.github.com> Date: Thu, 10 Nov 2022 11:46:21 +0100 Subject: [PATCH] Promtail: Exclude event message (#7462) **What this PR does / why we need it**: Windows Event Logs have an Event Message field that is intended for human eyes, and often contains data that already present in the event data XML. Omitting this field the same way we can already omit `user_data` and `event_data` can easily save a lot of bytes of data per event - Event ID 4264 alone has ~2KB of just text that is already present in `event_data`. **Which issue(s) this PR fixes**: Fixes #7395 **Special notes for your reviewer**: I also took the liberty to improve upon the existing test '`Test_renderEntries` by using unique values for each field rather than 10's everywhere. I expect this to conflict with my other PR, #7461. --- CHANGELOG.md | 1 + .../pkg/promtail/scrapeconfig/scrapeconfig.go | 3 ++ .../pkg/promtail/targets/windows/format.go | 4 +- .../promtail/targets/windows/target_test.go | 54 +++++++++++++++++-- .../sources/clients/promtail/configuration.md | 3 ++ 5 files changed, 59 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01c5e7586694..c47a36d3acf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,6 +100,7 @@ Check the history of the branch FIXME. * [6656](https://github.com/grafana/loki/pull/6656) **carlospeon**: Allow promtail to add matches to the journal reader * [7401](https://github.com/grafana/loki/pull/7401) **thepalbi**: Add timeout to GCP Logs push target * [7414](https://github.com/grafana/loki/pull/7414) **thepalbi**: Add basic tracing support +* [7462](https://github.com/grafana/loki/pull/7462) **MarNicGit**: Allow excluding event message from Windows Event Log entries. ##### Fixes * [7394](https://github.com/grafana/loki/pull/7394) **liguozhong**: Fix issue with the Cloudflare target that caused it to stop working after it received an error in the logpull request as explained in issue https://github.com/grafana/loki/issues/6150 diff --git a/clients/pkg/promtail/scrapeconfig/scrapeconfig.go b/clients/pkg/promtail/scrapeconfig/scrapeconfig.go index fa99768db0f3..ec8b0daa6c4a 100644 --- a/clients/pkg/promtail/scrapeconfig/scrapeconfig.go +++ b/clients/pkg/promtail/scrapeconfig/scrapeconfig.go @@ -236,6 +236,9 @@ type WindowsEventsTargetConfig struct { // ExcludeEventData allows to exclude the xml event data. ExcludeEventData bool `yaml:"exclude_event_data"` + // ExcludeEventMessage allows to exclude the human-friendly message contained in each windows event. + ExcludeEventMessage bool `yaml:"exclude_event_message"` + // ExcludeUserData allows to exclude the user data of each windows event. ExcludeUserData bool `yaml:"exclude_user_data"` diff --git a/clients/pkg/promtail/targets/windows/format.go b/clients/pkg/promtail/targets/windows/format.go index 03c83ef54ce1..9fc44cc62a8b 100644 --- a/clients/pkg/promtail/targets/windows/format.go +++ b/clients/pkg/promtail/targets/windows/format.go @@ -73,7 +73,6 @@ func formatLine(cfg *scrapeconfig.WindowsEventsTargetConfig, event win_eventlog. Keywords: event.Keywords, TimeCreated: event.TimeCreated.SystemTime, EventRecordID: event.EventRecordID, - Message: event.Message, } if !cfg.ExcludeEventData { @@ -82,6 +81,9 @@ func formatLine(cfg *scrapeconfig.WindowsEventsTargetConfig, event win_eventlog. if !cfg.ExcludeUserData { structuredEvent.UserData = string(event.UserData.InnerXML) } + if !cfg.ExcludeEventMessage { + structuredEvent.Message = event.Message + } if event.Correlation.ActivityID != "" || event.Correlation.RelatedActivityID != "" { structuredEvent.Correlation = &Correlation{ ActivityID: event.Correlation.ActivityID, diff --git a/clients/pkg/promtail/targets/windows/target_test.go b/clients/pkg/promtail/targets/windows/target_test.go index fffc123af550..932630b93a0b 100644 --- a/clients/pkg/promtail/targets/windows/target_test.go +++ b/clients/pkg/promtail/targets/windows/target_test.go @@ -134,10 +134,10 @@ func Test_renderEntries(t *testing.T) { { Source: win_eventlog.Provider{Name: "Application"}, EventID: 10, - Version: 10, - Level: 10, - Task: 10, - Opcode: 10, + Version: 20, + Level: 30, + Task: 40, + Opcode: 50, Keywords: "keywords", TimeCreated: win_eventlog.TimeCreated{SystemTime: time.Unix(0, 1).UTC().Format(time.RFC3339Nano)}, EventRecordID: 11, @@ -156,7 +156,51 @@ func Test_renderEntries(t *testing.T) { Labels: model.LabelSet{"channel": "channel", "computer": "local", "job": "windows-events"}, Entry: logproto.Entry{ Timestamp: time.Unix(0, 1).UTC(), - Line: `{"source":"Application","channel":"channel","computer":"local","event_id":10,"version":10,"level":10,"task":10,"opCode":10,"keywords":"keywords","timeCreated":"1970-01-01T00:00:00.000000001Z","eventRecordID":11,"correlation":{"activityID":"some activity","relatedActivityID":"some related activity"},"execution":{"processId":1,"threadId":5},"security":{"userId":"1"},"user_data":"userdata","event_data":"eventdata","message":"message"}`, + Line: `{"source":"Application","channel":"channel","computer":"local","event_id":10,"version":20,"level":30,"task":40,"opCode":50,"keywords":"keywords","timeCreated":"1970-01-01T00:00:00.000000001Z","eventRecordID":11,"correlation":{"activityID":"some activity","relatedActivityID":"some related activity"},"execution":{"processId":1,"threadId":5},"security":{"userId":"1"},"user_data":"userdata","event_data":"eventdata","message":"message"}`, + }, + }, + }, entries) +} + +func Test_renderEntries_ExcludeEventMessage(t *testing.T) { + client := fake.New(func() {}) + defer client.Stop() + ta, err := New(util_log.Logger, client, nil, &scrapeconfig.WindowsEventsTargetConfig{ + Labels: model.LabelSet{"job": "windows-events"}, + EventlogName: "Application", + Query: "*", + UseIncomingTimestamp: true, + ExcludeEventMessage: true, + }) + require.NoError(t, err) + defer ta.Stop() + entries := ta.renderEntries([]win_eventlog.Event{ + { + Source: win_eventlog.Provider{Name: "Application"}, + EventID: 10, + Version: 20, + Level: 30, + Task: 40, + Opcode: 50, + Keywords: "keywords", + TimeCreated: win_eventlog.TimeCreated{SystemTime: time.Unix(0, 1).UTC().Format(time.RFC3339Nano)}, + EventRecordID: 11, + Correlation: win_eventlog.Correlation{ActivityID: "some activity", RelatedActivityID: "some related activity"}, + Execution: win_eventlog.Execution{ThreadID: 5, ProcessID: 1}, + Channel: "channel", + Computer: "local", + Security: win_eventlog.Security{UserID: "1"}, + UserData: win_eventlog.UserData{InnerXML: []byte(`userdata`)}, + EventData: win_eventlog.EventData{InnerXML: []byte(`eventdata`)}, + Message: "message", + }, + }) + require.Equal(t, []api.Entry{ + { + Labels: model.LabelSet{"channel": "channel", "computer": "local", "job": "windows-events"}, + Entry: logproto.Entry{ + Timestamp: time.Unix(0, 1).UTC(), + Line: `{"source":"Application","channel":"channel","computer":"local","event_id":10,"version":20,"level":30,"task":40,"opCode":50,"keywords":"keywords","timeCreated":"1970-01-01T00:00:00.000000001Z","eventRecordID":11,"correlation":{"activityID":"some activity","relatedActivityID":"some related activity"},"execution":{"processId":1,"threadId":5},"security":{"userId":"1"},"user_data":"userdata","event_data":"eventdata"}`, }, }, }, entries) diff --git a/docs/sources/clients/promtail/configuration.md b/docs/sources/clients/promtail/configuration.md index a4aab1b3a9e6..a72c56bbdbad 100644 --- a/docs/sources/clients/promtail/configuration.md +++ b/docs/sources/clients/promtail/configuration.md @@ -931,6 +931,9 @@ You can add additional labels with the `labels` property. # Allows to exclude the xml event data. [exclude_event_data: | default = false] +# Allows to exclude the human-friendly event message. +[exclude_event_message: | default = false] + # Allows to exclude the user data of each windows event. [exclude_user_data: | default = false]