From 6fb1d2b7b8eadd6e4f1046814761496fa12d9559 Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Thu, 25 Mar 2021 11:31:44 -0500 Subject: [PATCH 1/5] [Heartbeat] Merge synthetic root fields into events Fixes https://github.com/elastic/beats/issues/24768 This allows synthetics to drive more field names without requiring heartbeat updates. Any fields in `root_fields` get merged into the event root. --- .../monitors/browser/synthexec/synthtypes.go | 29 +++++++++------- .../browser/synthexec/synthtypes_test.go | 34 +++++++++++++++++++ 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go index 4d293d86d72..3c3d11c1b93 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go @@ -16,30 +16,33 @@ import ( ) type SynthEvent struct { - Type string `json:"type"` - PackageVersion string `json:"package_version"` - Step *Step `json:"step"` - Journey *Journey `json:"journey"` - TimestampEpochMicros float64 `json:"@timestamp"` - Payload common.MapStr `json:"payload"` - Blob string `json:"blob"` - BlobMime string `json:"blob_mime"` - Error *SynthError `json:"error"` - URL string `json:"url"` - Status string `json:"status"` + Type string `json:"type"` + PackageVersion string `json:"package_version"` + Step *Step `json:"step"` + Journey *Journey `json:"journey"` + TimestampEpochMicros float64 `json:"@timestamp"` + Payload common.MapStr `json:"payload"` + Blob string `json:"blob"` + BlobMime string `json:"blob_mime"` + Error *SynthError `json:"error"` + URL string `json:"url"` + Status string `json:"status"` + RootFields map[string]interface{} `json:"root_fields"` index int } func (se SynthEvent) ToMap() (m common.MapStr) { // We don't add @timestamp to the map string since that's specially handled in beat.Event - m = common.MapStr{ + // Use the root fields as a base, and layer additional, stricter, fields on top + m = se.RootFields + m.DeepUpdate(common.MapStr{ "synthetics": common.MapStr{ "type": se.Type, "package_version": se.PackageVersion, "payload": se.Payload, "index": se.index, }, - } + }) if se.Blob != "" { m.Put("synthetics.blob", se.Blob) } diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go index 775c5380137..cf875b05387 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go @@ -5,9 +5,14 @@ package synthexec import ( + "encoding/json" "testing" "time" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/go-lookslike" + "github.com/elastic/go-lookslike/testslike" + "github.com/stretchr/testify/require" ) @@ -15,3 +20,32 @@ func TestSynthEventTimestamp(t *testing.T) { se := SynthEvent{TimestampEpochMicros: 1000} // 1ms require.Equal(t, time.Unix(0, int64(time.Millisecond)), se.Timestamp()) } + +func TestRootFields(t *testing.T) { + // Actually marshal to JSON and back to test the struct tags for deserialization from JSON + source := common.MapStr{ + "type": "journey/start", + "root_fields": map[string]interface{}{ + "synthetics": map[string]interface{}{ + "nested": "v1", + }, + "truly_at_root": "v2", + }, + } + jsonBytes, err := json.Marshal(source) + require.NoError(t, err) + se := &SynthEvent{} + err = json.Unmarshal(jsonBytes, se) + require.NoError(t, err) + + m := se.ToMap() + + // Test that even deep maps merge correctly + testslike.Test(t, lookslike.MustCompile(common.MapStr{ + "synthetics": common.MapStr{ + "type": "journey/start", + "nested": "v1", + }, + "truly_at_root": "v2", + }), m) +} From a0eff45146d44001c3fbbcba3ecfaf780586c665 Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Thu, 25 Mar 2021 11:35:35 -0500 Subject: [PATCH 2/5] changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 275ccc09166..11fcdb407f5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -812,6 +812,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add mime type detection for http responses. {pull}22976[22976] - Handle datastreams for fleet. {pull}24223[24223] - Add --sandbox option for browser monitor. {pull}24172[24172] +- Support additional 'root' fields from synthetics. {pull}24770[24770] *Journalbeat* From 5416288f8d8b5232aff1d965cbba2f7561e81744 Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Thu, 25 Mar 2021 17:01:36 -0500 Subject: [PATCH 3/5] Fix null case --- x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go index 3c3d11c1b93..ab5f1761dcf 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go @@ -34,7 +34,11 @@ type SynthEvent struct { func (se SynthEvent) ToMap() (m common.MapStr) { // We don't add @timestamp to the map string since that's specially handled in beat.Event // Use the root fields as a base, and layer additional, stricter, fields on top - m = se.RootFields + if se.RootFields != nil { + m = se.RootFields + } else { + m = common.MapStr{} + } m.DeepUpdate(common.MapStr{ "synthetics": common.MapStr{ "type": se.Type, From b55fd0a11d16b7e6987215673ee746412f37fc88 Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Thu, 25 Mar 2021 21:34:09 -0500 Subject: [PATCH 4/5] Fix URL behavior, add tests --- .../monitors/browser/synthexec/synthtypes.go | 16 ++- .../browser/synthexec/synthtypes_test.go | 135 ++++++++++++++---- .../sample-synthetics-config/heartbeat.yml | 6 - 3 files changed, 124 insertions(+), 33 deletions(-) diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go index ab5f1761dcf..ccec990d355 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go @@ -27,7 +27,7 @@ type SynthEvent struct { Error *SynthError `json:"error"` URL string `json:"url"` Status string `json:"status"` - RootFields map[string]interface{} `json:"root_fields"` + RootFields common.MapStr `json:"root_fields"` index int } @@ -36,17 +36,27 @@ func (se SynthEvent) ToMap() (m common.MapStr) { // Use the root fields as a base, and layer additional, stricter, fields on top if se.RootFields != nil { m = se.RootFields + // We handle url specially since it can be passed as a string, + // but expanded to match ECS + if urlStr, ok := m["url"].(string); ok { + if se.URL == "" { + se.URL = urlStr + } + } } else { m = common.MapStr{} } + m.DeepUpdate(common.MapStr{ "synthetics": common.MapStr{ "type": se.Type, "package_version": se.PackageVersion, - "payload": se.Payload, "index": se.index, }, }) + if len(se.Payload) > 0 { + m.Put("synthetics.payload", se.Payload) + } if se.Blob != "" { m.Put("synthetics.blob", se.Blob) } @@ -68,7 +78,7 @@ func (se SynthEvent) ToMap() (m common.MapStr) { if e != nil { logp.Warn("Could not parse synthetics URL '%s': %s", se.URL, e.Error()) } else { - m["url"] = wrappers.URLFields(u) + m.Put("url", wrappers.URLFields(u)) } } diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go index cf875b05387..7cd10b421a7 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go @@ -6,6 +6,8 @@ package synthexec import ( "encoding/json" + "github.com/elastic/beats/v7/heartbeat/monitors/wrappers" + "net/url" "testing" "time" @@ -21,31 +23,116 @@ func TestSynthEventTimestamp(t *testing.T) { require.Equal(t, time.Unix(0, int64(time.Millisecond)), se.Timestamp()) } -func TestRootFields(t *testing.T) { - // Actually marshal to JSON and back to test the struct tags for deserialization from JSON - source := common.MapStr{ - "type": "journey/start", - "root_fields": map[string]interface{}{ - "synthetics": map[string]interface{}{ - "nested": "v1", +func TestToMap(t *testing.T) { + testUrl, _ := url.Parse("http://testurl") + + type testCase struct { + name string + source common.MapStr + expected common.MapStr + } + + testCases := []testCase{ + { + "root fields with URL", + common.MapStr{ + "type": "journey/start", + "package_version": "1.2.3", + "root_fields": map[string]interface{}{ + "synthetics": map[string]interface{}{ + "nested": "v1", + }, + "truly_at_root": "v2", + }, + "url": testUrl.String(), + }, + common.MapStr{ + "synthetics": common.MapStr{ + "type": "journey/start", + "package_version": "1.2.3", + "nested": "v1", + }, + "url": wrappers.URLFields(testUrl), + "truly_at_root": "v2", }, - "truly_at_root": "v2", }, - } - jsonBytes, err := json.Marshal(source) - require.NoError(t, err) - se := &SynthEvent{} - err = json.Unmarshal(jsonBytes, se) - require.NoError(t, err) - - m := se.ToMap() - - // Test that even deep maps merge correctly - testslike.Test(t, lookslike.MustCompile(common.MapStr{ - "synthetics": common.MapStr{ - "type": "journey/start", - "nested": "v1", + { + "root fields, step metadata", + common.MapStr{ + "type": "step/start", + "package_version": "1.2.3", + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "status": "success", "index": 42}, + "root_fields": map[string]interface{}{ + "synthetics": map[string]interface{}{ + "nested": "v1", + }, + "truly_at_root": "v2", + }, + }, + common.MapStr{ + "synthetics": common.MapStr{ + "type": "step/start", + "package_version": "1.2.3", + "nested": "v1", + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "status": "success", "index": 42}, + }, + "truly_at_root": "v2", + }, + }, + { + "weird error, and blob, no URL", + common.MapStr{ + "type": "someType", + "package_version": "1.2.3", + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "index": 42, "status": "down"}, + "error": common.MapStr{ + "name": "MyErrorName", + "message": "MyErrorMessage", + "stack": "MyErrorStack", + }, + "blob": "ablob", + "blob_mime": "application/weird", + }, + common.MapStr{ + "synthetics": common.MapStr{ + "type": "someType", + "package_version": "1.2.3", + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "index": 42, "status": "down"}, + "error": common.MapStr{ + "name": "MyErrorName", + "message": "MyErrorMessage", + "stack": "MyErrorStack", + }, + "blob": "ablob", + "blob_mime": "application/weird", + }, + }, }, - "truly_at_root": "v2", - }), m) + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Actually marshal to JSON and back to test the struct tags for deserialization from JSON + jsonBytes, err := json.Marshal(tc.source) + require.NoError(t, err) + se := &SynthEvent{} + err = json.Unmarshal(jsonBytes, se) + require.NoError(t, err) + + m := se.ToMap() + + // Index will always be zero in thee tests, so helpfully include it + llvalidator := lookslike.Strict(lookslike.Compose( + lookslike.MustCompile(tc.expected), + lookslike.MustCompile(common.MapStr{"synthetics": common.MapStr{"index": 0}}), + )) + + // Test that even deep maps merge correctly + testslike.Test(t, llvalidator, m) + }) + } } diff --git a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml index 74fc2f4d885..79e18851c9b 100644 --- a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml +++ b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml @@ -9,8 +9,6 @@ heartbeat.monitors: enabled: true id: todos-suite name: Todos Suite - data_stream: - namespace: myns source: local: path: "/home/andrewvc/projects/synthetics/examples/todos/" @@ -21,14 +19,10 @@ heartbeat.monitors: urls: http://www.google.com schedule: "@every 15s" name: Simple HTTP - data_stream: - namespace: myns - type: browser enabled: false id: my-monitor name: My Monitor - data_stream: - namespace: myns source: inline: script: From cba90819f1feb6dfbbbf562d1aca1cb019f567d7 Mon Sep 17 00:00:00 2001 From: Andrew Cholakian Date: Fri, 26 Mar 2021 11:29:27 -0500 Subject: [PATCH 5/5] Fix formatting --- .../monitors/browser/synthexec/synthtypes.go | 24 ++++----- .../browser/synthexec/synthtypes_test.go | 49 ++++++++++--------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go index ccec990d355..40cf2e06242 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go @@ -16,18 +16,18 @@ import ( ) type SynthEvent struct { - Type string `json:"type"` - PackageVersion string `json:"package_version"` - Step *Step `json:"step"` - Journey *Journey `json:"journey"` - TimestampEpochMicros float64 `json:"@timestamp"` - Payload common.MapStr `json:"payload"` - Blob string `json:"blob"` - BlobMime string `json:"blob_mime"` - Error *SynthError `json:"error"` - URL string `json:"url"` - Status string `json:"status"` - RootFields common.MapStr `json:"root_fields"` + Type string `json:"type"` + PackageVersion string `json:"package_version"` + Step *Step `json:"step"` + Journey *Journey `json:"journey"` + TimestampEpochMicros float64 `json:"@timestamp"` + Payload common.MapStr `json:"payload"` + Blob string `json:"blob"` + BlobMime string `json:"blob_mime"` + Error *SynthError `json:"error"` + URL string `json:"url"` + Status string `json:"status"` + RootFields common.MapStr `json:"root_fields"` index int } diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go index 7cd10b421a7..daa2a710900 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go @@ -6,11 +6,12 @@ package synthexec import ( "encoding/json" - "github.com/elastic/beats/v7/heartbeat/monitors/wrappers" "net/url" "testing" "time" + "github.com/elastic/beats/v7/heartbeat/monitors/wrappers" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/go-lookslike" "github.com/elastic/go-lookslike/testslike" @@ -36,7 +37,7 @@ func TestToMap(t *testing.T) { { "root fields with URL", common.MapStr{ - "type": "journey/start", + "type": "journey/start", "package_version": "1.2.3", "root_fields": map[string]interface{}{ "synthetics": map[string]interface{}{ @@ -48,21 +49,21 @@ func TestToMap(t *testing.T) { }, common.MapStr{ "synthetics": common.MapStr{ - "type": "journey/start", + "type": "journey/start", "package_version": "1.2.3", - "nested": "v1", + "nested": "v1", }, - "url": wrappers.URLFields(testUrl), + "url": wrappers.URLFields(testUrl), "truly_at_root": "v2", }, }, { "root fields, step metadata", common.MapStr{ - "type": "step/start", + "type": "step/start", "package_version": "1.2.3", - "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, - "step": common.MapStr{"name": "MyStep", "status": "success", "index": 42}, + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "status": "success", "index": 42}, "root_fields": map[string]interface{}{ "synthetics": map[string]interface{}{ "nested": "v1", @@ -72,11 +73,11 @@ func TestToMap(t *testing.T) { }, common.MapStr{ "synthetics": common.MapStr{ - "type": "step/start", + "type": "step/start", "package_version": "1.2.3", - "nested": "v1", - "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, - "step": common.MapStr{"name": "MyStep", "status": "success", "index": 42}, + "nested": "v1", + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "status": "success", "index": 42}, }, "truly_at_root": "v2", }, @@ -84,30 +85,30 @@ func TestToMap(t *testing.T) { { "weird error, and blob, no URL", common.MapStr{ - "type": "someType", + "type": "someType", "package_version": "1.2.3", - "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, - "step": common.MapStr{"name": "MyStep", "index": 42, "status": "down"}, + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "index": 42, "status": "down"}, "error": common.MapStr{ - "name": "MyErrorName", + "name": "MyErrorName", "message": "MyErrorMessage", - "stack": "MyErrorStack", + "stack": "MyErrorStack", }, - "blob": "ablob", + "blob": "ablob", "blob_mime": "application/weird", }, common.MapStr{ "synthetics": common.MapStr{ - "type": "someType", + "type": "someType", "package_version": "1.2.3", - "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, - "step": common.MapStr{"name": "MyStep", "index": 42, "status": "down"}, + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "index": 42, "status": "down"}, "error": common.MapStr{ - "name": "MyErrorName", + "name": "MyErrorName", "message": "MyErrorMessage", - "stack": "MyErrorStack", + "stack": "MyErrorStack", }, - "blob": "ablob", + "blob": "ablob", "blob_mime": "application/weird", }, },