diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 275ccc09166..11fcdb407f5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -812,6 +812,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add mime type detection for http responses. {pull}22976[22976] - Handle datastreams for fleet. {pull}24223[24223] - Add --sandbox option for browser monitor. {pull}24172[24172] +- Support additional 'root' fields from synthetics. {pull}24770[24770] *Journalbeat* diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go index 4d293d86d72..40cf2e06242 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go @@ -27,18 +27,35 @@ type SynthEvent struct { Error *SynthError `json:"error"` URL string `json:"url"` Status string `json:"status"` + RootFields common.MapStr `json:"root_fields"` index int } func (se SynthEvent) ToMap() (m common.MapStr) { // We don't add @timestamp to the map string since that's specially handled in beat.Event - m = common.MapStr{ + // Use the root fields as a base, and layer additional, stricter, fields on top + if se.RootFields != nil { + m = se.RootFields + // We handle url specially since it can be passed as a string, + // but expanded to match ECS + if urlStr, ok := m["url"].(string); ok { + if se.URL == "" { + se.URL = urlStr + } + } + } else { + m = common.MapStr{} + } + + m.DeepUpdate(common.MapStr{ "synthetics": common.MapStr{ "type": se.Type, "package_version": se.PackageVersion, - "payload": se.Payload, "index": se.index, }, + }) + if len(se.Payload) > 0 { + m.Put("synthetics.payload", se.Payload) } if se.Blob != "" { m.Put("synthetics.blob", se.Blob) @@ -61,7 +78,7 @@ func (se SynthEvent) ToMap() (m common.MapStr) { if e != nil { logp.Warn("Could not parse synthetics URL '%s': %s", se.URL, e.Error()) } else { - m["url"] = wrappers.URLFields(u) + m.Put("url", wrappers.URLFields(u)) } } diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go index 775c5380137..daa2a710900 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go @@ -5,9 +5,17 @@ package synthexec import ( + "encoding/json" + "net/url" "testing" "time" + "github.com/elastic/beats/v7/heartbeat/monitors/wrappers" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/go-lookslike" + "github.com/elastic/go-lookslike/testslike" + "github.com/stretchr/testify/require" ) @@ -15,3 +23,117 @@ func TestSynthEventTimestamp(t *testing.T) { se := SynthEvent{TimestampEpochMicros: 1000} // 1ms require.Equal(t, time.Unix(0, int64(time.Millisecond)), se.Timestamp()) } + +func TestToMap(t *testing.T) { + testUrl, _ := url.Parse("http://testurl") + + type testCase struct { + name string + source common.MapStr + expected common.MapStr + } + + testCases := []testCase{ + { + "root fields with URL", + common.MapStr{ + "type": "journey/start", + "package_version": "1.2.3", + "root_fields": map[string]interface{}{ + "synthetics": map[string]interface{}{ + "nested": "v1", + }, + "truly_at_root": "v2", + }, + "url": testUrl.String(), + }, + common.MapStr{ + "synthetics": common.MapStr{ + "type": "journey/start", + "package_version": "1.2.3", + "nested": "v1", + }, + "url": wrappers.URLFields(testUrl), + "truly_at_root": "v2", + }, + }, + { + "root fields, step metadata", + common.MapStr{ + "type": "step/start", + "package_version": "1.2.3", + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "status": "success", "index": 42}, + "root_fields": map[string]interface{}{ + "synthetics": map[string]interface{}{ + "nested": "v1", + }, + "truly_at_root": "v2", + }, + }, + common.MapStr{ + "synthetics": common.MapStr{ + "type": "step/start", + "package_version": "1.2.3", + "nested": "v1", + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "status": "success", "index": 42}, + }, + "truly_at_root": "v2", + }, + }, + { + "weird error, and blob, no URL", + common.MapStr{ + "type": "someType", + "package_version": "1.2.3", + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "index": 42, "status": "down"}, + "error": common.MapStr{ + "name": "MyErrorName", + "message": "MyErrorMessage", + "stack": "MyErrorStack", + }, + "blob": "ablob", + "blob_mime": "application/weird", + }, + common.MapStr{ + "synthetics": common.MapStr{ + "type": "someType", + "package_version": "1.2.3", + "journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"}, + "step": common.MapStr{"name": "MyStep", "index": 42, "status": "down"}, + "error": common.MapStr{ + "name": "MyErrorName", + "message": "MyErrorMessage", + "stack": "MyErrorStack", + }, + "blob": "ablob", + "blob_mime": "application/weird", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Actually marshal to JSON and back to test the struct tags for deserialization from JSON + jsonBytes, err := json.Marshal(tc.source) + require.NoError(t, err) + se := &SynthEvent{} + err = json.Unmarshal(jsonBytes, se) + require.NoError(t, err) + + m := se.ToMap() + + // Index will always be zero in thee tests, so helpfully include it + llvalidator := lookslike.Strict(lookslike.Compose( + lookslike.MustCompile(tc.expected), + lookslike.MustCompile(common.MapStr{"synthetics": common.MapStr{"index": 0}}), + )) + + // Test that even deep maps merge correctly + testslike.Test(t, llvalidator, m) + }) + } +} diff --git a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml index 74fc2f4d885..79e18851c9b 100644 --- a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml +++ b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml @@ -9,8 +9,6 @@ heartbeat.monitors: enabled: true id: todos-suite name: Todos Suite - data_stream: - namespace: myns source: local: path: "/home/andrewvc/projects/synthetics/examples/todos/" @@ -21,14 +19,10 @@ heartbeat.monitors: urls: http://www.google.com schedule: "@every 15s" name: Simple HTTP - data_stream: - namespace: myns - type: browser enabled: false id: my-monitor name: My Monitor - data_stream: - namespace: myns source: inline: script: