[Heartbeat] Merge synthetic root fields into events (#24770) (#24860)

Fixes #24768 This allows synthetics to drive more field names without requiring heartbeat updates. Any fields in root_fields get merged into the event root. This also improves the testing in this area of the code, which was somewhat lean (and really was only tested in larger functional tests run elsewhere)
elastic · Mar 30, 2021 · c812dd6 · c812dd6
1 parent 3788227
commit c812dd6
Show file tree

Hide file tree

Showing 4 changed files with 143 additions and 9 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -521,6 +521,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Bundle synthetics deps with heartbeat docker image. {pull}23274[23274]
 - Handle datastreams for fleet. {pull}24223[24223]
 - Add --sandbox option for browser monitor. {pull}24172[24172]
+- Support additional 'root' fields from synthetics. {pull}24770[24770]
 
 *Heartbeat*
 

diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go
@@ -27,18 +27,35 @@ type SynthEvent struct {
 	Error                *SynthError   `json:"error"`
 	URL                  string        `json:"url"`
 	Status               string        `json:"status"`
+	RootFields           common.MapStr `json:"root_fields"`
 	index                int
 }
 
 func (se SynthEvent) ToMap() (m common.MapStr) {
 	// We don't add @timestamp to the map string since that's specially handled in beat.Event
-	m = common.MapStr{
+	// Use the root fields as a base, and layer additional, stricter, fields on top
+	if se.RootFields != nil {
+		m = se.RootFields
+		// We handle url specially since it can be passed as a string,
+		// but expanded to match ECS
+		if urlStr, ok := m["url"].(string); ok {
+			if se.URL == "" {
+				se.URL = urlStr
+			}
+		}
+	} else {
+		m = common.MapStr{}
+	}
+
+	m.DeepUpdate(common.MapStr{
 		"synthetics": common.MapStr{
 			"type":            se.Type,
 			"package_version": se.PackageVersion,
-			"payload":         se.Payload,
 			"index":           se.index,
 		},
+	})
+	if len(se.Payload) > 0 {
+		m.Put("synthetics.payload", se.Payload)
 	}
 	if se.Blob != "" {
 		m.Put("synthetics.blob", se.Blob)
@@ -61,7 +78,7 @@ func (se SynthEvent) ToMap() (m common.MapStr) {
 		if e != nil {
 			logp.Warn("Could not parse synthetics URL '%s': %s", se.URL, e.Error())
 		} else {
-			m["url"] = wrappers.URLFields(u)
+			m.Put("url", wrappers.URLFields(u))
 		}
 	}
 

diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go
@@ -5,13 +5,135 @@
 package synthexec
 
 import (
+	"encoding/json"
+	"net/url"
 	"testing"
 	"time"
 
+	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers"
+
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/go-lookslike"
+	"github.com/elastic/go-lookslike/testslike"
+
 	"github.com/stretchr/testify/require"
 )
 
 func TestSynthEventTimestamp(t *testing.T) {
 	se := SynthEvent{TimestampEpochMicros: 1000} // 1ms
 	require.Equal(t, time.Unix(0, int64(time.Millisecond)), se.Timestamp())
 }
+
+func TestToMap(t *testing.T) {
+	testUrl, _ := url.Parse("http://testurl")
+
+	type testCase struct {
+		name     string
+		source   common.MapStr
+		expected common.MapStr
+	}
+
+	testCases := []testCase{
+		{
+			"root fields with URL",
+			common.MapStr{
+				"type":            "journey/start",
+				"package_version": "1.2.3",
+				"root_fields": map[string]interface{}{
+					"synthetics": map[string]interface{}{
+						"nested": "v1",
+					},
+					"truly_at_root": "v2",
+				},
+				"url": testUrl.String(),
+			},
+			common.MapStr{
+				"synthetics": common.MapStr{
+					"type":            "journey/start",
+					"package_version": "1.2.3",
+					"nested":          "v1",
+				},
+				"url":           wrappers.URLFields(testUrl),
+				"truly_at_root": "v2",
+			},
+		},
+		{
+			"root fields, step metadata",
+			common.MapStr{
+				"type":            "step/start",
+				"package_version": "1.2.3",
+				"journey":         common.MapStr{"name": "MyJourney", "id": "MyJourney"},
+				"step":            common.MapStr{"name": "MyStep", "status": "success", "index": 42},
+				"root_fields": map[string]interface{}{
+					"synthetics": map[string]interface{}{
+						"nested": "v1",
+					},
+					"truly_at_root": "v2",
+				},
+			},
+			common.MapStr{
+				"synthetics": common.MapStr{
+					"type":            "step/start",
+					"package_version": "1.2.3",
+					"nested":          "v1",
+					"journey":         common.MapStr{"name": "MyJourney", "id": "MyJourney"},
+					"step":            common.MapStr{"name": "MyStep", "status": "success", "index": 42},
+				},
+				"truly_at_root": "v2",
+			},
+		},
+		{
+			"weird error, and blob, no URL",
+			common.MapStr{
+				"type":            "someType",
+				"package_version": "1.2.3",
+				"journey":         common.MapStr{"name": "MyJourney", "id": "MyJourney"},
+				"step":            common.MapStr{"name": "MyStep", "index": 42, "status": "down"},
+				"error": common.MapStr{
+					"name":    "MyErrorName",
+					"message": "MyErrorMessage",
+					"stack":   "MyErrorStack",
+				},
+				"blob":      "ablob",
+				"blob_mime": "application/weird",
+			},
+			common.MapStr{
+				"synthetics": common.MapStr{
+					"type":            "someType",
+					"package_version": "1.2.3",
+					"journey":         common.MapStr{"name": "MyJourney", "id": "MyJourney"},
+					"step":            common.MapStr{"name": "MyStep", "index": 42, "status": "down"},
+					"error": common.MapStr{
+						"name":    "MyErrorName",
+						"message": "MyErrorMessage",
+						"stack":   "MyErrorStack",
+					},
+					"blob":      "ablob",
+					"blob_mime": "application/weird",
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Actually marshal to JSON and back to test the struct tags for deserialization from JSON
+			jsonBytes, err := json.Marshal(tc.source)
+			require.NoError(t, err)
+			se := &SynthEvent{}
+			err = json.Unmarshal(jsonBytes, se)
+			require.NoError(t, err)
+
+			m := se.ToMap()
+
+			// Index will always be zero in thee tests, so helpfully include it
+			llvalidator := lookslike.Strict(lookslike.Compose(
+				lookslike.MustCompile(tc.expected),
+				lookslike.MustCompile(common.MapStr{"synthetics": common.MapStr{"index": 0}}),
+			))
+
+			// Test that even deep maps merge correctly
+			testslike.Test(t, llvalidator, m)
+		})
+	}
+}
diff --git a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
@@ -9,8 +9,6 @@ heartbeat.monitors:
   enabled: true
   id: todos-suite
   name: Todos Suite
-  data_stream:
-    namespace: myns
   source:
     local:
       path: "/home/andrewvc/projects/synthetics/examples/todos/"
@@ -21,14 +19,10 @@ heartbeat.monitors:
   urls: http://www.google.com
   schedule: "@every 15s"
   name: Simple HTTP
-  data_stream:
-    namespace: myns
 - type: browser
   enabled: false
   id: my-monitor
   name: My Monitor
-  data_stream:
-    namespace: myns
   source:
     inline:
       script: