[Heartbeat] Merge synthetic root fields into events

CHANGELOG.next.asciidoc

@@ -812,6 +812,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add mime type detection for http responses. {pull}22976[22976]
 - Handle datastreams for fleet. {pull}24223[24223]
 - Add --sandbox option for browser monitor. {pull}24172[24172]
+- Support additional 'root' fields from synthetics. {pull}24770[24770]
 
 *Journalbeat*
 

x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go

@@ -16,29 +16,46 @@ import (
 )
 
 type SynthEvent struct {
-	Type                 string        `json:"type"`
-	PackageVersion       string        `json:"package_version"`
-	Step                 *Step         `json:"step"`
-	Journey              *Journey      `json:"journey"`
-	TimestampEpochMicros float64       `json:"@timestamp"`
-	Payload              common.MapStr `json:"payload"`
-	Blob                 string        `json:"blob"`
-	BlobMime             string        `json:"blob_mime"`
-	Error                *SynthError   `json:"error"`
-	URL                  string        `json:"url"`
-	Status               string        `json:"status"`
+	Type                 string                 `json:"type"`
+	PackageVersion       string                 `json:"package_version"`
+	Step                 *Step                  `json:"step"`
+	Journey              *Journey               `json:"journey"`
+	TimestampEpochMicros float64                `json:"@timestamp"`
+	Payload              common.MapStr          `json:"payload"`
+	Blob                 string                 `json:"blob"`
+	BlobMime             string                 `json:"blob_mime"`
+	Error                *SynthError            `json:"error"`
+	URL                  string                 `json:"url"`
+	Status               string                 `json:"status"`
+	RootFields           common.MapStr			`json:"root_fields"`
 	index                int
 }
 
 func (se SynthEvent) ToMap() (m common.MapStr) {
 	// We don't add @timestamp to the map string since that's specially handled in beat.Event
-	m = common.MapStr{
+	// Use the root fields as a base, and layer additional, stricter, fields on top
+	if se.RootFields != nil {
+		m = se.RootFields
+		// We handle url specially since it can be passed as a string,
+		// but expanded to match ECS
+		if urlStr, ok := m["url"].(string); ok {
+			if se.URL == "" {
+				se.URL = urlStr
+			}
+		}
+	} else {
+		m = common.MapStr{}
+	}
+
+	m.DeepUpdate(common.MapStr{
 		"synthetics": common.MapStr{
 			"type":            se.Type,
 			"package_version": se.PackageVersion,
-			"payload":         se.Payload,
 			"index":           se.index,
 		},
+	})
+	if len(se.Payload) > 0 {
+		m.Put("synthetics.payload", se.Payload)
 	}
 	if se.Blob != "" {
 		m.Put("synthetics.blob", se.Blob)
@@ -61,7 +78,7 @@ func (se SynthEvent) ToMap() (m common.MapStr) {
 		if e != nil {
 			logp.Warn("Could not parse synthetics URL '%s': %s", se.URL, e.Error())
 		} else {
-			m["url"] = wrappers.URLFields(u)
+			m.Put("url", wrappers.URLFields(u))
 		}
 	}
 

x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go

@@ -5,13 +5,134 @@
 package synthexec
 
 import (
+	"encoding/json"
+	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers"
+	"net/url"
 	"testing"
 	"time"
 
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/go-lookslike"
+	"github.com/elastic/go-lookslike/testslike"
+
 	"github.com/stretchr/testify/require"
 )
 
 func TestSynthEventTimestamp(t *testing.T) {
 	se := SynthEvent{TimestampEpochMicros: 1000} // 1ms
 	require.Equal(t, time.Unix(0, int64(time.Millisecond)), se.Timestamp())
 }
+
+func TestToMap(t *testing.T) {
+	testUrl, _ := url.Parse("http://testurl")
+
+	type testCase struct {
+		name     string
+		source   common.MapStr
+		expected common.MapStr
+	}
+
+	testCases := []testCase{
+		{
+			"root fields with URL",
+			common.MapStr{
+				"type": "journey/start",
+				"package_version": "1.2.3",
+				"root_fields": map[string]interface{}{
+					"synthetics": map[string]interface{}{
+						"nested": "v1",
+					},
+					"truly_at_root": "v2",
+				},
+				"url": testUrl.String(),
+			},
+			common.MapStr{
+				"synthetics": common.MapStr{
+					"type":   "journey/start",
+					"package_version": "1.2.3",
+					"nested": "v1",
+				},
+				"url": wrappers.URLFields(testUrl),
+				"truly_at_root": "v2",
+			},
+		},
+		{
+			"root fields, step metadata",
+			common.MapStr{
+				"type": "step/start",
+				"package_version": "1.2.3",
+				"journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"},
+				"step": common.MapStr{"name": "MyStep", "status": "success", "index": 42},
+				"root_fields": map[string]interface{}{
+					"synthetics": map[string]interface{}{
+						"nested": "v1",
+					},
+					"truly_at_root": "v2",
+				},
+			},
+			common.MapStr{
+				"synthetics": common.MapStr{
+					"type":   "step/start",
+					"package_version": "1.2.3",
+					"nested": "v1",
+					"journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"},
+					"step": common.MapStr{"name": "MyStep", "status": "success", "index": 42},
+				},
+				"truly_at_root": "v2",
+			},
+		},
+		{
+			"weird error, and blob, no URL",
+			common.MapStr{
+				"type": "someType",
+				"package_version": "1.2.3",
+				"journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"},
+				"step": common.MapStr{"name": "MyStep", "index": 42, "status": "down"},
+				"error": common.MapStr{
+					"name": "MyErrorName",
+					"message": "MyErrorMessage",
+					"stack": "MyErrorStack",
+				},
+				"blob": "ablob",
+				"blob_mime": "application/weird",
+			},
+			common.MapStr{
+				"synthetics": common.MapStr{
+					"type":   "someType",
+					"package_version": "1.2.3",
+					"journey": common.MapStr{"name": "MyJourney", "id": "MyJourney"},
+					"step": common.MapStr{"name": "MyStep", "index": 42, "status": "down"},
+					"error": common.MapStr{
+						"name": "MyErrorName",
+						"message": "MyErrorMessage",
+						"stack": "MyErrorStack",
+					},
+					"blob": "ablob",
+					"blob_mime": "application/weird",
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Actually marshal to JSON and back to test the struct tags for deserialization from JSON
+			jsonBytes, err := json.Marshal(tc.source)
+			require.NoError(t, err)
+			se := &SynthEvent{}
+			err = json.Unmarshal(jsonBytes, se)
+			require.NoError(t, err)
+
+			m := se.ToMap()
+
+			// Index will always be zero in thee tests, so helpfully include it
+			llvalidator := lookslike.Strict(lookslike.Compose(
+				lookslike.MustCompile(tc.expected),
+				lookslike.MustCompile(common.MapStr{"synthetics": common.MapStr{"index": 0}}),
+			))
+
+			// Test that even deep maps merge correctly
+			testslike.Test(t, llvalidator, m)
+		})
+	}
+}

x-pack/heartbeat/sample-synthetics-config/heartbeat.yml

@@ -9,8 +9,6 @@ heartbeat.monitors:
   enabled: true
   id: todos-suite
   name: Todos Suite
-  data_stream:
-    namespace: myns
   source:
     local:
       path: "/home/andrewvc/projects/synthetics/examples/todos/"
@@ -21,14 +19,10 @@ heartbeat.monitors:
   urls: http://www.google.com
   schedule: "@every 15s"
   name: Simple HTTP
-  data_stream:
-    namespace: myns
 - type: browser
   enabled: false
   id: my-monitor
   name: My Monitor
-  data_stream:
-    namespace: myns
   source:
     inline:
       script: