[Heartbeat] Refactor config system

CHANGELOG.next.asciidoc

@@ -793,6 +793,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Heartbeat*
 
 - Add mime type detection for http responses. {pull}22976[22976]
+- Copy suite directories for synthetic checks to tmp dir before running. {pull}23347[23347]
 
 *Journalbeat*
 

go.sum

@@ -244,6 +244,8 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA=
 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
+github.com/elastic/beats v1.3.1 h1:hHzUBHCo3HJHxnRVwa0XlfZoxmP8Rxp7GQ0ZVELGY4A=
+github.com/elastic/beats v7.6.2+incompatible h1:jHdLv83KURaqWUC6f55iMyVP6LYZrgElfeqxKWcskVE=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
 github.com/elastic/ecs v1.6.0 h1:8NmgfnsjmKXh9hVsK3H2tZtfUptepNc3msJOAynhtmc=

heartbeat/beater/heartbeat.go

@@ -18,7 +18,6 @@
 package beater
 
 import (
-	"context"
 	"fmt"
 	"time"
 
@@ -100,13 +99,6 @@ func (bt *Heartbeat) Run(b *beat.Beat) error {
 		}
 	}
 
-	if len(bt.config.SyntheticSuites) > 0 {
-		err := bt.RunSyntheticSuiteMonitors(b)
-		if err != nil {
-			return err
-		}
-	}
-
 	if bt.config.Autodiscover != nil {
 		bt.autodiscover, err = bt.makeAutodiscover(b)
 		if err != nil {
@@ -168,50 +160,6 @@ func (bt *Heartbeat) RunReloadableMonitors(b *beat.Beat) (err error) {
 	return nil
 }
 
-// Provide hook to define journey list discovery from x-pack
-type JourneyLister func(ctx context.Context, suiteFile string, params common.MapStr) ([]string, error)
-
-var mainJourneyLister JourneyLister
-
-func RegisterJourneyLister(jl JourneyLister) {
-	mainJourneyLister = jl
-}
-
-func (bt *Heartbeat) RunSyntheticSuiteMonitors(b *beat.Beat) error {
-	// If we are running without XPack this will be nil
-	if mainJourneyLister == nil {
-		return nil
-	}
-	for _, suite := range bt.config.SyntheticSuites {
-		logp.Info("Listing suite %s", suite.Path)
-		journeyNames, err := mainJourneyLister(context.TODO(), suite.Path, suite.Params)
-		if err != nil {
-			return err
-		}
-		factory := monitors.NewFactory(b.Info, bt.scheduler, false)
-		for _, name := range journeyNames {
-			cfg, err := common.NewConfigFrom(map[string]interface{}{
-				"type":         "browser",
-				"path":         suite.Path,
-				"schedule":     suite.Schedule,
-				"params":       suite.Params,
-				"journey_name": name,
-				"name":         name,
-				"id":           name,
-			})
-			if err != nil {
-				return err
-			}
-			created, err := factory.Create(b.Publisher, cfg)
-			if err != nil {
-				return errors.Wrap(err, "could not create monitor")
-			}
-			created.Start()
-		}
-	}
-	return nil
-}
-
 // makeAutodiscover creates an autodiscover object ready to be started.
 func (bt *Heartbeat) makeAutodiscover(b *beat.Beat) (*autodiscover.Autodiscover, error) {
 	autodiscover, err := autodiscover.NewAutodiscover(

heartbeat/config/config.go

@@ -32,7 +32,7 @@ type Config struct {
 	ConfigMonitors  *common.Config       `config:"config.monitors"`
 	Scheduler       Scheduler            `config:"scheduler"`
 	Autodiscover    *autodiscover.Config `config:"autodiscover"`
-	SyntheticSuites []*SyntheticSuite    `config:"synthetic_suites"`
+	SyntheticSuites []*common.Config     `config:"synthetic_suites"`
 }
 
 // Scheduler defines the syntax of a heartbeat.yml scheduler block.
@@ -41,12 +41,5 @@ type Scheduler struct {
 	Location string `config:"location"`
 }
 
-type SyntheticSuite struct {
-	Path     string                 `config:"path"`
-	Name     string                 `config:"id_prefix"`
-	Schedule string                 `config:"schedule"`
-	Params   map[string]interface{} `config:"params"`
-}
-
 // DefaultConfig is the canonical instantiation of Config.
 var DefaultConfig = Config{}

heartbeat/monitors/active/http/http.go

@@ -32,8 +32,7 @@ import (
 )
 
 func init() {
-	monitors.RegisterActive("http", create)
-	monitors.RegisterActive("synthetics/http", create)
+	monitors.RegisterActive("http", create, "synthetics/http")
 }
 
 var debugf = logp.MakeDebug("http")

heartbeat/monitors/active/icmp/icmp.go

@@ -35,8 +35,7 @@ import (
 var debugf = logp.MakeDebug("icmp")
 
 func init() {
-	monitors.RegisterActive("icmp", create)
-	monitors.RegisterActive("synthetics/icmp", create)
+	monitors.RegisterActive("icmp", create, "synthetics/icmp")
 }
 
 func create(

heartbeat/monitors/active/tcp/tcp.go

@@ -39,8 +39,7 @@ import (
 )
 
 func init() {
-	monitors.RegisterActive("tcp", create)
-	monitors.RegisterActive("synthetics/tcp", create)
+	monitors.RegisterActive("tcp", create, "synthetics/tcp")
 }
 
 var debugf = logp.MakeDebug("tcp")

heartbeat/monitors/mocks_test.go

@@ -137,19 +137,22 @@ func createMockJob(name string, cfg *common.Config) ([]jobs.Job, error) {
 func mockPluginBuilder() pluginBuilder {
 	reg := monitoring.NewRegistry()
 
-	return pluginBuilder{"test", ActiveMonitor, func(s string, config *common.Config) ([]jobs.Job, int, error) {
-		// Declare a real config block with a required attr so we can see what happens when it doesn't work
-		unpacked := struct {
-			URLs []string `config:"urls" validate:"required"`
-		}{}
-		err := config.Unpack(&unpacked)
-		if err != nil {
-			return nil, 0, err
-		}
-		c := common.Config{}
-		j, err := createMockJob("test", &c)
-		return j, 1, err
-	}, newPluginCountersRecorder("test", reg)}
+	return pluginBuilder{
+		"test",
+		[]string{"testAlias"},
+		func(s string, config *common.Config) ([]jobs.Job, int, error) {
+			// Declare a real config block with a required attr so we can see what happens when it doesn't work
+			unpacked := struct {
+				URLs []string `config:"urls" validate:"required"`
+			}{}
+			err := config.Unpack(&unpacked)
+			if err != nil {
+				return nil, 0, err
+			}
+			c := common.Config{}
+			j, err := createMockJob("test", &c)
+			return j, 1, err
+		}, newPluginCountersRecorder("test", reg)}
 }
 
 func mockPluginsReg() *pluginsReg {

heartbeat/monitors/monitor.go

@@ -120,18 +120,18 @@ func newMonitorUnsafe(
 	// Extract just the Id, Type, and Enabled fields from the config
 	// We'll parse things more precisely later once we know what exact type of
 	// monitor we have
-	stdFields, err := stdfields.ConfigToStdMonitorFields(config)
+	standardFields, err := stdfields.ConfigToStdMonitorFields(config)
 	if err != nil {
 		return nil, err
 	}
 
-	monitorPlugin, found := registrar.get(stdFields.Type)
+	monitorPlugin, found := registrar.get(standardFields.Type)
 	if !found {
-		return nil, fmt.Errorf("monitor type %v does not exist, valid types are %v", stdFields.Type, registrar.monitorNames())
+		return nil, fmt.Errorf("monitor type %v does not exist, valid types are %v", standardFields.Type, registrar.monitorNames())
 	}
 
 	m := &Monitor{
-		stdFields:         stdFields,
+		stdFields:         standardFields,
 		pluginName:        monitorPlugin.name,
 		scheduler:         scheduler,
 		configuredJobs:    []*configuredJob{},

heartbeat/monitors/plugin.go

@@ -31,7 +31,7 @@ import (
 
 type pluginBuilder struct {
 	name    string
-	typ     Type
+	aliases []string
 	builder PluginBuilder
 	stats   registryRecorder
 }
@@ -62,7 +62,7 @@ func init() {
 		}
 
 		stats := statsForPlugin(p.name)
-		return globalPluginsReg.register(pluginBuilder{p.name, p.typ, p.builder, stats})
+		return globalPluginsReg.register(pluginBuilder{p.name, p.aliases, p.builder, stats})
 	})
 }
 
@@ -94,9 +94,9 @@ func newPluginsReg() *pluginsReg {
 }
 
 // RegisterActive registers a new active (as opposed to passive) monitor.
-func RegisterActive(name string, builder PluginBuilder) {
+func RegisterActive(name string, builder PluginBuilder, aliases ...string) {
 	stats := statsForPlugin(name)
-	if err := globalPluginsReg.add(pluginBuilder{name, ActiveMonitor, builder, stats}); err != nil {
+	if err := globalPluginsReg.add(pluginBuilder{name, aliases, builder, stats}); err != nil {
 		panic(err)
 	}
 }
@@ -106,20 +106,26 @@ func RegisterActive(name string, builder PluginBuilder) {
 type ErrPluginAlreadyExists pluginBuilder
 
 func (m ErrPluginAlreadyExists) Error() string {
-	return fmt.Sprintf("monitor plugin '%s' already exists", m.typ)
+	return fmt.Sprintf("monitor plugin named '%s' with aliases %v already exists", m.name, m.aliases)
 }
 
 func (r *pluginsReg) add(plugin pluginBuilder) error {
 	if _, exists := r.monitors[plugin.name]; exists {
 		return ErrPluginAlreadyExists(plugin)
 	}
 	r.monitors[plugin.name] = plugin
+	for _, alias := range plugin.aliases {
+		if _, exists := r.monitors[alias]; exists {
+			return ErrPluginAlreadyExists(plugin)
+		}
+		r.monitors[alias] = plugin
+	}
 	return nil
 }
 
 func (r *pluginsReg) register(plugin pluginBuilder) error {
 	if _, found := r.monitors[plugin.name]; found {
-		return fmt.Errorf("monitor type %v already exists", plugin.typ)
+		return fmt.Errorf("monitor type %v already exists", plugin.name)
 	}
 
 	r.monitors[plugin.name] = plugin

heartbeat/monitors/wrappers/monitors.go

@@ -38,33 +38,50 @@ import (
 
 // WrapCommon applies the common wrappers that all monitor jobs get.
 func WrapCommon(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.Job {
-	jobWrappers := []jobs.JobWrapper{
-		addMonitorMeta(stdMonFields, len(js) > 1),
-		addMonitorStatus(stdMonFields.Type),
-	}
-
-	if stdMonFields.Type != "browser" {
-		jobWrappers = append(jobWrappers, addMonitorDuration)
+	if stdMonFields.Type == "browser" {
+		return WrapBrowser(js, stdMonFields)
+	} else {
+		return WrapLightweight(js, stdMonFields)
 	}
+}
 
+func WrapLightweight(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.Job {
 	return jobs.WrapAllSeparately(
 		jobs.WrapAll(
 			js,
-			jobWrappers...,
+			addMonitorMeta(stdMonFields, len(js) > 1),
+			addMonitorStatus(stdMonFields.Type),
+			addMonitorDuration,
 		),
 		func() jobs.JobWrapper {
 			return makeAddSummary(stdMonFields.Type)
 		})
 }
 
+func WrapBrowser(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.Job {
+	return jobs.WrapAll(
+		js,
+		addMonitorMeta(stdMonFields, len(js) > 1),
+		addMonitorStatus(stdMonFields.Type),
+	)
+}
+
 // addMonitorMeta adds the id, name, and type fields to the monitor.
 func addMonitorMeta(stdMonFields stdfields.StdMonitorFields, isMulti bool) jobs.JobWrapper {
 	return func(job jobs.Job) jobs.Job {
 		return func(event *beat.Event) ([]jobs.Job, error) {
 			started := time.Now()
 			cont, e := job(event)
 			thisID := stdMonFields.ID
-
+			thisName := stdMonFields.Name
+			// Allow jobs to override the ID, useful for browser suites
+			// which do this logic on their own
+			if v, _ := event.GetValue("monitor.id"); v != nil {
+				thisID = v.(string)
+			}
+			if v, _ := event.GetValue("monitor.name"); v != nil {
+				thisName = v.(string)
+			}
 			if isMulti {
 				url, err := event.GetValue("url.full")
 				if err != nil {
@@ -78,7 +95,7 @@ func addMonitorMeta(stdMonFields stdfields.StdMonitorFields, isMulti bool) jobs.
 			fieldsToMerge := common.MapStr{
 				"monitor": common.MapStr{
 					"id":       thisID,
-					"name":     stdMonFields.Name,
+					"name":     thisName,
 					"type":     stdMonFields.Type,
 					"timespan": timespan(started, stdMonFields.Schedule, stdMonFields.Timeout),
 				},
@@ -120,13 +137,6 @@ func addMonitorStatus(monitorType string) jobs.JobWrapper {
 		return func(event *beat.Event) ([]jobs.Job, error) {
 			cont, err := origJob(event)
 
-			// Non-summary browser events have no status associated
-			if monitorType == "browser" {
-				if t, _ := event.GetValue("synthetics.type"); t != "heartbeat/summary" {
-					return cont, nil
-				}
-			}
-
 			fields := common.MapStr{
 				"monitor": common.MapStr{
 					"status": look.Status(err),
@@ -167,6 +177,7 @@ func makeAddSummary(monitorType string) jobs.JobWrapper {
 	// state struct here.
 	state := struct {
 		mtx        sync.Mutex
+		monitorId  string
 		remaining  uint16
 		up         uint16
 		down       uint16
@@ -191,6 +202,10 @@ func makeAddSummary(monitorType string) jobs.JobWrapper {
 
 	return func(job jobs.Job) jobs.Job {
 		return func(event *beat.Event) ([]jobs.Job, error) {
+			if v, _ := event.GetValue("monitor.id"); v != state.monitorId {
+				resetState()
+			}
+
 			cont, jobErr := job(event)
 			state.mtx.Lock()
 			defer state.mtx.Unlock()
@@ -208,7 +223,6 @@ func makeAddSummary(monitorType string) jobs.JobWrapper {
 				}
 			}
 
-			// No error check needed here
 			event.PutValue("monitor.check_group", state.checkGroup)
 
 			// Adjust the total remaining to account for new continuations
@@ -220,15 +234,7 @@ func makeAddSummary(monitorType string) jobs.JobWrapper {
 			if state.remaining == 0 {
 				up := state.up
 				down := state.down
-				if monitorType == "browser" {
-					if eventStatus == "down" {
-						up = 0
-						down = 1
-					} else {
-						up = 1
-						down = 0
-					}
-				}
+
 				eventext.MergeEventFields(event, common.MapStr{
 					"summary": common.MapStr{
 						"up":   up,