From 6a0b645e2f69f2d97eb9b714d3539758f3b7bfda Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Mon, 4 Jan 2021 13:44:47 -0600
Subject: [PATCH 01/41] [Uptime] Copy synthetics suites to tmpdir before
 running

The rationale here is that doing so resolves any file permissions issues
that may be present due to the suite directory being shared to the
container as read only OR due to incompatible UIDs between the docker
container and the host.

Fixes https://github.com/elastic/synthetics/issues/156
---
 .../monitors/browser/synthexec/synthexec.go   | 25 ++++++++++++++++---
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
index 41a5c1ae88c..413db1baade 100644
--- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
@@ -10,6 +10,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -23,17 +24,33 @@ import (
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
+
+	"github.com/otiai10/copy"
 )
 
 const debugSelector = "synthexec"
 
 func init() {
-	beater.RegisterJourneyLister(ListJourneys)
+	beater.RegisterJourneyLister(InitSuite)
+}
+
+func InitSuite(ctx context.Context, origSuitePath string, params common.MapStr) (journeyNames []string, err error) {
+	dir, err := ioutil.TempDir("/tmp", "elastic-synthetics-")
+	if err != nil {
+		return nil, err
+	}
+
+	err = copy.Copy(origSuitePath, dir)
+	if err != nil {
+		return nil, err
+	}
+
+	return ListJourneys(ctx, dir, params)
 }
 
 // ListJourneys takes the given suite performs a dry run, capturing the Journey names, and returns the list.
-func ListJourneys(ctx context.Context, suiteFile string, params common.MapStr) (journeyNames []string, err error) {
-	dir, err := getSuiteDir(suiteFile)
+func ListJourneys(ctx context.Context, suitePath string, params common.MapStr) (journeyNames []string, err error) {
+	dir, err := getSuiteDir(suitePath)
 	if err != nil {
 		return nil, err
 	}
@@ -53,7 +70,7 @@ func ListJourneys(ctx context.Context, suiteFile string, params common.MapStr) (
 		}
 	}
 
-	cmdFactory, err := suiteCommandFactory(dir, suiteFile, "--dry-run")
+	cmdFactory, err := suiteCommandFactory(dir, suitePath, "--dry-run")
 	if err != nil {
 		return nil, err
 	}

From 2c48c5533e3d8be927941e971782691ebf374148 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Mon, 4 Jan 2021 14:28:20 -0600
Subject: [PATCH 02/41] Add changelog

---
 CHANGELOG.next.asciidoc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 691f5aac284..8ad7fb4fba3 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -793,6 +793,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Heartbeat*
 
 - Add mime type detection for http responses. {pull}22976[22976]
+- Copy suite directories for synthetic checks to tmp dir before running. {pull}23347[23347]
 
 *Journalbeat*
 

From 1ab0fd62125a6db6cddbfe5cc352a16295c35594 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Mon, 4 Jan 2021 14:35:22 -0600
Subject: [PATCH 03/41] Fix var naming

---
 heartbeat/beater/heartbeat.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go
index 13509c6bb32..5aedcea42df 100644
--- a/heartbeat/beater/heartbeat.go
+++ b/heartbeat/beater/heartbeat.go
@@ -169,7 +169,7 @@ func (bt *Heartbeat) RunReloadableMonitors(b *beat.Beat) (err error) {
 }
 
 // Provide hook to define journey list discovery from x-pack
-type JourneyLister func(ctx context.Context, suiteFile string, params common.MapStr) ([]string, error)
+type JourneyLister func(ctx context.Context, suitePath string, params common.MapStr) ([]string, error)
 
 var mainJourneyLister JourneyLister
 

From 168c6a075b58bb32f76b4e3d642fbe742738fa02 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Tue, 5 Jan 2021 07:17:11 -0600
Subject: [PATCH 04/41] Remote journeyps

---
 heartbeat/beater/heartbeat.go | 112 ++++++++++++++++++++++++++++++++--
 heartbeat/config/config.go    |  31 +++++++++-
 2 files changed, 137 insertions(+), 6 deletions(-)

diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go
index 5aedcea42df..3b06219cc98 100644
--- a/heartbeat/beater/heartbeat.go
+++ b/heartbeat/beater/heartbeat.go
@@ -20,6 +20,7 @@ package beater
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"time"
 
 	"github.com/pkg/errors"
@@ -36,6 +37,9 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common/reload"
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/management"
+
+
+	dirCopy "github.com/otiai10/copy"
 )
 
 // Heartbeat represents the root datastructure of this beat.
@@ -182,9 +186,47 @@ func (bt *Heartbeat) RunSyntheticSuiteMonitors(b *beat.Beat) error {
 	if mainJourneyLister == nil {
 		return nil
 	}
-	for _, suite := range bt.config.SyntheticSuites {
-		logp.Info("Listing suite %s", suite.Path)
-		journeyNames, err := mainJourneyLister(context.TODO(), suite.Path, suite.Params)
+	for _, rawSuiteCfg := range bt.config.SyntheticSuites {
+		suite := &config.SyntheticSuite{}
+		err := rawSuiteCfg.Unpack(suite)
+		if err != nil {
+			logp.Err("could not parse suite config: %s", err)
+			continue
+		}
+
+		var suiteReloader SuiteReloader
+
+		switch suite.Type {
+		case "local":
+			localConfig := &config.LocalSyntheticSuite{}
+			err := rawSuiteCfg.Unpack(localConfig)
+			if err != nil {
+				logp.Err("could not parse local synthetic suite: %s", err)
+				continue
+			}
+			suiteReloader, err = NewLocalReloader(localConfig.Path)
+			if err != nil {
+				logp.Err("could not load local synthetics suite: %s", err)
+				continue
+			}
+		case "zipurl":
+			localConfig := &config.LocalSyntheticSuite{}
+			err := rawSuiteCfg.Unpack(localConfig)
+			if err != nil {
+				logp.Err("could not parse zip URL synthetic suite: %s", err)
+				continue
+			}
+		case "github":
+			localConfig := &config.LocalSyntheticSuite{}
+			err := rawSuiteCfg.Unpack(localConfig)
+			if err != nil {
+				logp.Err("could not parse github synthetic suite: %s", err)
+				continue
+			}
+		}
+
+		logp.Info("Listing suite %s", suiteReloader.WorkingPath())
+		journeyNames, err := mainJourneyLister(context.TODO(), suiteReloader.WorkingPath(), suite.Params)
 		if err != nil {
 			return err
 		}
@@ -192,7 +234,7 @@ func (bt *Heartbeat) RunSyntheticSuiteMonitors(b *beat.Beat) error {
 		for _, name := range journeyNames {
 			cfg, err := common.NewConfigFrom(map[string]interface{}{
 				"type":         "browser",
-				"path":         suite.Path,
+				"path":         suiteReloader.WorkingPath(),
 				"schedule":     suite.Schedule,
 				"params":       suite.Params,
 				"journey_name": name,
@@ -232,3 +274,65 @@ func (bt *Heartbeat) makeAutodiscover(b *beat.Beat) (*autodiscover.Autodiscover,
 func (bt *Heartbeat) Stop() {
 	close(bt.done)
 }
+
+type SuiteReloader interface {
+	String() string
+	Check() (bool, error)
+	WorkingPath() string
+}
+
+func NewLocalReloader(origSuitePath string) (*LocalReloader, error) {
+	dir, err := ioutil.TempDir("/tmp", "elastic-synthetics-")
+	if err != nil {
+		return nil, err
+	}
+
+	err = dirCopy.Copy(origSuitePath, dir)
+	if err != nil {
+		return nil, err
+	}
+
+	return &LocalReloader{workPath: origSuitePath}, nil
+}
+
+type LocalReloader struct {
+	origPath string
+	workPath string
+}
+
+func (l *LocalReloader) String() string {
+	return fmt.Sprintf("[Local Synthetics Suite origPath=%s workingPath=%s]", l.origPath, l.WorkingPath())
+}
+
+// Only loads once on startup, no rechecks
+func (l *LocalReloader) Check() (bool, error) {
+	return false, nil
+}
+
+func (l *LocalReloader) WorkingPath() string {
+	return l.workPath
+}
+
+func NewZipURLReloader(url string, headers map[string]string) (*ZipURLReloader, error) {
+	return &ZipURLReloader{URL: url, Headers: headers}, nil
+}
+
+type ZipURLReloader struct {
+	URL string
+	Headers map[string]string
+	etag string // used to determine if the URL contents has changed
+	workingPath string
+}
+
+func (z *ZipURLReloader) String() string {
+	return fmt.Sprintf("[ZipURL Synthetics suite url=%s workingPath=%s]", z.URL, z.workingPath)
+}
+
+func (z *ZipURLReloader) Check() (bool, error) {
+	panic("implement me")
+}
+
+func (z *ZipURLReloader) WorkingPath() string {
+	panic("implement me")
+}
+
diff --git a/heartbeat/config/config.go b/heartbeat/config/config.go
index bfee8096f39..eff6a257b85 100644
--- a/heartbeat/config/config.go
+++ b/heartbeat/config/config.go
@@ -32,7 +32,7 @@ type Config struct {
 	ConfigMonitors  *common.Config       `config:"config.monitors"`
 	Scheduler       Scheduler            `config:"scheduler"`
 	Autodiscover    *autodiscover.Config `config:"autodiscover"`
-	SyntheticSuites []*SyntheticSuite    `config:"synthetic_suites"`
+	SyntheticSuites []*common.Config    `config:"synthetic_suites"`
 }
 
 // Scheduler defines the syntax of a heartbeat.yml scheduler block.
@@ -42,10 +42,37 @@ type Scheduler struct {
 }
 
 type SyntheticSuite struct {
-	Path     string                 `config:"path"`
+	Type	 string					`config:"type"`
 	Name     string                 `config:"id_prefix"`
 	Schedule string                 `config:"schedule"`
 	Params   map[string]interface{} `config:"params"`
+	RawConfig *common.Config
+}
+
+type LocalSyntheticSuite struct {
+	Path     string                 `config:"path"`
+	SyntheticSuite
+}
+
+type PollingSyntheticSuite struct {
+	CheckEvery int `config:"check_every"`
+	SyntheticSuite
+}
+
+// GithubSyntheticSuite handles configs for github repos, using the API defined here:
+// https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#download-a-repository-archive-tar.
+type GithubSyntheticSuite struct {
+	Owner	string `config:"owner"`
+	Repo string `config:"repo"`
+	Ref string `config:"ref"`
+	UrlBase string `config:"string"`
+	PollingSyntheticSuite
+}
+
+type ZipUrlSyntheticSuite struct {
+	Url string `config:"url""`
+	Headers map[string]string `config:"headers"`
+	PollingSyntheticSuite
 }
 
 // DefaultConfig is the canonical instantiation of Config.

From e94fe34c880c829207f8a83ac527891250719948 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Tue, 5 Jan 2021 14:47:55 -0600
Subject: [PATCH 05/41] Checkpoint

---
 heartbeat/beater/heartbeat.go                 | 25 +++++++++++--------
 .../sample-synthetics-config/heartbeat.yml    |  2 ++
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go
index 3b06219cc98..bc845197899 100644
--- a/heartbeat/beater/heartbeat.go
+++ b/heartbeat/beater/heartbeat.go
@@ -103,7 +103,6 @@ func (bt *Heartbeat) Run(b *beat.Beat) error {
 			return err
 		}
 	}
-
 	if len(bt.config.SyntheticSuites) > 0 {
 		err := bt.RunSyntheticSuiteMonitors(b)
 		if err != nil {
@@ -197,39 +196,43 @@ func (bt *Heartbeat) RunSyntheticSuiteMonitors(b *beat.Beat) error {
 		var suiteReloader SuiteReloader
 
 		switch suite.Type {
-		case "local":
+		case "zipurl":
 			localConfig := &config.LocalSyntheticSuite{}
 			err := rawSuiteCfg.Unpack(localConfig)
 			if err != nil {
-				logp.Err("could not parse local synthetic suite: %s", err)
+				logp.Err("could not parse zip URL synthetic suite: %s", err)
 				continue
 			}
-			suiteReloader, err = NewLocalReloader(localConfig.Path)
+		case "github":
+			localConfig := &config.LocalSyntheticSuite{}
+			err := rawSuiteCfg.Unpack(localConfig)
 			if err != nil {
-				logp.Err("could not load local synthetics suite: %s", err)
+				logp.Err("could not parse github synthetic suite: %s", err)
 				continue
 			}
-		case "zipurl":
+		case "local":
 			localConfig := &config.LocalSyntheticSuite{}
 			err := rawSuiteCfg.Unpack(localConfig)
 			if err != nil {
-				logp.Err("could not parse zip URL synthetic suite: %s", err)
+				logp.Err("could not parse local synthetic suite: %s", err)
 				continue
 			}
-		case "github":
-			localConfig := &config.LocalSyntheticSuite{}
-			err := rawSuiteCfg.Unpack(localConfig)
+			suiteReloader, err = NewLocalReloader(localConfig.Path)
 			if err != nil {
-				logp.Err("could not parse github synthetic suite: %s", err)
+				logp.Err("could not load local synthetics suite: %s", err)
 				continue
 			}
+		default:
+			return fmt.Errorf("suite type not specified! Expected 'local', 'github', or 'zipurl'")
 		}
 
+		logp.Warn("PRELIST %v", suiteReloader)
 		logp.Info("Listing suite %s", suiteReloader.WorkingPath())
 		journeyNames, err := mainJourneyLister(context.TODO(), suiteReloader.WorkingPath(), suite.Params)
 		if err != nil {
 			return err
 		}
+		logp.Warn("POSTLIST")
 		factory := monitors.NewFactory(b.Info, bt.scheduler, false)
 		for _, name := range journeyNames {
 			cfg, err := common.NewConfigFrom(map[string]interface{}{
diff --git a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
index a47f798246a..28f4461a840 100644
--- a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
+++ b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
@@ -6,6 +6,7 @@ heartbeat.config.monitors:
 
 heartbeat.synthetic_suites:
 - name: Todos suite
+  type: "local"
   path: "/home/andrewvc/projects/synthetics/examples/todos"
   schedule: "@every 1m"
 
@@ -16,6 +17,7 @@ heartbeat.monitors:
   schedule: "@every 15s"
   name: Simple HTTP
 - type: browser
+  enabled: false
   id: my-monitor
   name: My Monitor
   script: |-

From a6581ba97ba41cf0a73d67fba4a72fbeeb5820ad Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Tue, 5 Jan 2021 16:50:51 -0600
Subject: [PATCH 06/41] Checkpoint

---
 heartbeat/beater/heartbeat.go                 | 49 +++++++++++++++----
 .../browser/suitefactory/suitefactory.go      | 33 +++++++++++++
 .../monitors/browser/synthexec/synthexec.go   |  5 --
 3 files changed, 72 insertions(+), 15 deletions(-)
 create mode 100644 x-pack/heartbeat/monitors/browser/suitefactory/suitefactory.go

diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go
index bc845197899..faa82d5e393 100644
--- a/heartbeat/beater/heartbeat.go
+++ b/heartbeat/beater/heartbeat.go
@@ -171,13 +171,48 @@ func (bt *Heartbeat) RunReloadableMonitors(b *beat.Beat) (err error) {
 	return nil
 }
 
-// Provide hook to define journey list discovery from x-pack
+var suiteFactory cfgfile.RunnerFactory
+
+func RegisterSuiteFactory(factory cfgfile.RunnerFactory) {
+	suiteFactory = factory
+}
+
+// RunCentralMgmtMonitors loads any central management configured configs.
+func (bt *Heartbeat) RunSuiteMonitors(b *beat.Beat) {
+	monitors := cfgfile.NewRunnerList(management.DebugK, suiteFactory, b.Publisher)
+	reload.Register.MustRegisterList(b.Info.Beat+".suites", monitors)
+}
+
 type JourneyLister func(ctx context.Context, suitePath string, params common.MapStr) ([]string, error)
 
-var mainJourneyLister JourneyLister
+// Provide hook to define journey list discovery from x-pack
+var suiteReloader SuiteReloader
+
+type SuiteReloader interface {
+	Check(factory *monitors.RunnerFactory) error
+	Run(factory *monitors.RunnerFactory) error
+}
+
+// RunReloadableMonitors runs the `heartbeat.config.monitors` portion of the yaml config if present.
+func (bt *Heartbeat) RunReloadableSuites(b *beat.Beat) (err error) {
+	// If we are running without XPack this will be nil
+	if suiteReloader == nil {
+		return nil
+	}
+
+	// Check monitor configs
+	if err := suiteReloader.Check(bt.dynamicFactory); err != nil {
+		logp.Error(errors.Wrap(err, "error loading reloadable monitors"))
+	}
+
+	// Execute the monitor
+	go suiteReloader.Run(bt.dynamicFactory)
+
+	return nil
+}
 
-func RegisterJourneyLister(jl JourneyLister) {
-	mainJourneyLister = jl
+func RegisterSuiteReloader(sr SuiteReloader) {
+	suiteReloader = sr
 }
 
 func (bt *Heartbeat) RunSyntheticSuiteMonitors(b *beat.Beat) error {
@@ -278,12 +313,6 @@ func (bt *Heartbeat) Stop() {
 	close(bt.done)
 }
 
-type SuiteReloader interface {
-	String() string
-	Check() (bool, error)
-	WorkingPath() string
-}
-
 func NewLocalReloader(origSuitePath string) (*LocalReloader, error) {
 	dir, err := ioutil.TempDir("/tmp", "elastic-synthetics-")
 	if err != nil {
diff --git a/x-pack/heartbeat/monitors/browser/suitefactory/suitefactory.go b/x-pack/heartbeat/monitors/browser/suitefactory/suitefactory.go
new file mode 100644
index 00000000000..1af28db5ae0
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/suitefactory/suitefactory.go
@@ -0,0 +1,33 @@
+package suitefactory
+
+import (
+	"github.com/elastic/beats/v7/heartbeat/beater"
+	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/elastic/beats/v7/libbeat/cfgfile"
+	"github.com/elastic/beats/v7/libbeat/common"
+)
+
+func init() {
+	sf := NewStdSuiteFactory()
+	beater.RegisterSuiteFactory(sf)
+}
+
+func NewStdSuiteFactory() *StdSuiteFactory {
+	return &StdSuiteFactory{}
+}
+
+type StdSuiteFactory struct {
+	//
+}
+
+func (s *StdSuiteFactory) Create(p beat.PipelineConnector, config *common.Config) (cfgfile.Runner, error) {
+	panic("implement me")
+}
+
+func (s *StdSuiteFactory) CheckConfig(config *common.Config) error {
+	panic("implement me")
+}
+
+
+
+
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
index 413db1baade..d4e9d9eaba2 100644
--- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
@@ -19,7 +19,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/elastic/beats/v7/heartbeat/beater"
 	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
@@ -30,10 +29,6 @@ import (
 
 const debugSelector = "synthexec"
 
-func init() {
-	beater.RegisterJourneyLister(InitSuite)
-}
-
 func InitSuite(ctx context.Context, origSuitePath string, params common.MapStr) (journeyNames []string, err error) {
 	dir, err := ioutil.TempDir("/tmp", "elastic-synthetics-")
 	if err != nil {

From 3ba5203843bf523007075219b4ce6d925f117eea Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Wed, 6 Jan 2021 10:09:18 -0600
Subject: [PATCH 07/41] Checkpoint

---
 heartbeat/beater/heartbeat.go                 | 33 ++----------------
 heartbeat/config/config.go                    | 34 -------------------
 .../monitors/browser/suitefactory/config.go   |  1 +
 3 files changed, 4 insertions(+), 64 deletions(-)
 create mode 100644 x-pack/heartbeat/monitors/browser/suitefactory/config.go

diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go
index faa82d5e393..0d282523ddd 100644
--- a/heartbeat/beater/heartbeat.go
+++ b/heartbeat/beater/heartbeat.go
@@ -179,42 +179,15 @@ func RegisterSuiteFactory(factory cfgfile.RunnerFactory) {
 
 // RunCentralMgmtMonitors loads any central management configured configs.
 func (bt *Heartbeat) RunSuiteMonitors(b *beat.Beat) {
+	if suiteFactory == nil {
+		return
+	}
 	monitors := cfgfile.NewRunnerList(management.DebugK, suiteFactory, b.Publisher)
 	reload.Register.MustRegisterList(b.Info.Beat+".suites", monitors)
 }
 
 type JourneyLister func(ctx context.Context, suitePath string, params common.MapStr) ([]string, error)
 
-// Provide hook to define journey list discovery from x-pack
-var suiteReloader SuiteReloader
-
-type SuiteReloader interface {
-	Check(factory *monitors.RunnerFactory) error
-	Run(factory *monitors.RunnerFactory) error
-}
-
-// RunReloadableMonitors runs the `heartbeat.config.monitors` portion of the yaml config if present.
-func (bt *Heartbeat) RunReloadableSuites(b *beat.Beat) (err error) {
-	// If we are running without XPack this will be nil
-	if suiteReloader == nil {
-		return nil
-	}
-
-	// Check monitor configs
-	if err := suiteReloader.Check(bt.dynamicFactory); err != nil {
-		logp.Error(errors.Wrap(err, "error loading reloadable monitors"))
-	}
-
-	// Execute the monitor
-	go suiteReloader.Run(bt.dynamicFactory)
-
-	return nil
-}
-
-func RegisterSuiteReloader(sr SuiteReloader) {
-	suiteReloader = sr
-}
-
 func (bt *Heartbeat) RunSyntheticSuiteMonitors(b *beat.Beat) error {
 	// If we are running without XPack this will be nil
 	if mainJourneyLister == nil {
diff --git a/heartbeat/config/config.go b/heartbeat/config/config.go
index eff6a257b85..8d60c9cb49a 100644
--- a/heartbeat/config/config.go
+++ b/heartbeat/config/config.go
@@ -41,39 +41,5 @@ type Scheduler struct {
 	Location string `config:"location"`
 }
 
-type SyntheticSuite struct {
-	Type	 string					`config:"type"`
-	Name     string                 `config:"id_prefix"`
-	Schedule string                 `config:"schedule"`
-	Params   map[string]interface{} `config:"params"`
-	RawConfig *common.Config
-}
-
-type LocalSyntheticSuite struct {
-	Path     string                 `config:"path"`
-	SyntheticSuite
-}
-
-type PollingSyntheticSuite struct {
-	CheckEvery int `config:"check_every"`
-	SyntheticSuite
-}
-
-// GithubSyntheticSuite handles configs for github repos, using the API defined here:
-// https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#download-a-repository-archive-tar.
-type GithubSyntheticSuite struct {
-	Owner	string `config:"owner"`
-	Repo string `config:"repo"`
-	Ref string `config:"ref"`
-	UrlBase string `config:"string"`
-	PollingSyntheticSuite
-}
-
-type ZipUrlSyntheticSuite struct {
-	Url string `config:"url""`
-	Headers map[string]string `config:"headers"`
-	PollingSyntheticSuite
-}
-
 // DefaultConfig is the canonical instantiation of Config.
 var DefaultConfig = Config{}
diff --git a/x-pack/heartbeat/monitors/browser/suitefactory/config.go b/x-pack/heartbeat/monitors/browser/suitefactory/config.go
new file mode 100644
index 00000000000..b98fe5a87c5
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/suitefactory/config.go
@@ -0,0 +1 @@
+package suitefactory

From d5aacfe7c459484d1b114d5532d908aceae569dc Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Wed, 6 Jan 2021 10:09:36 -0600
Subject: [PATCH 08/41] Checkpoint

---
 .../monitors/browser/suitefactory/config.go   | 36 ++++++++++
 .../browser/suitefactory/suitefactory.go      | 70 ++++++++++++++++++-
 2 files changed, 103 insertions(+), 3 deletions(-)

diff --git a/x-pack/heartbeat/monitors/browser/suitefactory/config.go b/x-pack/heartbeat/monitors/browser/suitefactory/config.go
index b98fe5a87c5..d5e9f508d41 100644
--- a/x-pack/heartbeat/monitors/browser/suitefactory/config.go
+++ b/x-pack/heartbeat/monitors/browser/suitefactory/config.go
@@ -1 +1,37 @@
 package suitefactory
+
+import "github.com/elastic/beats/v7/libbeat/common"
+
+type SyntheticSuite struct {
+	Type	 string					`config:"type"`
+	Name     string                 `config:"id_prefix"`
+	Schedule string                 `config:"schedule"`
+	Params   map[string]interface{} `config:"params"`
+	RawConfig *common.Config
+}
+
+type LocalSyntheticSuite struct {
+	Path     string                 `config:"path"`
+	SyntheticSuite
+}
+
+type PollingSyntheticSuite struct {
+	CheckEvery int `config:"check_every"`
+	SyntheticSuite
+}
+
+// GithubSyntheticSuite handles configs for github repos, using the API defined here:
+// https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#download-a-repository-archive-tar.
+type GithubSyntheticSuite struct {
+	Owner	string `config:"owner"`
+	Repo string `config:"repo"`
+	Ref string `config:"ref"`
+	UrlBase string `config:"string"`
+	PollingSyntheticSuite
+}
+
+type ZipUrlSyntheticSuite struct {
+	Url string `config:"url""`
+	Headers map[string]string `config:"headers"`
+	PollingSyntheticSuite
+}
diff --git a/x-pack/heartbeat/monitors/browser/suitefactory/suitefactory.go b/x-pack/heartbeat/monitors/browser/suitefactory/suitefactory.go
index 1af28db5ae0..340e1f70b09 100644
--- a/x-pack/heartbeat/monitors/browser/suitefactory/suitefactory.go
+++ b/x-pack/heartbeat/monitors/browser/suitefactory/suitefactory.go
@@ -1,10 +1,17 @@
 package suitefactory
 
 import (
+	"context"
+	"fmt"
 	"github.com/elastic/beats/v7/heartbeat/beater"
+	"github.com/elastic/beats/v7/heartbeat/monitors"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/cfgfile"
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/logp"
+	b "github.com/elastic/beats/v7/metricbeat/module/beat"
+	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/synthexec"
+	"github.com/pkg/errors"
 )
 
 func init() {
@@ -21,13 +28,70 @@ type StdSuiteFactory struct {
 }
 
 func (s *StdSuiteFactory) Create(p beat.PipelineConnector, config *common.Config) (cfgfile.Runner, error) {
-	panic("implement me")
+	suite := &SyntheticSuite{}
+	err := config.Unpack(suite)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse suite config: %w", err)
+	}
+
+	var suitePath string
+	var suiteParams map[string]interface{}
+	switch suite.Type {
+	case "zipurl":
+		unpacked := &ZipUrlSyntheticSuite{}
+		err := config.Unpack(unpacked)
+		if err != nil {
+			return nil, fmt.Errorf("could not parse zip URL synthetic suite: %w", err)
+		}
+	case "github":
+		unpacked := &GithubSyntheticSuite{}
+		err := config.Unpack(unpacked)
+		if err != nil {
+			return nil, fmt.Errorf("could not parse github synthetic suite: %w", err)
+		}
+	case "local":
+		unpacked := &LocalSyntheticSuite{}
+		err := config.Unpack(unpacked)
+		if err != nil {
+			return nil, fmt.Errorf("could not parse local synthetic suite: %w", err)
+		}
+		suitePath = unpacked.Path
+		suiteParams = unpacked.Params
+	default:
+		return nil, fmt.Errorf("suite type not specified! Expected 'local', 'github', or 'zipurl'")
+	}
+
+	logp.Info("Listing suite %s", suitePath)
+	journeyNames, err := synthexec.ListJourneys(context.TODO(), suitePath, suiteParams)
+	if err != nil {
+		return nil, err
+    }
+	logp.Warn("POSTLIST")
+	factory := monitors.NewFactory(b.Info, bt.scheduler, false)
+	for _, name := range journeyNames {
+		cfg, err := common.NewConfigFrom(map[string]interface{}{
+			"type":         "browser",
+			"path":         suiteReloader.WorkingPath(),
+			"schedule":     suite.Schedule,
+			"params":       suite.Params,
+			"journey_name": name,
+			"name":         name,
+			"id":           name,
+		})
+		if err != nil {
+				  return err
+				  }
+		created, err := factory.Create(b.Publisher, cfg)
+		if err != nil {
+				  return errors.Wrap(err, "could not create monitor")
+				  }
+		created.Start()
+	}
 }
 
 func (s *StdSuiteFactory) CheckConfig(config *common.Config) error {
-	panic("implement me")
+	return nil
 }
 
 
 
-

From 3e47422424ac9a1598a4edc0c54c3d8eae62d491 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Fri, 8 Jan 2021 09:19:20 -0600
Subject: [PATCH 09/41] Suites as jobs

---
 heartbeat/beater/heartbeat.go                 | 146 ------------------
 heartbeat/monitors/monitor.go                 |  23 ++-
 heartbeat/monitors/wrappers/monitors.go       |   6 +-
 x-pack/heartbeat/monitors/browser/browser.go  |  13 +-
 x-pack/heartbeat/monitors/browser/config.go   |   6 +-
 .../browser/suitefactory/suitefactory.go      |   2 +-
 .../monitors/synthetic_suite/config.go        |  81 ++++++++++
 .../monitors/synthetic_suite/suite_runner.go  |  66 ++++++++
 .../synthetic_suite/synthetic_suite.go        |  63 ++++++++
 .../{browser => }/synthexec/enrich.go         |   0
 .../{browser => }/synthexec/enrich_test.go    |   0
 .../synthexec/execmultiplexer.go              |   0
 .../{browser => }/synthexec/synthexec.go      |   4 +-
 .../{browser => }/synthexec/synthexec_test.go |   0
 .../{browser => }/synthexec/synthtypes.go     |   0
 .../synthexec/synthtypes_test.go              |   0
 .../{browser => }/synthexec/testcmd/main.go   |   0
 .../synthexec/testcmd/sample.ndjson           |   0
 18 files changed, 240 insertions(+), 170 deletions(-)
 create mode 100644 x-pack/heartbeat/monitors/synthetic_suite/config.go
 create mode 100644 x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go
 create mode 100644 x-pack/heartbeat/monitors/synthetic_suite/synthetic_suite.go
 rename x-pack/heartbeat/monitors/{browser => }/synthexec/enrich.go (100%)
 rename x-pack/heartbeat/monitors/{browser => }/synthexec/enrich_test.go (100%)
 rename x-pack/heartbeat/monitors/{browser => }/synthexec/execmultiplexer.go (100%)
 rename x-pack/heartbeat/monitors/{browser => }/synthexec/synthexec.go (98%)
 rename x-pack/heartbeat/monitors/{browser => }/synthexec/synthexec_test.go (100%)
 rename x-pack/heartbeat/monitors/{browser => }/synthexec/synthtypes.go (100%)
 rename x-pack/heartbeat/monitors/{browser => }/synthexec/synthtypes_test.go (100%)
 rename x-pack/heartbeat/monitors/{browser => }/synthexec/testcmd/main.go (100%)
 rename x-pack/heartbeat/monitors/{browser => }/synthexec/testcmd/sample.ndjson (100%)

diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go
index 0d282523ddd..48478d61499 100644
--- a/heartbeat/beater/heartbeat.go
+++ b/heartbeat/beater/heartbeat.go
@@ -18,9 +18,7 @@
 package beater
 
 import (
-	"context"
 	"fmt"
-	"io/ioutil"
 	"time"
 
 	"github.com/pkg/errors"
@@ -37,9 +35,6 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common/reload"
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/libbeat/management"
-
-
-	dirCopy "github.com/otiai10/copy"
 )
 
 // Heartbeat represents the root datastructure of this beat.
@@ -103,12 +98,6 @@ func (bt *Heartbeat) Run(b *beat.Beat) error {
 			return err
 		}
 	}
-	if len(bt.config.SyntheticSuites) > 0 {
-		err := bt.RunSyntheticSuiteMonitors(b)
-		if err != nil {
-			return err
-		}
-	}
 
 	if bt.config.Autodiscover != nil {
 		bt.autodiscover, err = bt.makeAutodiscover(b)
@@ -186,85 +175,6 @@ func (bt *Heartbeat) RunSuiteMonitors(b *beat.Beat) {
 	reload.Register.MustRegisterList(b.Info.Beat+".suites", monitors)
 }
 
-type JourneyLister func(ctx context.Context, suitePath string, params common.MapStr) ([]string, error)
-
-func (bt *Heartbeat) RunSyntheticSuiteMonitors(b *beat.Beat) error {
-	// If we are running without XPack this will be nil
-	if mainJourneyLister == nil {
-		return nil
-	}
-	for _, rawSuiteCfg := range bt.config.SyntheticSuites {
-		suite := &config.SyntheticSuite{}
-		err := rawSuiteCfg.Unpack(suite)
-		if err != nil {
-			logp.Err("could not parse suite config: %s", err)
-			continue
-		}
-
-		var suiteReloader SuiteReloader
-
-		switch suite.Type {
-		case "zipurl":
-			localConfig := &config.LocalSyntheticSuite{}
-			err := rawSuiteCfg.Unpack(localConfig)
-			if err != nil {
-				logp.Err("could not parse zip URL synthetic suite: %s", err)
-				continue
-			}
-		case "github":
-			localConfig := &config.LocalSyntheticSuite{}
-			err := rawSuiteCfg.Unpack(localConfig)
-			if err != nil {
-				logp.Err("could not parse github synthetic suite: %s", err)
-				continue
-			}
-		case "local":
-			localConfig := &config.LocalSyntheticSuite{}
-			err := rawSuiteCfg.Unpack(localConfig)
-			if err != nil {
-				logp.Err("could not parse local synthetic suite: %s", err)
-				continue
-			}
-			suiteReloader, err = NewLocalReloader(localConfig.Path)
-			if err != nil {
-				logp.Err("could not load local synthetics suite: %s", err)
-				continue
-			}
-		default:
-			return fmt.Errorf("suite type not specified! Expected 'local', 'github', or 'zipurl'")
-		}
-
-		logp.Warn("PRELIST %v", suiteReloader)
-		logp.Info("Listing suite %s", suiteReloader.WorkingPath())
-		journeyNames, err := mainJourneyLister(context.TODO(), suiteReloader.WorkingPath(), suite.Params)
-		if err != nil {
-			return err
-		}
-		logp.Warn("POSTLIST")
-		factory := monitors.NewFactory(b.Info, bt.scheduler, false)
-		for _, name := range journeyNames {
-			cfg, err := common.NewConfigFrom(map[string]interface{}{
-				"type":         "browser",
-				"path":         suiteReloader.WorkingPath(),
-				"schedule":     suite.Schedule,
-				"params":       suite.Params,
-				"journey_name": name,
-				"name":         name,
-				"id":           name,
-			})
-			if err != nil {
-				return err
-			}
-			created, err := factory.Create(b.Publisher, cfg)
-			if err != nil {
-				return errors.Wrap(err, "could not create monitor")
-			}
-			created.Start()
-		}
-	}
-	return nil
-}
-
 // makeAutodiscover creates an autodiscover object ready to be started.
 func (bt *Heartbeat) makeAutodiscover(b *beat.Beat) (*autodiscover.Autodiscover, error) {
 	autodiscover, err := autodiscover.NewAutodiscover(
@@ -285,59 +195,3 @@ func (bt *Heartbeat) makeAutodiscover(b *beat.Beat) (*autodiscover.Autodiscover,
 func (bt *Heartbeat) Stop() {
 	close(bt.done)
 }
-
-func NewLocalReloader(origSuitePath string) (*LocalReloader, error) {
-	dir, err := ioutil.TempDir("/tmp", "elastic-synthetics-")
-	if err != nil {
-		return nil, err
-	}
-
-	err = dirCopy.Copy(origSuitePath, dir)
-	if err != nil {
-		return nil, err
-	}
-
-	return &LocalReloader{workPath: origSuitePath}, nil
-}
-
-type LocalReloader struct {
-	origPath string
-	workPath string
-}
-
-func (l *LocalReloader) String() string {
-	return fmt.Sprintf("[Local Synthetics Suite origPath=%s workingPath=%s]", l.origPath, l.WorkingPath())
-}
-
-// Only loads once on startup, no rechecks
-func (l *LocalReloader) Check() (bool, error) {
-	return false, nil
-}
-
-func (l *LocalReloader) WorkingPath() string {
-	return l.workPath
-}
-
-func NewZipURLReloader(url string, headers map[string]string) (*ZipURLReloader, error) {
-	return &ZipURLReloader{URL: url, Headers: headers}, nil
-}
-
-type ZipURLReloader struct {
-	URL string
-	Headers map[string]string
-	etag string // used to determine if the URL contents has changed
-	workingPath string
-}
-
-func (z *ZipURLReloader) String() string {
-	return fmt.Sprintf("[ZipURL Synthetics suite url=%s workingPath=%s]", z.URL, z.workingPath)
-}
-
-func (z *ZipURLReloader) Check() (bool, error) {
-	panic("implement me")
-}
-
-func (z *ZipURLReloader) WorkingPath() string {
-	panic("implement me")
-}
-
diff --git a/heartbeat/monitors/monitor.go b/heartbeat/monitors/monitor.go
index 66e7317482f..6ffea01126e 100644
--- a/heartbeat/monitors/monitor.go
+++ b/heartbeat/monitors/monitor.go
@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"github.com/elastic/beats/v7/libbeat/cfgfile"
 	"sync"
 
 	"github.com/elastic/beats/v7/heartbeat/monitors/stdfields"
@@ -93,6 +94,10 @@ func (e ErrDuplicateMonitorID) Error() string {
 	return fmt.Sprintf("monitor ID %s is configured for multiple monitors! IDs must be unique values.", e.ID)
 }
 
+type MonitorAlike interface {
+	cfgfile.Runner
+}
+
 // newMonitor Creates a new monitor, without leaking resources in the event of an error.
 func newMonitor(
 	config *common.Config,
@@ -100,7 +105,7 @@ func newMonitor(
 	pipelineConnector beat.PipelineConnector,
 	scheduler *scheduler.Scheduler,
 	allowWatches bool,
-) (*Monitor, error) {
+) (MonitorAlike, error) {
 	m, err := newMonitorUnsafe(config, registrar, pipelineConnector, scheduler, allowWatches)
 	if m != nil && err != nil {
 		m.Stop()
@@ -116,22 +121,28 @@ func newMonitorUnsafe(
 	pipelineConnector beat.PipelineConnector,
 	scheduler *scheduler.Scheduler,
 	allowWatches bool,
-) (*Monitor, error) {
+) (MonitorAlike, error) {
 	// Extract just the Id, Type, and Enabled fields from the config
 	// We'll parse things more precisely later once we know what exact type of
 	// monitor we have
-	stdFields, err := stdfields.ConfigToStdMonitorFields(config)
+	standardFields, err := stdfields.ConfigToStdMonitorFields(config)
 	if err != nil {
 		return nil, err
 	}
 
-	monitorPlugin, found := registrar.get(stdFields.Type)
+	monitorPlugin, found := registrar.get(standardFields.Type)
 	if !found {
-		return nil, fmt.Errorf("monitor type %v does not exist, valid types are %v", stdFields.Type, registrar.monitorNames())
+		return nil, fmt.Errorf("monitor type %v does not exist, valid types are %v", standardFields.Type, registrar.monitorNames())
+	}
+
+	if standardFields.Type == "synthetic_suite" {
+
+	} else {
+
 	}
 
 	m := &Monitor{
-		stdFields:         stdFields,
+		stdFields:         standardFields,
 		pluginName:        monitorPlugin.name,
 		scheduler:         scheduler,
 		configuredJobs:    []*configuredJob{},
diff --git a/heartbeat/monitors/wrappers/monitors.go b/heartbeat/monitors/wrappers/monitors.go
index 3b5965ac01c..0ad592ae78f 100644
--- a/heartbeat/monitors/wrappers/monitors.go
+++ b/heartbeat/monitors/wrappers/monitors.go
@@ -64,7 +64,11 @@ func addMonitorMeta(stdMonFields stdfields.StdMonitorFields, isMulti bool) jobs.
 			started := time.Now()
 			cont, e := job(event)
 			thisID := stdMonFields.ID
-
+			// Allow jobs to override the ID, useful for suites
+			// which do this logic on their own
+			if v, _ := event.GetValue("monitor.id"); v != nil {
+				thisID = v.(string)
+			}
 			if isMulti {
 				url, err := event.GetValue("url.full")
 				if err != nil {
diff --git a/x-pack/heartbeat/monitors/browser/browser.go b/x-pack/heartbeat/monitors/browser/browser.go
index f3cb1e6483e..f09e015084a 100644
--- a/x-pack/heartbeat/monitors/browser/browser.go
+++ b/x-pack/heartbeat/monitors/browser/browser.go
@@ -15,7 +15,7 @@ import (
 	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
-	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/synthexec"
+	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthexec"
 )
 
 func init() {
@@ -52,14 +52,7 @@ func create(name string, cfg *common.Config) (js []jobs.Job, endpoints int, err
 		return nil, 0, err
 	}
 
-	var j jobs.Job
-	if config.Path != "" {
-		j, err = synthexec.SuiteJob(context.TODO(), config.Path, config.JourneyName, config.Params)
-		if err != nil {
-			return nil, 0, err
-		}
-	} else {
-		j = synthexec.InlineJourneyJob(context.TODO(), config.Script, config.Params)
-	}
+
+	j := synthexec.InlineJourneyJob(context.TODO(), config.Script, config.Params)
 	return []jobs.Job{j}, 1, nil
 }
diff --git a/x-pack/heartbeat/monitors/browser/config.go b/x-pack/heartbeat/monitors/browser/config.go
index 9c998d7d036..b0a14d7dca0 100644
--- a/x-pack/heartbeat/monitors/browser/config.go
+++ b/x-pack/heartbeat/monitors/browser/config.go
@@ -11,15 +11,13 @@ import (
 )
 
 type Config struct {
-	Path        string        `config:"path"`
 	Script      string        `config:"script"`
 	Params      common.MapStr `config:"script_params"`
-	JourneyName string        `config:"journey_name"`
 }
 
 func (c *Config) Validate() error {
-	if c.Script != "" && c.Path != "" {
-		return fmt.Errorf("both path and script specified! Only one of these options may be present!")
+	if c.Script != "" {
+		return fmt.Errorf("no script specified for journey!")
 	}
 	return nil
 }
diff --git a/x-pack/heartbeat/monitors/browser/suitefactory/suitefactory.go b/x-pack/heartbeat/monitors/browser/suitefactory/suitefactory.go
index 340e1f70b09..e792f6ac12b 100644
--- a/x-pack/heartbeat/monitors/browser/suitefactory/suitefactory.go
+++ b/x-pack/heartbeat/monitors/browser/suitefactory/suitefactory.go
@@ -10,7 +10,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
 	b "github.com/elastic/beats/v7/metricbeat/module/beat"
-	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/synthexec"
+	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthexec"
 	"github.com/pkg/errors"
 )
 
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/config.go b/x-pack/heartbeat/monitors/synthetic_suite/config.go
new file mode 100644
index 00000000000..983d24e9937
--- /dev/null
+++ b/x-pack/heartbeat/monitors/synthetic_suite/config.go
@@ -0,0 +1,81 @@
+package synthetic_suite
+
+import (
+	"fmt"
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/otiai10/copy"
+	"io/ioutil"
+)
+
+type SuiteFetcher interface {
+	Fetch() error
+	Workdir() string
+}
+
+type BaseSuite struct {
+	Type	 string					`config:"type"`
+	Name     string                 `config:"id_prefix"`
+	Schedule string                 `config:"schedule"`
+	Params   map[string]interface{} `config:"params"`
+	RawConfig *common.Config
+}
+
+type PollingSuite struct {
+	CheckEvery int `config:"check_every"`
+	SyntheticSuite
+}
+
+type LocalSuite struct {
+	Path     string                 `config:"path"`
+	SyntheticSuite
+}
+
+func (l LocalSuite) Fetch() error {
+	dir, err := ioutil.TempDir("/tmp", "elastic-synthetics-")
+	if err != nil {
+		return fmt.Errorf("could not create tmp dir: %w", err)
+	}
+
+	err = copy.Copy(l.Path, dir)
+	if err != nil {
+		return fmt.Errorf("could not copy suite: %w", err)
+	}
+	return nil
+}
+
+func (l LocalSuite) Workdir() string {
+	panic("implement me")
+}
+
+// GithubSuite handles configs for github repos, using the API defined here:
+// https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#download-a-repository-archive-tar.
+type GithubSuite struct {
+	Owner	string `config:"owner"`
+	Repo string `config:"repo"`
+	Ref string `config:"ref"`
+	UrlBase string `config:"string"`
+	PollingSuite
+}
+
+func (g GithubSuite) Fetch() error {
+	panic("implement me")
+}
+
+func (g GithubSuite) Workdir() string {
+	panic("implement me")
+}
+
+type ZipURLSuite struct {
+	Url string `config:"url"`
+	Headers map[string]string `config:"headers"`
+	PollingSuite
+}
+
+func (z ZipURLSuite) Fetch() error {
+	panic("implement me")
+}
+
+func (z ZipURLSuite) Workdir() string {
+	panic("implement me")
+}
+
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go b/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go
new file mode 100644
index 00000000000..5f701080ff8
--- /dev/null
+++ b/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go
@@ -0,0 +1,66 @@
+package synthetic_suite
+
+import (
+	"context"
+	"fmt"
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/logp"
+)
+
+
+type JourneyLister func(ctx context.Context, suitePath string, params common.MapStr) (journeyNames []string, err error)
+
+var journeyListSingleton JourneyLister
+
+func RegisterJourneyLister(jl JourneyLister) {
+	journeyListSingleton = jl
+}
+
+type SyntheticSuite struct {
+	rawCfg          *common.Config
+	suiteCfg        *BaseSuite
+	fetcher         SuiteFetcher
+}
+
+func NewSuite(rawCfg *common.Config) (ss *SyntheticSuite, err error) {
+	if journeyListSingleton == nil {
+		return nil, fmt.Errorf("synthetic monitoring is only supported with x-pack heartbeat")
+	}
+
+	ss = &SyntheticSuite{
+		rawCfg: rawCfg,
+	}
+
+	err = rawCfg.Unpack(ss.suiteCfg)
+	if err != nil {
+		logp.Err("could not parse suite config: %s", err)
+	}
+
+	switch ss.suiteCfg.Type {
+	case "local":
+		ss.fetcher = LocalSuite{}
+	case "github":
+		ss.fetcher = GithubSuite{}
+	case "zip_url":
+		ss.fetcher = ZipURLSuite{}
+	}
+
+	err = ss.rawCfg.Unpack(&ss.fetcher)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse local synthetic suite: %s", err)
+	}
+
+	return
+}
+
+func (s *SyntheticSuite) String() string {
+	panic("implement me")
+}
+
+func (s *SyntheticSuite) Start() {
+	panic("implement me")
+}
+
+func (s *SyntheticSuite) Stop() {
+	panic("implement me")
+}
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/synthetic_suite.go b/x-pack/heartbeat/monitors/synthetic_suite/synthetic_suite.go
new file mode 100644
index 00000000000..8165bb4b405
--- /dev/null
+++ b/x-pack/heartbeat/monitors/synthetic_suite/synthetic_suite.go
@@ -0,0 +1,63 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package synthetic_suite
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/user"
+	"sync"
+
+	"github.com/elastic/beats/v7/heartbeat/monitors"
+	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/logp"
+	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthexec"
+)
+
+func init() {
+	monitors.RegisterActive("synthetic_suite", create)
+	monitors.RegisterActive("synthetic/suite", create)
+}
+
+var showExperimentalOnce = sync.Once{}
+
+var NotSyntheticsCapableError = fmt.Errorf("synthetic monitors cannot be created outside the official elastic docker image")
+
+func create(name string, cfg *common.Config) (js []jobs.Job, endpoints int, err error) {
+	// We don't want users running synthetics in environments that don't have the required GUI libraries etc, so we check
+	// this flag. When we're ready to support the many possible configurations of systems outside the docker environment
+	// we can remove this check.
+	if os.Getenv("ELASTIC_SYNTHETICS_CAPABLE") != "true" {
+		return nil, 0, NotSyntheticsCapableError
+	}
+
+	showExperimentalOnce.Do(func() {
+		logp.Info("Synthetic monitor detected! Please note synthetic monitors are an experimental unsupported feature!")
+	})
+
+	curUser, err := user.Current()
+	if err != nil {
+		return nil, 0, fmt.Errorf("could not determine current user for script monitor %w: ", err)
+	}
+	if curUser.Uid == "0" {
+		return nil, 0, fmt.Errorf("script monitors cannot be run as root! Current UID is %s", curUser.Uid)
+	}
+
+	ss, err := NewSuite(cfg)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	// TODO: Run this before each suite job invocation, not just at startup
+	ss.fetcher.Fetch()
+
+	j, err := synthexec.SuiteJob(context.TODO(), ss.fetcher.Workdir(), ss.suiteCfg.Params)
+	if err != nil {
+			return nil, 0, err
+		}
+	return []jobs.Job{j}, 1, nil
+}
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/enrich.go b/x-pack/heartbeat/monitors/synthexec/enrich.go
similarity index 100%
rename from x-pack/heartbeat/monitors/browser/synthexec/enrich.go
rename to x-pack/heartbeat/monitors/synthexec/enrich.go
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go b/x-pack/heartbeat/monitors/synthexec/enrich_test.go
similarity index 100%
rename from x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go
rename to x-pack/heartbeat/monitors/synthexec/enrich_test.go
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go b/x-pack/heartbeat/monitors/synthexec/execmultiplexer.go
similarity index 100%
rename from x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go
rename to x-pack/heartbeat/monitors/synthexec/execmultiplexer.go
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go b/x-pack/heartbeat/monitors/synthexec/synthexec.go
similarity index 98%
rename from x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
rename to x-pack/heartbeat/monitors/synthexec/synthexec.go
index d4e9d9eaba2..ecd7f9f1e19 100644
--- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
+++ b/x-pack/heartbeat/monitors/synthexec/synthexec.go
@@ -89,8 +89,8 @@ Outer:
 }
 
 // SuiteJob will run a single journey by name from the given suite.
-func SuiteJob(ctx context.Context, suiteFile string, journeyName string, params common.MapStr) (jobs.Job, error) {
-	newCmd, err := suiteCommandFactory(suiteFile, suiteFile, "--screenshots", "--journey-name", journeyName)
+func SuiteJob(ctx context.Context, suiteFile string, params common.MapStr) (jobs.Job, error) {
+	newCmd, err := suiteCommandFactory(suiteFile, suiteFile, "--screenshots")
 	if err != nil {
 		return nil, err
 	}
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go b/x-pack/heartbeat/monitors/synthexec/synthexec_test.go
similarity index 100%
rename from x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go
rename to x-pack/heartbeat/monitors/synthexec/synthexec_test.go
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go b/x-pack/heartbeat/monitors/synthexec/synthtypes.go
similarity index 100%
rename from x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go
rename to x-pack/heartbeat/monitors/synthexec/synthtypes.go
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go b/x-pack/heartbeat/monitors/synthexec/synthtypes_test.go
similarity index 100%
rename from x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go
rename to x-pack/heartbeat/monitors/synthexec/synthtypes_test.go
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/testcmd/main.go b/x-pack/heartbeat/monitors/synthexec/testcmd/main.go
similarity index 100%
rename from x-pack/heartbeat/monitors/browser/synthexec/testcmd/main.go
rename to x-pack/heartbeat/monitors/synthexec/testcmd/main.go
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/testcmd/sample.ndjson b/x-pack/heartbeat/monitors/synthexec/testcmd/sample.ndjson
similarity index 100%
rename from x-pack/heartbeat/monitors/browser/synthexec/testcmd/sample.ndjson
rename to x-pack/heartbeat/monitors/synthexec/testcmd/sample.ndjson

From f7971f4c7b5dab2132e744feea53f986b8c6d4b7 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Fri, 8 Jan 2021 10:13:18 -0600
Subject: [PATCH 10/41] Checkpoint

---
 x-pack/heartbeat/main.go                      |  3 +-
 .../{browser => synthetic_script}/config.go   |  2 +-
 .../suitefactory/config.go                    |  0
 .../suitefactory/suitefactory.go              |  0
 .../synthetic_script.go}                      |  7 +-
 .../monitors/synthetic_suite/config.go        | 77 ++++++++++++++-----
 .../monitors/synthetic_suite/suite_runner.go  | 40 ++++------
 .../synthetic_suite/synthetic_suite.go        |  4 +-
 .../sample-synthetics-config/heartbeat.yml    | 24 +++---
 9 files changed, 91 insertions(+), 66 deletions(-)
 rename x-pack/heartbeat/monitors/{browser => synthetic_script}/config.go (95%)
 rename x-pack/heartbeat/monitors/{browser => synthetic_script}/suitefactory/config.go (100%)
 rename x-pack/heartbeat/monitors/{browser => synthetic_script}/suitefactory/suitefactory.go (100%)
 rename x-pack/heartbeat/monitors/{browser/browser.go => synthetic_script/synthetic_script.go} (88%)

diff --git a/x-pack/heartbeat/main.go b/x-pack/heartbeat/main.go
index 4c0f220edf6..18fcad761e5 100644
--- a/x-pack/heartbeat/main.go
+++ b/x-pack/heartbeat/main.go
@@ -9,7 +9,8 @@ import (
 
 	_ "github.com/elastic/beats/v7/heartbeat/include"
 	"github.com/elastic/beats/v7/x-pack/heartbeat/cmd"
-	_ "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser"
+	_ "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthetic_script"
+	_ "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthetic_suite"
 )
 
 func main() {
diff --git a/x-pack/heartbeat/monitors/browser/config.go b/x-pack/heartbeat/monitors/synthetic_script/config.go
similarity index 95%
rename from x-pack/heartbeat/monitors/browser/config.go
rename to x-pack/heartbeat/monitors/synthetic_script/config.go
index b0a14d7dca0..205049e3709 100644
--- a/x-pack/heartbeat/monitors/browser/config.go
+++ b/x-pack/heartbeat/monitors/synthetic_script/config.go
@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package browser
+package synthetic_script
 
 import (
 	"fmt"
diff --git a/x-pack/heartbeat/monitors/browser/suitefactory/config.go b/x-pack/heartbeat/monitors/synthetic_script/suitefactory/config.go
similarity index 100%
rename from x-pack/heartbeat/monitors/browser/suitefactory/config.go
rename to x-pack/heartbeat/monitors/synthetic_script/suitefactory/config.go
diff --git a/x-pack/heartbeat/monitors/browser/suitefactory/suitefactory.go b/x-pack/heartbeat/monitors/synthetic_script/suitefactory/suitefactory.go
similarity index 100%
rename from x-pack/heartbeat/monitors/browser/suitefactory/suitefactory.go
rename to x-pack/heartbeat/monitors/synthetic_script/suitefactory/suitefactory.go
diff --git a/x-pack/heartbeat/monitors/browser/browser.go b/x-pack/heartbeat/monitors/synthetic_script/synthetic_script.go
similarity index 88%
rename from x-pack/heartbeat/monitors/browser/browser.go
rename to x-pack/heartbeat/monitors/synthetic_script/synthetic_script.go
index f09e015084a..e041fcf6bdc 100644
--- a/x-pack/heartbeat/monitors/browser/browser.go
+++ b/x-pack/heartbeat/monitors/synthetic_script/synthetic_script.go
@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package browser
+package synthetic_script
 
 import (
 	"context"
@@ -19,8 +19,11 @@ import (
 )
 
 func init() {
+	// TODO: Remove this deprecated option, it was part of experimental,
+	// so no need for deprecation phase
 	monitors.RegisterActive("browser", create)
-	monitors.RegisterActive("synthetic/browser", create)
+	monitors.RegisterActive("synthetic_script", create)
+	monitors.RegisterActive("synthetic/script", create)
 }
 
 var showExperimentalOnce = sync.Once{}
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/config.go b/x-pack/heartbeat/monitors/synthetic_suite/config.go
index 983d24e9937..fe025880375 100644
--- a/x-pack/heartbeat/monitors/synthetic_suite/config.go
+++ b/x-pack/heartbeat/monitors/synthetic_suite/config.go
@@ -3,34 +3,69 @@ package synthetic_suite
 import (
 	"fmt"
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/otiai10/copy"
 	"io/ioutil"
 )
 
-type SuiteFetcher interface {
+type Config struct {
+	Schedule string                 `config:"schedule"`
+	Params   map[string]interface{} `config:"params"`
+	RawConfig *common.Config
+	Source *Source `config:"source"`
+}
+
+type Source struct {
+	Local      *LocalSource  `config:"local"`
+	Github     *GithubSource `config:"github"`
+	ZipURL     *ZipURLSource `config:"zip_url"`
+	ActiveMemo ISource       // cache for selected source
+}
+
+func (s *Source) active() ISource {
+	logp.Warn("IN ACTIVE!!!")
+	if s.ActiveMemo != nil {
+		return s.ActiveMemo
+	}
+
+	if s.Local != nil {
+		s.ActiveMemo = s.Local
+	} else if s.Github != nil {
+		s.ActiveMemo = s.Github
+	} else if s.ZipURL != nil {
+		s.ActiveMemo = s.ZipURL
+	}
+
+	return s.ActiveMemo
+}
+
+func (s *Source) Validate() error {
+	if  s.active() == nil {
+		return fmt.Errorf("no valid source specified! Choose one of local, github, zip_url")
+	}
+	return nil
+}
+
+type ISource interface {
 	Fetch() error
 	Workdir() string
 }
 
-type BaseSuite struct {
+type BaseSource struct {
 	Type	 string					`config:"type"`
-	Name     string                 `config:"id_prefix"`
-	Schedule string                 `config:"schedule"`
-	Params   map[string]interface{} `config:"params"`
-	RawConfig *common.Config
 }
 
-type PollingSuite struct {
+type PollingSource struct {
 	CheckEvery int `config:"check_every"`
-	SyntheticSuite
+	BaseSource
 }
 
-type LocalSuite struct {
+type LocalSource struct {
 	Path     string                 `config:"path"`
-	SyntheticSuite
+	BaseSource
 }
 
-func (l LocalSuite) Fetch() error {
+func (l *LocalSource) Fetch() error {
 	dir, err := ioutil.TempDir("/tmp", "elastic-synthetics-")
 	if err != nil {
 		return fmt.Errorf("could not create tmp dir: %w", err)
@@ -43,39 +78,39 @@ func (l LocalSuite) Fetch() error {
 	return nil
 }
 
-func (l LocalSuite) Workdir() string {
+func (l *LocalSource) Workdir() string {
 	panic("implement me")
 }
 
-// GithubSuite handles configs for github repos, using the API defined here:
+// GithubSource handles configs for github repos, using the API defined here:
 // https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#download-a-repository-archive-tar.
-type GithubSuite struct {
+type GithubSource struct {
 	Owner	string `config:"owner"`
 	Repo string `config:"repo"`
 	Ref string `config:"ref"`
 	UrlBase string `config:"string"`
-	PollingSuite
+	PollingSource
 }
 
-func (g GithubSuite) Fetch() error {
+func (g *GithubSource) Fetch() error {
 	panic("implement me")
 }
 
-func (g GithubSuite) Workdir() string {
+func (g *GithubSource) Workdir() string {
 	panic("implement me")
 }
 
-type ZipURLSuite struct {
+type ZipURLSource struct {
 	Url string `config:"url"`
 	Headers map[string]string `config:"headers"`
-	PollingSuite
+	PollingSource
 }
 
-func (z ZipURLSuite) Fetch() error {
+func (z *ZipURLSource) Fetch() error {
 	panic("implement me")
 }
 
-func (z ZipURLSuite) Workdir() string {
+func (z *ZipURLSource) Workdir() string {
 	panic("implement me")
 }
 
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go b/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go
index 5f701080ff8..5fa742683c2 100644
--- a/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go
+++ b/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go
@@ -2,7 +2,6 @@ package synthetic_suite
 
 import (
 	"context"
-	"fmt"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
@@ -12,21 +11,12 @@ type JourneyLister func(ctx context.Context, suitePath string, params common.Map
 
 var journeyListSingleton JourneyLister
 
-func RegisterJourneyLister(jl JourneyLister) {
-	journeyListSingleton = jl
-}
-
 type SyntheticSuite struct {
-	rawCfg          *common.Config
-	suiteCfg        *BaseSuite
-	fetcher         SuiteFetcher
+	rawCfg   *common.Config
+	suiteCfg *Config
 }
 
 func NewSuite(rawCfg *common.Config) (ss *SyntheticSuite, err error) {
-	if journeyListSingleton == nil {
-		return nil, fmt.Errorf("synthetic monitoring is only supported with x-pack heartbeat")
-	}
-
 	ss = &SyntheticSuite{
 		rawCfg: rawCfg,
 	}
@@ -36,20 +26,6 @@ func NewSuite(rawCfg *common.Config) (ss *SyntheticSuite, err error) {
 		logp.Err("could not parse suite config: %s", err)
 	}
 
-	switch ss.suiteCfg.Type {
-	case "local":
-		ss.fetcher = LocalSuite{}
-	case "github":
-		ss.fetcher = GithubSuite{}
-	case "zip_url":
-		ss.fetcher = ZipURLSuite{}
-	}
-
-	err = ss.rawCfg.Unpack(&ss.fetcher)
-	if err != nil {
-		return nil, fmt.Errorf("could not parse local synthetic suite: %s", err)
-	}
-
 	return
 }
 
@@ -57,6 +33,14 @@ func (s *SyntheticSuite) String() string {
 	panic("implement me")
 }
 
+func (s *SyntheticSuite) Fetch() error {
+	return s.suiteCfg.Source.active().Fetch()
+}
+
+func (s *SyntheticSuite) Workdir() string {
+	return s.suiteCfg.Source.active().Workdir()
+}
+
 func (s *SyntheticSuite) Start() {
 	panic("implement me")
 }
@@ -64,3 +48,7 @@ func (s *SyntheticSuite) Start() {
 func (s *SyntheticSuite) Stop() {
 	panic("implement me")
 }
+
+func (s *SyntheticSuite) Params() map[string]interface{} {
+	return s.suiteCfg.Params
+}
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/synthetic_suite.go b/x-pack/heartbeat/monitors/synthetic_suite/synthetic_suite.go
index 8165bb4b405..42fd70fb13a 100644
--- a/x-pack/heartbeat/monitors/synthetic_suite/synthetic_suite.go
+++ b/x-pack/heartbeat/monitors/synthetic_suite/synthetic_suite.go
@@ -53,9 +53,9 @@ func create(name string, cfg *common.Config) (js []jobs.Job, endpoints int, err
 	}
 
 	// TODO: Run this before each suite job invocation, not just at startup
-	ss.fetcher.Fetch()
+	ss.Fetch()
 
-	j, err := synthexec.SuiteJob(context.TODO(), ss.fetcher.Workdir(), ss.suiteCfg.Params)
+	j, err := synthexec.SuiteJob(context.TODO(), ss.Workdir(), ss.Params())
 	if err != nil {
 			return nil, 0, err
 		}
diff --git a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
index 28f4461a840..9df063eb539 100644
--- a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
+++ b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
@@ -4,19 +4,22 @@ heartbeat.config.monitors:
   reload.enabled: false
   reload.period: 5s
 
-heartbeat.synthetic_suites:
-- name: Todos suite
-  type: "local"
-  path: "/home/andrewvc/projects/synthetics/examples/todos"
-  schedule: "@every 1m"
-
 heartbeat.monitors:
+- type: synthetic_suite
+  enabled: true
+  id: todos-suite
+  name: Todos Suite
+  source:
+    local:
+      path: "/home/andrewvc/projects/synthetics/examples/todos"
+  schedule: '@every 1m'
 - type: http
+  enabled: false
   id: SimpleHTTP
   urls: http://www.google.com
   schedule: "@every 15s"
   name: Simple HTTP
-- type: browser
+- type: synthetic_script
   enabled: false
   id: my-monitor
   name: My Monitor
@@ -39,12 +42,7 @@ setup.template.settings:
   index.number_of_shards: 1
   index.codec: best_compression
 setup.kibana:
-output.elasticsearch:
-  hosts:
-  - localhost:9200
-  protocol: http
-  username: elastic
-  password: changeme
+output.console: ~
 processors:
 - add_observer_metadata:
 

From fd4d03db005fb1dce9fcc596f0ee874253a6aec2 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Sat, 9 Jan 2021 21:57:34 -0600
Subject: [PATCH 11/41] Checkpoint

---
 x-pack/heartbeat/monitors/synthetic_suite/config.go | 13 ++++++-------
 .../monitors/synthetic_suite/suite_runner.go        | 10 +++++-----
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/x-pack/heartbeat/monitors/synthetic_suite/config.go b/x-pack/heartbeat/monitors/synthetic_suite/config.go
index fe025880375..4851dcf9b71 100644
--- a/x-pack/heartbeat/monitors/synthetic_suite/config.go
+++ b/x-pack/heartbeat/monitors/synthetic_suite/config.go
@@ -39,12 +39,12 @@ func (s *Source) active() ISource {
 	return s.ActiveMemo
 }
 
-func (s *Source) Validate() error {
-	if  s.active() == nil {
-		return fmt.Errorf("no valid source specified! Choose one of local, github, zip_url")
-	}
-	return nil
-}
+//func (s *Source) Validate() error {
+//	if  s.active() == nil {
+//		return fmt.Errorf("no valid source specified! Choose one of local, github, zip_url")
+//	}
+//	return nil
+//}
 
 type ISource interface {
 	Fetch() error
@@ -113,4 +113,3 @@ func (z *ZipURLSource) Fetch() error {
 func (z *ZipURLSource) Workdir() string {
 	panic("implement me")
 }
-
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go b/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go
index 5fa742683c2..6814d6aaf21 100644
--- a/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go
+++ b/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go
@@ -16,17 +16,17 @@ type SyntheticSuite struct {
 	suiteCfg *Config
 }
 
-func NewSuite(rawCfg *common.Config) (ss *SyntheticSuite, err error) {
-	ss = &SyntheticSuite{
+func NewSuite(rawCfg *common.Config) (*SyntheticSuite, error) {
+	ss := &SyntheticSuite{
 		rawCfg: rawCfg,
+		suiteCfg: &Config{},
 	}
-
-	err = rawCfg.Unpack(ss.suiteCfg)
+	err := rawCfg.Unpack(ss.suiteCfg)
 	if err != nil {
 		logp.Err("could not parse suite config: %s", err)
 	}
 
-	return
+	return ss, err
 }
 
 func (s *SyntheticSuite) String() string {

From d27f69595cb04b5a4c6269f69cc7917f83f6b4d6 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Sun, 10 Jan 2021 08:45:02 -0600
Subject: [PATCH 12/41] Sort of works

---
 .../monitors/synthetic_suite/config.go        | 25 +++++++++----------
 .../monitors/synthetic_suite/sources.go       |  1 +
 .../heartbeat/monitors/synthexec/synthexec.go | 11 ++++++--
 3 files changed, 22 insertions(+), 15 deletions(-)
 create mode 100644 x-pack/heartbeat/monitors/synthetic_suite/sources.go

diff --git a/x-pack/heartbeat/monitors/synthetic_suite/config.go b/x-pack/heartbeat/monitors/synthetic_suite/config.go
index 4851dcf9b71..f8acd8c844c 100644
--- a/x-pack/heartbeat/monitors/synthetic_suite/config.go
+++ b/x-pack/heartbeat/monitors/synthetic_suite/config.go
@@ -3,7 +3,6 @@ package synthetic_suite
 import (
 	"fmt"
 	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/otiai10/copy"
 	"io/ioutil"
 )
@@ -23,7 +22,6 @@ type Source struct {
 }
 
 func (s *Source) active() ISource {
-	logp.Warn("IN ACTIVE!!!")
 	if s.ActiveMemo != nil {
 		return s.ActiveMemo
 	}
@@ -39,12 +37,12 @@ func (s *Source) active() ISource {
 	return s.ActiveMemo
 }
 
-//func (s *Source) Validate() error {
-//	if  s.active() == nil {
-//		return fmt.Errorf("no valid source specified! Choose one of local, github, zip_url")
-//	}
-//	return nil
-//}
+func (s *Source) Validate() error {
+	if  s.active() == nil {
+		return fmt.Errorf("no valid source specified! Choose one of local, github, zip_url")
+	}
+	return nil
+}
 
 type ISource interface {
 	Fetch() error
@@ -61,17 +59,18 @@ type PollingSource struct {
 }
 
 type LocalSource struct {
-	Path     string                 `config:"path"`
+	OrigPath     string                 `config:"path"`
+	workingPath string
 	BaseSource
 }
 
-func (l *LocalSource) Fetch() error {
-	dir, err := ioutil.TempDir("/tmp", "elastic-synthetics-")
+func (l *LocalSource) Fetch() (err error) {
+	l.workingPath, err = ioutil.TempDir("/tmp", "elastic-synthetics-")
 	if err != nil {
 		return fmt.Errorf("could not create tmp dir: %w", err)
 	}
 
-	err = copy.Copy(l.Path, dir)
+	err = copy.Copy(l.OrigPath, l.workingPath)
 	if err != nil {
 		return fmt.Errorf("could not copy suite: %w", err)
 	}
@@ -79,7 +78,7 @@ func (l *LocalSource) Fetch() error {
 }
 
 func (l *LocalSource) Workdir() string {
-	panic("implement me")
+	return l.workingPath
 }
 
 // GithubSource handles configs for github repos, using the API defined here:
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/sources.go b/x-pack/heartbeat/monitors/synthetic_suite/sources.go
new file mode 100644
index 00000000000..005d1f68f34
--- /dev/null
+++ b/x-pack/heartbeat/monitors/synthetic_suite/sources.go
@@ -0,0 +1 @@
+package synthetic_suite
diff --git a/x-pack/heartbeat/monitors/synthexec/synthexec.go b/x-pack/heartbeat/monitors/synthexec/synthexec.go
index ecd7f9f1e19..c2532c78c5b 100644
--- a/x-pack/heartbeat/monitors/synthexec/synthexec.go
+++ b/x-pack/heartbeat/monitors/synthexec/synthexec.go
@@ -338,7 +338,14 @@ func runSimpleCommand(cmd *exec.Cmd, dir string) error {
 	return err
 }
 
+// getNpmRoot gets the closest ancestor path that contains package.json.
 func getNpmRoot(path string) (string, error) {
+	return getNpmRootIn(path, path)
+}
+
+// getNpmRootIn does the same as getNpmRoot but remembers the original path for
+// debugging.
+func getNpmRootIn(path, origPath string) (string, error) {
 	candidate := filepath.Join(path, "package.json")
 	_, err := os.Lstat(candidate)
 	if err == nil {
@@ -347,7 +354,7 @@ func getNpmRoot(path string) (string, error) {
 	// Try again one level up
 	parent := filepath.Dir(path)
 	if len(parent) < 2 {
-		return "", fmt.Errorf("no package.json found")
+		return "", fmt.Errorf("no package.json found in %s", origPath)
 	}
-	return getNpmRoot(parent)
+	return getNpmRootIn(parent, origPath)
 }

From 47320ce8a99f5f34e4afa2485eec3a93347412b0 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Tue, 12 Jan 2021 13:44:08 -0600
Subject: [PATCH 13/41] Reorg

---
 .../monitors/synthetic_suite/config.go        | 102 +-----------------
 .../monitors/synthetic_suite/source/github.go |  19 ++++
 .../monitors/synthetic_suite/source/local.go  |  30 ++++++
 .../monitors/synthetic_suite/source/source.go |  50 +++++++++
 .../monitors/synthetic_suite/source/zipurl.go |  15 +++
 .../monitors/synthetic_suite/suite_runner.go  |  11 +-
 6 files changed, 118 insertions(+), 109 deletions(-)
 create mode 100644 x-pack/heartbeat/monitors/synthetic_suite/source/github.go
 create mode 100644 x-pack/heartbeat/monitors/synthetic_suite/source/local.go
 create mode 100644 x-pack/heartbeat/monitors/synthetic_suite/source/source.go
 create mode 100644 x-pack/heartbeat/monitors/synthetic_suite/source/zipurl.go

diff --git a/x-pack/heartbeat/monitors/synthetic_suite/config.go b/x-pack/heartbeat/monitors/synthetic_suite/config.go
index f8acd8c844c..88807cb8903 100644
--- a/x-pack/heartbeat/monitors/synthetic_suite/config.go
+++ b/x-pack/heartbeat/monitors/synthetic_suite/config.go
@@ -1,114 +1,16 @@
 package synthetic_suite
 
 import (
-	"fmt"
 	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/otiai10/copy"
-	"io/ioutil"
+	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthetic_suite/source"
 )
 
 type Config struct {
 	Schedule string                 `config:"schedule"`
 	Params   map[string]interface{} `config:"params"`
 	RawConfig *common.Config
-	Source *Source `config:"source"`
+	Source *source.Source `config:"source"`
 }
 
-type Source struct {
-	Local      *LocalSource  `config:"local"`
-	Github     *GithubSource `config:"github"`
-	ZipURL     *ZipURLSource `config:"zip_url"`
-	ActiveMemo ISource       // cache for selected source
-}
-
-func (s *Source) active() ISource {
-	if s.ActiveMemo != nil {
-		return s.ActiveMemo
-	}
-
-	if s.Local != nil {
-		s.ActiveMemo = s.Local
-	} else if s.Github != nil {
-		s.ActiveMemo = s.Github
-	} else if s.ZipURL != nil {
-		s.ActiveMemo = s.ZipURL
-	}
-
-	return s.ActiveMemo
-}
-
-func (s *Source) Validate() error {
-	if  s.active() == nil {
-		return fmt.Errorf("no valid source specified! Choose one of local, github, zip_url")
-	}
-	return nil
-}
-
-type ISource interface {
-	Fetch() error
-	Workdir() string
-}
-
-type BaseSource struct {
-	Type	 string					`config:"type"`
-}
-
-type PollingSource struct {
-	CheckEvery int `config:"check_every"`
-	BaseSource
-}
-
-type LocalSource struct {
-	OrigPath     string                 `config:"path"`
-	workingPath string
-	BaseSource
-}
 
-func (l *LocalSource) Fetch() (err error) {
-	l.workingPath, err = ioutil.TempDir("/tmp", "elastic-synthetics-")
-	if err != nil {
-		return fmt.Errorf("could not create tmp dir: %w", err)
-	}
 
-	err = copy.Copy(l.OrigPath, l.workingPath)
-	if err != nil {
-		return fmt.Errorf("could not copy suite: %w", err)
-	}
-	return nil
-}
-
-func (l *LocalSource) Workdir() string {
-	return l.workingPath
-}
-
-// GithubSource handles configs for github repos, using the API defined here:
-// https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#download-a-repository-archive-tar.
-type GithubSource struct {
-	Owner	string `config:"owner"`
-	Repo string `config:"repo"`
-	Ref string `config:"ref"`
-	UrlBase string `config:"string"`
-	PollingSource
-}
-
-func (g *GithubSource) Fetch() error {
-	panic("implement me")
-}
-
-func (g *GithubSource) Workdir() string {
-	panic("implement me")
-}
-
-type ZipURLSource struct {
-	Url string `config:"url"`
-	Headers map[string]string `config:"headers"`
-	PollingSource
-}
-
-func (z *ZipURLSource) Fetch() error {
-	panic("implement me")
-}
-
-func (z *ZipURLSource) Workdir() string {
-	panic("implement me")
-}
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/source/github.go b/x-pack/heartbeat/monitors/synthetic_suite/source/github.go
new file mode 100644
index 00000000000..6e264a99430
--- /dev/null
+++ b/x-pack/heartbeat/monitors/synthetic_suite/source/github.go
@@ -0,0 +1,19 @@
+package source
+
+// GithubSource handles configs for github repos, using the API defined here:
+// https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#download-a-repository-archive-tar.
+type GithubSource struct {
+	Owner	string `config:"owner"`
+	Repo string `config:"repo"`
+	Ref string `config:"ref"`
+	UrlBase string `config:"url_base"`
+	PollingSource
+}
+
+func (g *GithubSource) Fetch() error {
+	panic("implement me")
+}
+
+func (g *GithubSource) Workdir() string {
+	panic("implement me")
+}
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/source/local.go b/x-pack/heartbeat/monitors/synthetic_suite/source/local.go
new file mode 100644
index 00000000000..4f0029e10ae
--- /dev/null
+++ b/x-pack/heartbeat/monitors/synthetic_suite/source/local.go
@@ -0,0 +1,30 @@
+package source
+
+import (
+	"fmt"
+	"io/ioutil"
+	"github.com/otiai10/copy"
+)
+
+type LocalSource struct {
+	OrigPath     string                `config:"path"`
+	workingPath string
+	BaseSource
+}
+
+func (l *LocalSource) Fetch() (err error) {
+	l.workingPath, err = ioutil.TempDir("/tmp", "elastic-synthetics-")
+	if err != nil {
+		return fmt.Errorf("could not create tmp dir: %w", err)
+	}
+
+	err = copy.Copy(l.OrigPath, l.workingPath)
+	if err != nil {
+		return fmt.Errorf("could not copy suite: %w", err)
+	}
+	return nil
+}
+
+func (l *LocalSource) Workdir() string {
+	return l.workingPath
+}
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/source/source.go b/x-pack/heartbeat/monitors/synthetic_suite/source/source.go
new file mode 100644
index 00000000000..4d40eca8e3a
--- /dev/null
+++ b/x-pack/heartbeat/monitors/synthetic_suite/source/source.go
@@ -0,0 +1,50 @@
+package source
+
+import (
+	"fmt"
+)
+
+type Source struct {
+	Local      *LocalSource  `config:"local"`
+	Github     *GithubSource `config:"github"`
+	ZipURL     *ZipURLSource `config:"zip_url"`
+	ActiveMemo ISource       // cache for selected source
+}
+
+func (s *Source) Active() ISource {
+	if s.ActiveMemo != nil {
+		return s.ActiveMemo
+	}
+
+	if s.Local != nil {
+		s.ActiveMemo = s.Local
+	} else if s.Github != nil {
+		s.ActiveMemo = s.Github
+	} else if s.ZipURL != nil {
+		s.ActiveMemo = s.ZipURL
+	}
+
+	return s.ActiveMemo
+}
+
+func (s *Source) Validate() error {
+	if  s.Active() == nil {
+		return fmt.Errorf("no valid source specified! Choose one of local, github, zip_url")
+	}
+	return nil
+}
+
+type ISource interface {
+	Fetch() error
+	Workdir() string
+}
+
+type BaseSource struct {
+	Type	 string					`config:"type"`
+}
+
+type PollingSource struct {
+	CheckEvery int `config:"check_every"`
+	BaseSource
+}
+
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/source/zipurl.go b/x-pack/heartbeat/monitors/synthetic_suite/source/zipurl.go
new file mode 100644
index 00000000000..fcc1fda1518
--- /dev/null
+++ b/x-pack/heartbeat/monitors/synthetic_suite/source/zipurl.go
@@ -0,0 +1,15 @@
+package source
+
+type ZipURLSource struct {
+	Url string `config:"url"`
+	Headers map[string]string `config:"headers"`
+	PollingSource
+}
+
+func (z *ZipURLSource) Fetch() error {
+	panic("implement me")
+}
+
+func (z *ZipURLSource) Workdir() string {
+	panic("implement me")
+}
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go b/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go
index 6814d6aaf21..effeeef91cf 100644
--- a/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go
+++ b/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go
@@ -34,20 +34,13 @@ func (s *SyntheticSuite) String() string {
 }
 
 func (s *SyntheticSuite) Fetch() error {
-	return s.suiteCfg.Source.active().Fetch()
+	return s.suiteCfg.Source.Active().Fetch()
 }
 
 func (s *SyntheticSuite) Workdir() string {
-	return s.suiteCfg.Source.active().Workdir()
+	return s.suiteCfg.Source.Active().Workdir()
 }
 
-func (s *SyntheticSuite) Start() {
-	panic("implement me")
-}
-
-func (s *SyntheticSuite) Stop() {
-	panic("implement me")
-}
 
 func (s *SyntheticSuite) Params() map[string]interface{} {
 	return s.suiteCfg.Params

From 7c05aa6fceb449c7c28cd9eebb354f1914c2b876 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Tue, 12 Jan 2021 15:57:49 -0600
Subject: [PATCH 14/41] Simplification attempt

---
 heartbeat/monitors/wrappers/monitors.go       | 22 ++++++++++++++++++-
 x-pack/heartbeat/monitors/synthexec/enrich.go | 12 +++++++++-
 .../sample-synthetics-config/heartbeat.yml    |  5 ++++-
 3 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/heartbeat/monitors/wrappers/monitors.go b/heartbeat/monitors/wrappers/monitors.go
index 0ad592ae78f..a7753a9b229 100644
--- a/heartbeat/monitors/wrappers/monitors.go
+++ b/heartbeat/monitors/wrappers/monitors.go
@@ -171,6 +171,7 @@ func makeAddSummary(monitorType string) jobs.JobWrapper {
 	// state struct here.
 	state := struct {
 		mtx        sync.Mutex
+		monitorId  string
 		remaining  uint16
 		up         uint16
 		down       uint16
@@ -195,6 +196,10 @@ func makeAddSummary(monitorType string) jobs.JobWrapper {
 
 	return func(job jobs.Job) jobs.Job {
 		return func(event *beat.Event) ([]jobs.Job, error) {
+			if v, _ := event.GetValue("monitor.id"); v != state.monitorId {
+				resetState()
+			}
+
 			cont, jobErr := job(event)
 			state.mtx.Lock()
 			defer state.mtx.Unlock()
@@ -212,8 +217,20 @@ func makeAddSummary(monitorType string) jobs.JobWrapper {
 				}
 			}
 
+			// Jobs can append to the check group to delineate discrete jobs
+			// A nifty property here is that check groups remain a single unique value
+			// but you can still use an efficient prefix search in a pinch to group those
+			// if necessary.
+			checkGroupExt, _ := event.Meta.GetValue("check_group_ext")
+
 			// No error check needed here
-			event.PutValue("monitor.check_group", state.checkGroup)
+			var cg string
+			if checkGroupExt != "" {
+				cg = fmt.Sprintf("%s-%s", state.checkGroup, checkGroupExt)
+			} else {
+				cg = state.checkGroup
+			}
+			event.PutValue("monitor.check_group", cg)
 
 			// Adjust the total remaining to account for new continuations
 			state.remaining += uint16(len(cont))
@@ -224,6 +241,9 @@ func makeAddSummary(monitorType string) jobs.JobWrapper {
 			if state.remaining == 0 {
 				up := state.up
 				down := state.down
+
+				// Browser monitors only count as a single up/down
+				// event
 				if monitorType == "browser" {
 					if eventStatus == "down" {
 						up = 0
diff --git a/x-pack/heartbeat/monitors/synthexec/enrich.go b/x-pack/heartbeat/monitors/synthexec/enrich.go
index 426b5da2bd7..8768c4b8062 100644
--- a/x-pack/heartbeat/monitors/synthexec/enrich.go
+++ b/x-pack/heartbeat/monitors/synthexec/enrich.go
@@ -17,6 +17,8 @@ import (
 // where relevant to properly enrich *beat.Event instances.
 type journeyEnricher struct {
 	journeyComplete bool
+	monitorId string
+	checkGroupExt string
 	errorCount      int
 	lastError       error
 	stepCount       int
@@ -28,7 +30,9 @@ type journeyEnricher struct {
 }
 
 func newJourneyEnricher() *journeyEnricher {
-	return &journeyEnricher{}
+	return &journeyEnricher{
+		checkGroupExt: "init",
+	}
 }
 
 func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent) error {
@@ -37,6 +41,7 @@ func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent) error {
 		// Record start and end so we can calculate journey duration accurately later
 		switch se.Type {
 		case "journey/start":
+			je.checkGroupExt = se.Journey.Name
 			je.start = event.Timestamp
 		case "journey/end":
 			je.end = event.Timestamp
@@ -54,6 +59,11 @@ func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent) error {
 }
 
 func (je *journeyEnricher) enrichSynthEvent(event *beat.Event, se *SynthEvent) error {
+	// For synthetics we manually set the monitor ID since we run
+	// the whole suite as one shot
+	event.PutValue("monitor.id", se.Journey.Name)
+	event.Meta.Put("check_group_ext", je.checkGroupExt)
+
 	switch se.Type {
 	case "journey/end":
 		je.journeyComplete = true
diff --git a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
index 9df063eb539..1dbfd5b781e 100644
--- a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
+++ b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
@@ -42,7 +42,10 @@ setup.template.settings:
   index.number_of_shards: 1
   index.codec: best_compression
 setup.kibana:
-output.console: ~
+output.elasticsearch:
+  host: "127.0.0.1:9200"
+  username: elastic
+  password: changeme
 processors:
 - add_observer_metadata:
 

From 367548ca07ed8632df4152c5f1654e9c19188955 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Tue, 12 Jan 2021 17:47:44 -0600
Subject: [PATCH 15/41] checkpoint

---
 go.sum                                        |  2 +
 heartbeat/monitors/monitor.go                 |  6 --
 heartbeat/monitors/wrappers/monitors.go       | 60 +++++-------
 x-pack/heartbeat/main.go                      |  3 +-
 .../{synthetic_suite => synthetic}/config.go  |  4 +-
 .../source/github.go                          |  0
 .../source/local.go                           |  0
 .../source/source.go                          |  0
 .../source/zipurl.go                          |  0
 .../suite_runner.go                           |  3 +-
 .../synthetic_suite.go                        |  7 +-
 .../monitors/synthetic_script/config.go       | 25 -----
 .../synthetic_script/suitefactory/config.go   | 37 -------
 .../suitefactory/suitefactory.go              | 97 -------------------
 .../synthetic_script/synthetic_script.go      | 61 ------------
 .../monitors/synthetic_suite/sources.go       |  1 -
 x-pack/heartbeat/monitors/synthexec/enrich.go | 65 +++++++++++--
 .../heartbeat/monitors/synthexec/synthexec.go |  9 +-
 .../sample-synthetics-config/heartbeat.yml    | 41 ++++----
 19 files changed, 116 insertions(+), 305 deletions(-)
 rename x-pack/heartbeat/monitors/{synthetic_suite => synthetic}/config.go (88%)
 rename x-pack/heartbeat/monitors/{synthetic_suite => synthetic}/source/github.go (100%)
 rename x-pack/heartbeat/monitors/{synthetic_suite => synthetic}/source/local.go (100%)
 rename x-pack/heartbeat/monitors/{synthetic_suite => synthetic}/source/source.go (100%)
 rename x-pack/heartbeat/monitors/{synthetic_suite => synthetic}/source/zipurl.go (100%)
 rename x-pack/heartbeat/monitors/{synthetic_suite => synthetic}/suite_runner.go (97%)
 rename x-pack/heartbeat/monitors/{synthetic_suite => synthetic}/synthetic_suite.go (92%)
 delete mode 100644 x-pack/heartbeat/monitors/synthetic_script/config.go
 delete mode 100644 x-pack/heartbeat/monitors/synthetic_script/suitefactory/config.go
 delete mode 100644 x-pack/heartbeat/monitors/synthetic_script/suitefactory/suitefactory.go
 delete mode 100644 x-pack/heartbeat/monitors/synthetic_script/synthetic_script.go
 delete mode 100644 x-pack/heartbeat/monitors/synthetic_suite/sources.go

diff --git a/go.sum b/go.sum
index a9ec46b68c4..baf3232fae0 100644
--- a/go.sum
+++ b/go.sum
@@ -244,6 +244,8 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA=
 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
+github.com/elastic/beats v1.3.1 h1:hHzUBHCo3HJHxnRVwa0XlfZoxmP8Rxp7GQ0ZVELGY4A=
+github.com/elastic/beats v7.6.2+incompatible h1:jHdLv83KURaqWUC6f55iMyVP6LYZrgElfeqxKWcskVE=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
 github.com/elastic/ecs v1.6.0 h1:8NmgfnsjmKXh9hVsK3H2tZtfUptepNc3msJOAynhtmc=
diff --git a/heartbeat/monitors/monitor.go b/heartbeat/monitors/monitor.go
index 6ffea01126e..e106c9ce272 100644
--- a/heartbeat/monitors/monitor.go
+++ b/heartbeat/monitors/monitor.go
@@ -135,12 +135,6 @@ func newMonitorUnsafe(
 		return nil, fmt.Errorf("monitor type %v does not exist, valid types are %v", standardFields.Type, registrar.monitorNames())
 	}
 
-	if standardFields.Type == "synthetic_suite" {
-
-	} else {
-
-	}
-
 	m := &Monitor{
 		stdFields:         standardFields,
 		pluginName:        monitorPlugin.name,
diff --git a/heartbeat/monitors/wrappers/monitors.go b/heartbeat/monitors/wrappers/monitors.go
index a7753a9b229..e86f4adf3ed 100644
--- a/heartbeat/monitors/wrappers/monitors.go
+++ b/heartbeat/monitors/wrappers/monitors.go
@@ -19,6 +19,7 @@ package wrappers
 
 import (
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -43,18 +44,27 @@ func WrapCommon(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.J
 		addMonitorStatus(stdMonFields.Type),
 	}
 
-	if stdMonFields.Type != "browser" {
+	if stdMonFields.Type == "synthetic" {
 		jobWrappers = append(jobWrappers, addMonitorDuration)
 	}
 
-	return jobs.WrapAllSeparately(
-		jobs.WrapAll(
-			js,
-			jobWrappers...,
-		),
-		func() jobs.JobWrapper {
-			return makeAddSummary(stdMonFields.Type)
-		})
+	baseWrapped := jobs.WrapAll(
+		js,
+		jobWrappers...,
+	)
+	if stdMonFields.Type == "synthetic" {
+		return jobs.WrapAllSeparately(
+			baseWrapped,
+			func() jobs.JobWrapper {
+				return makeAddSummary(stdMonFields.Type)
+			})
+	}
+	return baseWrapped
+}
+
+func isSyntheticType(typ string) bool {
+	logp.Warn("CHECK TYP %s", typ)
+	return strings.HasPrefix(typ, "synthetic_")
 }
 
 // addMonitorMeta adds the id, name, and type fields to the monitor.
@@ -64,11 +74,15 @@ func addMonitorMeta(stdMonFields stdfields.StdMonitorFields, isMulti bool) jobs.
 			started := time.Now()
 			cont, e := job(event)
 			thisID := stdMonFields.ID
+			thisName := stdMonFields.Name
 			// Allow jobs to override the ID, useful for suites
 			// which do this logic on their own
 			if v, _ := event.GetValue("monitor.id"); v != nil {
 				thisID = v.(string)
 			}
+			if v, _ := event.GetValue("monitor.name"); v != nil {
+				thisName = v.(string)
+			}
 			if isMulti {
 				url, err := event.GetValue("url.full")
 				if err != nil {
@@ -82,7 +96,7 @@ func addMonitorMeta(stdMonFields stdfields.StdMonitorFields, isMulti bool) jobs.
 			fieldsToMerge := common.MapStr{
 				"monitor": common.MapStr{
 					"id":       thisID,
-					"name":     stdMonFields.Name,
+					"name":     thisName,
 					"type":     stdMonFields.Type,
 					"timespan": timespan(started, stdMonFields.Schedule, stdMonFields.Timeout),
 				},
@@ -217,20 +231,7 @@ func makeAddSummary(monitorType string) jobs.JobWrapper {
 				}
 			}
 
-			// Jobs can append to the check group to delineate discrete jobs
-			// A nifty property here is that check groups remain a single unique value
-			// but you can still use an efficient prefix search in a pinch to group those
-			// if necessary.
-			checkGroupExt, _ := event.Meta.GetValue("check_group_ext")
-
-			// No error check needed here
-			var cg string
-			if checkGroupExt != "" {
-				cg = fmt.Sprintf("%s-%s", state.checkGroup, checkGroupExt)
-			} else {
-				cg = state.checkGroup
-			}
-			event.PutValue("monitor.check_group", cg)
+			event.PutValue("monitor.check_group", state.checkGroup)
 
 			// Adjust the total remaining to account for new continuations
 			state.remaining += uint16(len(cont))
@@ -242,17 +243,6 @@ func makeAddSummary(monitorType string) jobs.JobWrapper {
 				up := state.up
 				down := state.down
 
-				// Browser monitors only count as a single up/down
-				// event
-				if monitorType == "browser" {
-					if eventStatus == "down" {
-						up = 0
-						down = 1
-					} else {
-						up = 1
-						down = 0
-					}
-				}
 				eventext.MergeEventFields(event, common.MapStr{
 					"summary": common.MapStr{
 						"up":   up,
diff --git a/x-pack/heartbeat/main.go b/x-pack/heartbeat/main.go
index 18fcad761e5..70e32f5cf77 100644
--- a/x-pack/heartbeat/main.go
+++ b/x-pack/heartbeat/main.go
@@ -9,8 +9,7 @@ import (
 
 	_ "github.com/elastic/beats/v7/heartbeat/include"
 	"github.com/elastic/beats/v7/x-pack/heartbeat/cmd"
-	_ "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthetic_script"
-	_ "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthetic_suite"
+	_ "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthetic"
 )
 
 func main() {
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/config.go b/x-pack/heartbeat/monitors/synthetic/config.go
similarity index 88%
rename from x-pack/heartbeat/monitors/synthetic_suite/config.go
rename to x-pack/heartbeat/monitors/synthetic/config.go
index 88807cb8903..aa822fd9a82 100644
--- a/x-pack/heartbeat/monitors/synthetic_suite/config.go
+++ b/x-pack/heartbeat/monitors/synthetic/config.go
@@ -1,8 +1,8 @@
-package synthetic_suite
+package synthetic
 
 import (
 	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthetic_suite/source"
+	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthetic/source"
 )
 
 type Config struct {
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/source/github.go b/x-pack/heartbeat/monitors/synthetic/source/github.go
similarity index 100%
rename from x-pack/heartbeat/monitors/synthetic_suite/source/github.go
rename to x-pack/heartbeat/monitors/synthetic/source/github.go
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/source/local.go b/x-pack/heartbeat/monitors/synthetic/source/local.go
similarity index 100%
rename from x-pack/heartbeat/monitors/synthetic_suite/source/local.go
rename to x-pack/heartbeat/monitors/synthetic/source/local.go
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/source/source.go b/x-pack/heartbeat/monitors/synthetic/source/source.go
similarity index 100%
rename from x-pack/heartbeat/monitors/synthetic_suite/source/source.go
rename to x-pack/heartbeat/monitors/synthetic/source/source.go
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/source/zipurl.go b/x-pack/heartbeat/monitors/synthetic/source/zipurl.go
similarity index 100%
rename from x-pack/heartbeat/monitors/synthetic_suite/source/zipurl.go
rename to x-pack/heartbeat/monitors/synthetic/source/zipurl.go
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go b/x-pack/heartbeat/monitors/synthetic/suite_runner.go
similarity index 97%
rename from x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go
rename to x-pack/heartbeat/monitors/synthetic/suite_runner.go
index effeeef91cf..6f936b5b479 100644
--- a/x-pack/heartbeat/monitors/synthetic_suite/suite_runner.go
+++ b/x-pack/heartbeat/monitors/synthetic/suite_runner.go
@@ -1,4 +1,4 @@
-package synthetic_suite
+package synthetic
 
 import (
 	"context"
@@ -41,7 +41,6 @@ func (s *SyntheticSuite) Workdir() string {
 	return s.suiteCfg.Source.Active().Workdir()
 }
 
-
 func (s *SyntheticSuite) Params() map[string]interface{} {
 	return s.suiteCfg.Params
 }
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/synthetic_suite.go b/x-pack/heartbeat/monitors/synthetic/synthetic_suite.go
similarity index 92%
rename from x-pack/heartbeat/monitors/synthetic_suite/synthetic_suite.go
rename to x-pack/heartbeat/monitors/synthetic/synthetic_suite.go
index 42fd70fb13a..f73381e3e63 100644
--- a/x-pack/heartbeat/monitors/synthetic_suite/synthetic_suite.go
+++ b/x-pack/heartbeat/monitors/synthetic/synthetic_suite.go
@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package synthetic_suite
+package synthetic
 
 import (
 	"context"
@@ -19,8 +19,9 @@ import (
 )
 
 func init() {
-	monitors.RegisterActive("synthetic_suite", create)
-	monitors.RegisterActive("synthetic/suite", create)
+	monitors.RegisterActive("browser", create)
+	monitors.RegisterActive("synthetic", create)
+	monitors.RegisterActive("synthetics/synthetic", create)
 }
 
 var showExperimentalOnce = sync.Once{}
diff --git a/x-pack/heartbeat/monitors/synthetic_script/config.go b/x-pack/heartbeat/monitors/synthetic_script/config.go
deleted file mode 100644
index 205049e3709..00000000000
--- a/x-pack/heartbeat/monitors/synthetic_script/config.go
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-package synthetic_script
-
-import (
-	"fmt"
-
-	"github.com/elastic/beats/v7/libbeat/common"
-)
-
-type Config struct {
-	Script      string        `config:"script"`
-	Params      common.MapStr `config:"script_params"`
-}
-
-func (c *Config) Validate() error {
-	if c.Script != "" {
-		return fmt.Errorf("no script specified for journey!")
-	}
-	return nil
-}
-
-var defaultConfig = Config{}
diff --git a/x-pack/heartbeat/monitors/synthetic_script/suitefactory/config.go b/x-pack/heartbeat/monitors/synthetic_script/suitefactory/config.go
deleted file mode 100644
index d5e9f508d41..00000000000
--- a/x-pack/heartbeat/monitors/synthetic_script/suitefactory/config.go
+++ /dev/null
@@ -1,37 +0,0 @@
-package suitefactory
-
-import "github.com/elastic/beats/v7/libbeat/common"
-
-type SyntheticSuite struct {
-	Type	 string					`config:"type"`
-	Name     string                 `config:"id_prefix"`
-	Schedule string                 `config:"schedule"`
-	Params   map[string]interface{} `config:"params"`
-	RawConfig *common.Config
-}
-
-type LocalSyntheticSuite struct {
-	Path     string                 `config:"path"`
-	SyntheticSuite
-}
-
-type PollingSyntheticSuite struct {
-	CheckEvery int `config:"check_every"`
-	SyntheticSuite
-}
-
-// GithubSyntheticSuite handles configs for github repos, using the API defined here:
-// https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#download-a-repository-archive-tar.
-type GithubSyntheticSuite struct {
-	Owner	string `config:"owner"`
-	Repo string `config:"repo"`
-	Ref string `config:"ref"`
-	UrlBase string `config:"string"`
-	PollingSyntheticSuite
-}
-
-type ZipUrlSyntheticSuite struct {
-	Url string `config:"url""`
-	Headers map[string]string `config:"headers"`
-	PollingSyntheticSuite
-}
diff --git a/x-pack/heartbeat/monitors/synthetic_script/suitefactory/suitefactory.go b/x-pack/heartbeat/monitors/synthetic_script/suitefactory/suitefactory.go
deleted file mode 100644
index e792f6ac12b..00000000000
--- a/x-pack/heartbeat/monitors/synthetic_script/suitefactory/suitefactory.go
+++ /dev/null
@@ -1,97 +0,0 @@
-package suitefactory
-
-import (
-	"context"
-	"fmt"
-	"github.com/elastic/beats/v7/heartbeat/beater"
-	"github.com/elastic/beats/v7/heartbeat/monitors"
-	"github.com/elastic/beats/v7/libbeat/beat"
-	"github.com/elastic/beats/v7/libbeat/cfgfile"
-	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/elastic/beats/v7/libbeat/logp"
-	b "github.com/elastic/beats/v7/metricbeat/module/beat"
-	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthexec"
-	"github.com/pkg/errors"
-)
-
-func init() {
-	sf := NewStdSuiteFactory()
-	beater.RegisterSuiteFactory(sf)
-}
-
-func NewStdSuiteFactory() *StdSuiteFactory {
-	return &StdSuiteFactory{}
-}
-
-type StdSuiteFactory struct {
-	//
-}
-
-func (s *StdSuiteFactory) Create(p beat.PipelineConnector, config *common.Config) (cfgfile.Runner, error) {
-	suite := &SyntheticSuite{}
-	err := config.Unpack(suite)
-	if err != nil {
-		return nil, fmt.Errorf("could not parse suite config: %w", err)
-	}
-
-	var suitePath string
-	var suiteParams map[string]interface{}
-	switch suite.Type {
-	case "zipurl":
-		unpacked := &ZipUrlSyntheticSuite{}
-		err := config.Unpack(unpacked)
-		if err != nil {
-			return nil, fmt.Errorf("could not parse zip URL synthetic suite: %w", err)
-		}
-	case "github":
-		unpacked := &GithubSyntheticSuite{}
-		err := config.Unpack(unpacked)
-		if err != nil {
-			return nil, fmt.Errorf("could not parse github synthetic suite: %w", err)
-		}
-	case "local":
-		unpacked := &LocalSyntheticSuite{}
-		err := config.Unpack(unpacked)
-		if err != nil {
-			return nil, fmt.Errorf("could not parse local synthetic suite: %w", err)
-		}
-		suitePath = unpacked.Path
-		suiteParams = unpacked.Params
-	default:
-		return nil, fmt.Errorf("suite type not specified! Expected 'local', 'github', or 'zipurl'")
-	}
-
-	logp.Info("Listing suite %s", suitePath)
-	journeyNames, err := synthexec.ListJourneys(context.TODO(), suitePath, suiteParams)
-	if err != nil {
-		return nil, err
-    }
-	logp.Warn("POSTLIST")
-	factory := monitors.NewFactory(b.Info, bt.scheduler, false)
-	for _, name := range journeyNames {
-		cfg, err := common.NewConfigFrom(map[string]interface{}{
-			"type":         "browser",
-			"path":         suiteReloader.WorkingPath(),
-			"schedule":     suite.Schedule,
-			"params":       suite.Params,
-			"journey_name": name,
-			"name":         name,
-			"id":           name,
-		})
-		if err != nil {
-				  return err
-				  }
-		created, err := factory.Create(b.Publisher, cfg)
-		if err != nil {
-				  return errors.Wrap(err, "could not create monitor")
-				  }
-		created.Start()
-	}
-}
-
-func (s *StdSuiteFactory) CheckConfig(config *common.Config) error {
-	return nil
-}
-
-
-
diff --git a/x-pack/heartbeat/monitors/synthetic_script/synthetic_script.go b/x-pack/heartbeat/monitors/synthetic_script/synthetic_script.go
deleted file mode 100644
index e041fcf6bdc..00000000000
--- a/x-pack/heartbeat/monitors/synthetic_script/synthetic_script.go
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-package synthetic_script
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/user"
-	"sync"
-
-	"github.com/elastic/beats/v7/heartbeat/monitors"
-	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
-	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/elastic/beats/v7/libbeat/logp"
-	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthexec"
-)
-
-func init() {
-	// TODO: Remove this deprecated option, it was part of experimental,
-	// so no need for deprecation phase
-	monitors.RegisterActive("browser", create)
-	monitors.RegisterActive("synthetic_script", create)
-	monitors.RegisterActive("synthetic/script", create)
-}
-
-var showExperimentalOnce = sync.Once{}
-
-var NotSyntheticsCapableError = fmt.Errorf("synthetic monitors cannot be created outside the official elastic docker image")
-
-func create(name string, cfg *common.Config) (js []jobs.Job, endpoints int, err error) {
-	// We don't want users running synthetics in environments that don't have the required GUI libraries etc, so we check
-	// this flag. When we're ready to support the many possible configurations of systems outside the docker environment
-	// we can remove this check.
-	if os.Getenv("ELASTIC_SYNTHETICS_CAPABLE") != "true" {
-		return nil, 0, NotSyntheticsCapableError
-	}
-
-	showExperimentalOnce.Do(func() {
-		logp.Info("Synthetic monitor detected! Please note synthetic monitors are an experimental unsupported feature!")
-	})
-
-	curUser, err := user.Current()
-	if err != nil {
-		return nil, 0, fmt.Errorf("could not determine current user for script monitor %w: ", err)
-	}
-	if curUser.Uid == "0" {
-		return nil, 0, fmt.Errorf("script monitors cannot be run as root! Current UID is %s", curUser.Uid)
-	}
-
-	config := defaultConfig
-	if err := cfg.Unpack(&config); err != nil {
-		return nil, 0, err
-	}
-
-
-	j := synthexec.InlineJourneyJob(context.TODO(), config.Script, config.Params)
-	return []jobs.Job{j}, 1, nil
-}
diff --git a/x-pack/heartbeat/monitors/synthetic_suite/sources.go b/x-pack/heartbeat/monitors/synthetic_suite/sources.go
deleted file mode 100644
index 005d1f68f34..00000000000
--- a/x-pack/heartbeat/monitors/synthetic_suite/sources.go
+++ /dev/null
@@ -1 +0,0 @@
-package synthetic_suite
diff --git a/x-pack/heartbeat/monitors/synthexec/enrich.go b/x-pack/heartbeat/monitors/synthexec/enrich.go
index 8768c4b8062..a6b70b22456 100644
--- a/x-pack/heartbeat/monitors/synthexec/enrich.go
+++ b/x-pack/heartbeat/monitors/synthexec/enrich.go
@@ -6,6 +6,8 @@ package synthexec
 
 import (
 	"fmt"
+	"github.com/elastic/beats/v7/libbeat/logp"
+	"github.com/gofrs/uuid"
 	"time"
 
 	"github.com/elastic/beats/v7/heartbeat/eventext"
@@ -13,12 +15,26 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 )
 
+type enricher func (event *beat.Event, se *SynthEvent) error
+
+type streamEnricher struct {
+	je *journeyEnricher
+}
+
+func (e *streamEnricher) enrich(event *beat.Event, se *SynthEvent) error {
+	if e.je == nil || (se != nil && se.Type == "journey/start") {
+		e.je = newJourneyEnricher()
+	}
+
+	return e.je.enrich(event, se)
+}
+
 // journeyEnricher holds state across received SynthEvents retaining fields
 // where relevant to properly enrich *beat.Event instances.
 type journeyEnricher struct {
 	journeyComplete bool
-	monitorId string
-	checkGroupExt string
+	journey *Journey
+	checkGroup      string
 	errorCount      int
 	lastError       error
 	stepCount       int
@@ -31,8 +47,16 @@ type journeyEnricher struct {
 
 func newJourneyEnricher() *journeyEnricher {
 	return &journeyEnricher{
-		checkGroupExt: "init",
+		checkGroup: makeUuid(),
+	}
+}
+
+func makeUuid() string {
+	u, err := uuid.NewV1()
+	if err != nil {
+		panic("Cannot generate v1 UUID, this should never happen!")
 	}
+	return u.String()
 }
 
 func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent) error {
@@ -41,7 +65,9 @@ func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent) error {
 		// Record start and end so we can calculate journey duration accurately later
 		switch se.Type {
 		case "journey/start":
-			je.checkGroupExt = se.Journey.Name
+			je.lastError = nil
+			je.checkGroup = makeUuid()
+			je.journey = se.Journey
 			je.start = event.Timestamp
 		case "journey/end":
 			je.end = event.Timestamp
@@ -52,21 +78,25 @@ func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent) error {
 
 	// No more synthEvents? In this case this is the summary event
 	if se == nil {
-		return je.createSummary(event)
+		return nil
 	}
 
+	eventext.MergeEventFields(event, common.MapStr{
+		"monitor": common.MapStr{
+			"id": je.journey.Id,
+			"name": je.journey.Name,
+			"check_group": je.checkGroup,
+		},
+	})
+
 	return je.enrichSynthEvent(event, se)
 }
 
 func (je *journeyEnricher) enrichSynthEvent(event *beat.Event, se *SynthEvent) error {
-	// For synthetics we manually set the monitor ID since we run
-	// the whole suite as one shot
-	event.PutValue("monitor.id", se.Journey.Name)
-	event.Meta.Put("check_group_ext", je.checkGroupExt)
-
 	switch se.Type {
 	case "journey/end":
 		je.journeyComplete = true
+		return je.createSummary(event)
 	case "step/end":
 		je.stepCount++
 	}
@@ -92,6 +122,15 @@ func (je *journeyEnricher) enrichSynthEvent(event *beat.Event, se *SynthEvent) e
 }
 
 func (je *journeyEnricher) createSummary(event *beat.Event) error {
+	var up, down int
+	if je.errorCount > 0 {
+		up = 0
+		down = 1
+	} else {
+		up = 1
+		down = 0
+	}
+
 	if je.journeyComplete {
 		eventext.MergeEventFields(event, common.MapStr{
 			"url": je.urlFields,
@@ -103,9 +142,15 @@ func (je *journeyEnricher) createSummary(event *beat.Event) error {
 					"us": int64(je.end.Sub(je.start) / time.Microsecond),
 				},
 			},
+			"summary": common.MapStr{
+				"up": up,
+				"down": down,
+			},
 		})
+		logp.Warn("TRIGGER SUMMARY %#v", event.Fields)
 		return je.lastError
 	}
+
 	return fmt.Errorf("journey did not finish executing, %d steps ran", je.stepCount)
 }
 
diff --git a/x-pack/heartbeat/monitors/synthexec/synthexec.go b/x-pack/heartbeat/monitors/synthexec/synthexec.go
index c2532c78c5b..7ebec92a4bb 100644
--- a/x-pack/heartbeat/monitors/synthexec/synthexec.go
+++ b/x-pack/heartbeat/monitors/synthexec/synthexec.go
@@ -132,19 +132,20 @@ func startCmdJob(ctx context.Context, newCmd func() *exec.Cmd, stdinStr *string,
 		if err != nil {
 			return nil, err
 		}
-		return []jobs.Job{readResultsJob(ctx, mpx.SynthEvents(), newJourneyEnricher())}, nil
+		senr := streamEnricher{}
+		return []jobs.Job{readResultsJob(ctx, mpx.SynthEvents(), senr.enrich)}, nil
 	}
 }
 
 // readResultsJob adapts the output of an ExecMultiplexer into a Job, that uses continuations
 // to read all output.
-func readResultsJob(ctx context.Context, synthEvents <-chan *SynthEvent, je *journeyEnricher) jobs.Job {
+func readResultsJob(ctx context.Context, synthEvents <-chan *SynthEvent, enrich enricher) jobs.Job {
 	return func(event *beat.Event) (conts []jobs.Job, err error) {
 		select {
 		case se := <-synthEvents:
-			err = je.enrich(event, se)
+			err = enrich(event, se)
 			if se != nil {
-				return []jobs.Job{readResultsJob(ctx, synthEvents, je)}, err
+				return []jobs.Job{readResultsJob(ctx, synthEvents, enrich)}, err
 			} else {
 				return nil, err
 			}
diff --git a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
index 1dbfd5b781e..6eb715ffbd5 100644
--- a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
+++ b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
@@ -5,7 +5,7 @@ heartbeat.config.monitors:
   reload.period: 5s
 
 heartbeat.monitors:
-- type: synthetic_suite
+- type: browser
   enabled: true
   id: todos-suite
   name: Todos Suite
@@ -19,31 +19,32 @@ heartbeat.monitors:
   urls: http://www.google.com
   schedule: "@every 15s"
   name: Simple HTTP
-- type: synthetic_script
-  enabled: false
-  id: my-monitor
-  name: My Monitor
-  script: |-
-    step("load homepage", async () => {
-        await page.goto('https://www.elastic.co');
-    });
-    step("hover over products menu", async () => {
-        await page.hover('css=[data-nav-item=products]');
-    });
-    step("failme", async () => {
-        await page.hhover('css=[data-nav-item=products]');
-    });
-    step("skip me", async () => {
-        // noop
-    });
-  schedule: "@every 1m"
+#- type: synthetic
+#  enabled: false
+#  id: my-monitor
+#  name: My Monitor
+#  source:
+#    script:
+#      step("load homepage", async () => {
+#          await page.goto('https://www.elastic.co');
+#      });
+#      step("hover over products menu", async () => {
+#          await page.hover('css=[data-nav-item=products]');
+#      });
+#      step("failme", async () => {
+#          await page.hhover('css=[data-nav-item=products]');
+#      });
+#      step("skip me", async () => {
+#          // noop
+#      });
+#  schedule: "@every 1m"
 
 setup.template.settings:
   index.number_of_shards: 1
   index.codec: best_compression
 setup.kibana:
 output.elasticsearch:
-  host: "127.0.0.1:9200"
+  hosts: "127.0.0.1:9200"
   username: elastic
   password: changeme
 processors:

From 70f5281082a17ab05237e9908cb484f76b0fff83 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Wed, 13 Jan 2021 16:34:26 -0600
Subject: [PATCH 16/41] Things work again

---
 heartbeat/monitors/wrappers/monitors.go                | 10 ++--------
 x-pack/heartbeat/main.go                               |  2 +-
 .../synthetic_suite.go => browser/browser.go}          |  2 +-
 .../monitors/{synthetic => browser}/config.go          |  4 ++--
 .../monitors/{synthetic => browser}/source/github.go   |  0
 .../monitors/{synthetic => browser}/source/local.go    |  0
 .../monitors/{synthetic => browser}/source/source.go   |  0
 .../monitors/{synthetic => browser}/source/zipurl.go   |  0
 .../monitors/{synthetic => browser}/suite_runner.go    |  2 +-
 x-pack/heartbeat/monitors/synthexec/enrich.go          |  2 --
 10 files changed, 7 insertions(+), 15 deletions(-)
 rename x-pack/heartbeat/monitors/{synthetic/synthetic_suite.go => browser/browser.go} (99%)
 rename x-pack/heartbeat/monitors/{synthetic => browser}/config.go (73%)
 rename x-pack/heartbeat/monitors/{synthetic => browser}/source/github.go (100%)
 rename x-pack/heartbeat/monitors/{synthetic => browser}/source/local.go (100%)
 rename x-pack/heartbeat/monitors/{synthetic => browser}/source/source.go (100%)
 rename x-pack/heartbeat/monitors/{synthetic => browser}/source/zipurl.go (100%)
 rename x-pack/heartbeat/monitors/{synthetic => browser}/suite_runner.go (98%)

diff --git a/heartbeat/monitors/wrappers/monitors.go b/heartbeat/monitors/wrappers/monitors.go
index e86f4adf3ed..2a8a11ae4b1 100644
--- a/heartbeat/monitors/wrappers/monitors.go
+++ b/heartbeat/monitors/wrappers/monitors.go
@@ -19,7 +19,6 @@ package wrappers
 
 import (
 	"fmt"
-	"strings"
 	"sync"
 	"time"
 
@@ -44,7 +43,7 @@ func WrapCommon(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.J
 		addMonitorStatus(stdMonFields.Type),
 	}
 
-	if stdMonFields.Type == "synthetic" {
+	if stdMonFields.Type == "browser" {
 		jobWrappers = append(jobWrappers, addMonitorDuration)
 	}
 
@@ -52,7 +51,7 @@ func WrapCommon(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.J
 		js,
 		jobWrappers...,
 	)
-	if stdMonFields.Type == "synthetic" {
+	if stdMonFields.Type != "browser" {
 		return jobs.WrapAllSeparately(
 			baseWrapped,
 			func() jobs.JobWrapper {
@@ -62,11 +61,6 @@ func WrapCommon(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.J
 	return baseWrapped
 }
 
-func isSyntheticType(typ string) bool {
-	logp.Warn("CHECK TYP %s", typ)
-	return strings.HasPrefix(typ, "synthetic_")
-}
-
 // addMonitorMeta adds the id, name, and type fields to the monitor.
 func addMonitorMeta(stdMonFields stdfields.StdMonitorFields, isMulti bool) jobs.JobWrapper {
 	return func(job jobs.Job) jobs.Job {
diff --git a/x-pack/heartbeat/main.go b/x-pack/heartbeat/main.go
index 70e32f5cf77..4c0f220edf6 100644
--- a/x-pack/heartbeat/main.go
+++ b/x-pack/heartbeat/main.go
@@ -9,7 +9,7 @@ import (
 
 	_ "github.com/elastic/beats/v7/heartbeat/include"
 	"github.com/elastic/beats/v7/x-pack/heartbeat/cmd"
-	_ "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthetic"
+	_ "github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser"
 )
 
 func main() {
diff --git a/x-pack/heartbeat/monitors/synthetic/synthetic_suite.go b/x-pack/heartbeat/monitors/browser/browser.go
similarity index 99%
rename from x-pack/heartbeat/monitors/synthetic/synthetic_suite.go
rename to x-pack/heartbeat/monitors/browser/browser.go
index f73381e3e63..656f68b365b 100644
--- a/x-pack/heartbeat/monitors/synthetic/synthetic_suite.go
+++ b/x-pack/heartbeat/monitors/browser/browser.go
@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package synthetic
+package browser
 
 import (
 	"context"
diff --git a/x-pack/heartbeat/monitors/synthetic/config.go b/x-pack/heartbeat/monitors/browser/config.go
similarity index 73%
rename from x-pack/heartbeat/monitors/synthetic/config.go
rename to x-pack/heartbeat/monitors/browser/config.go
index aa822fd9a82..9bcbf69bc28 100644
--- a/x-pack/heartbeat/monitors/synthetic/config.go
+++ b/x-pack/heartbeat/monitors/browser/config.go
@@ -1,8 +1,8 @@
-package synthetic
+package browser
 
 import (
 	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthetic/source"
+	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/source"
 )
 
 type Config struct {
diff --git a/x-pack/heartbeat/monitors/synthetic/source/github.go b/x-pack/heartbeat/monitors/browser/source/github.go
similarity index 100%
rename from x-pack/heartbeat/monitors/synthetic/source/github.go
rename to x-pack/heartbeat/monitors/browser/source/github.go
diff --git a/x-pack/heartbeat/monitors/synthetic/source/local.go b/x-pack/heartbeat/monitors/browser/source/local.go
similarity index 100%
rename from x-pack/heartbeat/monitors/synthetic/source/local.go
rename to x-pack/heartbeat/monitors/browser/source/local.go
diff --git a/x-pack/heartbeat/monitors/synthetic/source/source.go b/x-pack/heartbeat/monitors/browser/source/source.go
similarity index 100%
rename from x-pack/heartbeat/monitors/synthetic/source/source.go
rename to x-pack/heartbeat/monitors/browser/source/source.go
diff --git a/x-pack/heartbeat/monitors/synthetic/source/zipurl.go b/x-pack/heartbeat/monitors/browser/source/zipurl.go
similarity index 100%
rename from x-pack/heartbeat/monitors/synthetic/source/zipurl.go
rename to x-pack/heartbeat/monitors/browser/source/zipurl.go
diff --git a/x-pack/heartbeat/monitors/synthetic/suite_runner.go b/x-pack/heartbeat/monitors/browser/suite_runner.go
similarity index 98%
rename from x-pack/heartbeat/monitors/synthetic/suite_runner.go
rename to x-pack/heartbeat/monitors/browser/suite_runner.go
index 6f936b5b479..1e15457500f 100644
--- a/x-pack/heartbeat/monitors/synthetic/suite_runner.go
+++ b/x-pack/heartbeat/monitors/browser/suite_runner.go
@@ -1,4 +1,4 @@
-package synthetic
+package browser
 
 import (
 	"context"
diff --git a/x-pack/heartbeat/monitors/synthexec/enrich.go b/x-pack/heartbeat/monitors/synthexec/enrich.go
index a6b70b22456..cc001ef8bdf 100644
--- a/x-pack/heartbeat/monitors/synthexec/enrich.go
+++ b/x-pack/heartbeat/monitors/synthexec/enrich.go
@@ -6,7 +6,6 @@ package synthexec
 
 import (
 	"fmt"
-	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/gofrs/uuid"
 	"time"
 
@@ -147,7 +146,6 @@ func (je *journeyEnricher) createSummary(event *beat.Event) error {
 				"down": down,
 			},
 		})
-		logp.Warn("TRIGGER SUMMARY %#v", event.Fields)
 		return je.lastError
 	}
 

From 3c66ed11f39499f333bda91ed6cb5e6cca075cc4 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Wed, 13 Jan 2021 18:09:34 -0600
Subject: [PATCH 17/41] Switch from multiple plugin registration to aliases

---
 heartbeat/monitors/active/http/http.go       |  3 +--
 heartbeat/monitors/active/icmp/icmp.go       |  3 +--
 heartbeat/monitors/active/tcp/tcp.go         |  3 +--
 heartbeat/monitors/mocks_test.go             |  6 +++++-
 heartbeat/monitors/plugin.go                 | 18 ++++++++++++------
 x-pack/heartbeat/monitors/browser/browser.go |  4 +---
 6 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/heartbeat/monitors/active/http/http.go b/heartbeat/monitors/active/http/http.go
index 74463663567..f14e9c90842 100644
--- a/heartbeat/monitors/active/http/http.go
+++ b/heartbeat/monitors/active/http/http.go
@@ -32,8 +32,7 @@ import (
 )
 
 func init() {
-	monitors.RegisterActive("http", create)
-	monitors.RegisterActive("synthetics/http", create)
+	monitors.RegisterActive("http", create, "synthetics/http")
 }
 
 var debugf = logp.MakeDebug("http")
diff --git a/heartbeat/monitors/active/icmp/icmp.go b/heartbeat/monitors/active/icmp/icmp.go
index f9119ab19ec..7feb76e6274 100644
--- a/heartbeat/monitors/active/icmp/icmp.go
+++ b/heartbeat/monitors/active/icmp/icmp.go
@@ -35,8 +35,7 @@ import (
 var debugf = logp.MakeDebug("icmp")
 
 func init() {
-	monitors.RegisterActive("icmp", create)
-	monitors.RegisterActive("synthetics/icmp", create)
+	monitors.RegisterActive("icmp", create, "synthetics/icmp")
 }
 
 func create(
diff --git a/heartbeat/monitors/active/tcp/tcp.go b/heartbeat/monitors/active/tcp/tcp.go
index 6be682ee560..f9d3499b681 100644
--- a/heartbeat/monitors/active/tcp/tcp.go
+++ b/heartbeat/monitors/active/tcp/tcp.go
@@ -39,8 +39,7 @@ import (
 )
 
 func init() {
-	monitors.RegisterActive("tcp", create)
-	monitors.RegisterActive("synthetics/tcp", create)
+	monitors.RegisterActive("tcp", create, "synthetics/tcp")
 }
 
 var debugf = logp.MakeDebug("tcp")
diff --git a/heartbeat/monitors/mocks_test.go b/heartbeat/monitors/mocks_test.go
index 629eeb43a43..3a0fc9f3241 100644
--- a/heartbeat/monitors/mocks_test.go
+++ b/heartbeat/monitors/mocks_test.go
@@ -137,7 +137,11 @@ func createMockJob(name string, cfg *common.Config) ([]jobs.Job, error) {
 func mockPluginBuilder() pluginBuilder {
 	reg := monitoring.NewRegistry()
 
-	return pluginBuilder{"test", ActiveMonitor, func(s string, config *common.Config) ([]jobs.Job, int, error) {
+	return pluginBuilder{
+		"test",
+		"testAlias",
+		ActiveMonitor,
+		func(s string, config *common.Config) ([]jobs.Job, int, error) {
 		// Declare a real config block with a required attr so we can see what happens when it doesn't work
 		unpacked := struct {
 			URLs []string `config:"urls" validate:"required"`
diff --git a/heartbeat/monitors/plugin.go b/heartbeat/monitors/plugin.go
index a75f8990477..e4f72a36605 100644
--- a/heartbeat/monitors/plugin.go
+++ b/heartbeat/monitors/plugin.go
@@ -31,7 +31,7 @@ import (
 
 type pluginBuilder struct {
 	name    string
-	typ     Type
+	aliases []string
 	builder PluginBuilder
 	stats   registryRecorder
 }
@@ -62,7 +62,7 @@ func init() {
 		}
 
 		stats := statsForPlugin(p.name)
-		return globalPluginsReg.register(pluginBuilder{p.name, p.typ, p.builder, stats})
+		return globalPluginsReg.register(pluginBuilder{p.name, p.aliases, p.builder, stats})
 	})
 }
 
@@ -94,9 +94,9 @@ func newPluginsReg() *pluginsReg {
 }
 
 // RegisterActive registers a new active (as opposed to passive) monitor.
-func RegisterActive(name string, builder PluginBuilder) {
+func RegisterActive(name string, builder PluginBuilder, aliases ...string,) {
 	stats := statsForPlugin(name)
-	if err := globalPluginsReg.add(pluginBuilder{name, ActiveMonitor, builder, stats}); err != nil {
+	if err := globalPluginsReg.add(pluginBuilder{name, aliases, builder, stats}); err != nil {
 		panic(err)
 	}
 }
@@ -106,7 +106,7 @@ func RegisterActive(name string, builder PluginBuilder) {
 type ErrPluginAlreadyExists pluginBuilder
 
 func (m ErrPluginAlreadyExists) Error() string {
-	return fmt.Sprintf("monitor plugin '%s' already exists", m.typ)
+	return fmt.Sprintf("monitor plugin named '%s' with aliases %v already exists", m.name, m.aliases)
 }
 
 func (r *pluginsReg) add(plugin pluginBuilder) error {
@@ -114,12 +114,18 @@ func (r *pluginsReg) add(plugin pluginBuilder) error {
 		return ErrPluginAlreadyExists(plugin)
 	}
 	r.monitors[plugin.name] = plugin
+	for _, alias := range plugin.aliases {
+		if _, exists := r.monitors[alias]; exists {
+			return ErrPluginAlreadyExists(plugin)
+		}
+		r.monitors[alias] = plugin
+	}
 	return nil
 }
 
 func (r *pluginsReg) register(plugin pluginBuilder) error {
 	if _, found := r.monitors[plugin.name]; found {
-		return fmt.Errorf("monitor type %v already exists", plugin.typ)
+		return fmt.Errorf("monitor type %v already exists", plugin.name)
 	}
 
 	r.monitors[plugin.name] = plugin
diff --git a/x-pack/heartbeat/monitors/browser/browser.go b/x-pack/heartbeat/monitors/browser/browser.go
index 656f68b365b..104a68d5aec 100644
--- a/x-pack/heartbeat/monitors/browser/browser.go
+++ b/x-pack/heartbeat/monitors/browser/browser.go
@@ -19,9 +19,7 @@ import (
 )
 
 func init() {
-	monitors.RegisterActive("browser", create)
-	monitors.RegisterActive("synthetic", create)
-	monitors.RegisterActive("synthetics/synthetic", create)
+	monitors.RegisterActive("browser", create, "synthetic", "synthetics/synthetic")
 }
 
 var showExperimentalOnce = sync.Once{}

From a96277fe5e24667d27e80b746dfe01ea8ba7072a Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Thu, 14 Jan 2021 16:14:54 -0600
Subject: [PATCH 18/41] Cleanup

---
 heartbeat/beater/heartbeat.go                 | 15 ------
 heartbeat/config/config.go                    |  2 +-
 heartbeat/monitors/mocks_test.go              | 27 ++++++-----
 heartbeat/monitors/monitor.go                 |  9 +---
 heartbeat/monitors/plugin.go                  |  2 +-
 heartbeat/monitors/wrappers/monitors.go       | 46 +++++++++----------
 x-pack/heartbeat/monitors/browser/browser.go  |  4 +-
 x-pack/heartbeat/monitors/browser/config.go   | 13 +++---
 .../monitors/browser/source/github.go         | 10 ++--
 .../monitors/browser/source/local.go          |  7 ++-
 .../monitors/browser/source/source.go         |  9 ++--
 .../monitors/browser/source/zipurl.go         |  6 ++-
 .../monitors/browser/suite_runner.go          |  8 +++-
 x-pack/heartbeat/monitors/synthexec/enrich.go | 13 +++---
 14 files changed, 85 insertions(+), 86 deletions(-)

diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go
index 48478d61499..e944798887e 100644
--- a/heartbeat/beater/heartbeat.go
+++ b/heartbeat/beater/heartbeat.go
@@ -160,21 +160,6 @@ func (bt *Heartbeat) RunReloadableMonitors(b *beat.Beat) (err error) {
 	return nil
 }
 
-var suiteFactory cfgfile.RunnerFactory
-
-func RegisterSuiteFactory(factory cfgfile.RunnerFactory) {
-	suiteFactory = factory
-}
-
-// RunCentralMgmtMonitors loads any central management configured configs.
-func (bt *Heartbeat) RunSuiteMonitors(b *beat.Beat) {
-	if suiteFactory == nil {
-		return
-	}
-	monitors := cfgfile.NewRunnerList(management.DebugK, suiteFactory, b.Publisher)
-	reload.Register.MustRegisterList(b.Info.Beat+".suites", monitors)
-}
-
 // makeAutodiscover creates an autodiscover object ready to be started.
 func (bt *Heartbeat) makeAutodiscover(b *beat.Beat) (*autodiscover.Autodiscover, error) {
 	autodiscover, err := autodiscover.NewAutodiscover(
diff --git a/heartbeat/config/config.go b/heartbeat/config/config.go
index 8d60c9cb49a..052d472ea39 100644
--- a/heartbeat/config/config.go
+++ b/heartbeat/config/config.go
@@ -32,7 +32,7 @@ type Config struct {
 	ConfigMonitors  *common.Config       `config:"config.monitors"`
 	Scheduler       Scheduler            `config:"scheduler"`
 	Autodiscover    *autodiscover.Config `config:"autodiscover"`
-	SyntheticSuites []*common.Config    `config:"synthetic_suites"`
+	SyntheticSuites []*common.Config     `config:"synthetic_suites"`
 }
 
 // Scheduler defines the syntax of a heartbeat.yml scheduler block.
diff --git a/heartbeat/monitors/mocks_test.go b/heartbeat/monitors/mocks_test.go
index 3a0fc9f3241..970fdd01f12 100644
--- a/heartbeat/monitors/mocks_test.go
+++ b/heartbeat/monitors/mocks_test.go
@@ -139,21 +139,20 @@ func mockPluginBuilder() pluginBuilder {
 
 	return pluginBuilder{
 		"test",
-		"testAlias",
-		ActiveMonitor,
+		[]string{"testAlias"},
 		func(s string, config *common.Config) ([]jobs.Job, int, error) {
-		// Declare a real config block with a required attr so we can see what happens when it doesn't work
-		unpacked := struct {
-			URLs []string `config:"urls" validate:"required"`
-		}{}
-		err := config.Unpack(&unpacked)
-		if err != nil {
-			return nil, 0, err
-		}
-		c := common.Config{}
-		j, err := createMockJob("test", &c)
-		return j, 1, err
-	}, newPluginCountersRecorder("test", reg)}
+			// Declare a real config block with a required attr so we can see what happens when it doesn't work
+			unpacked := struct {
+				URLs []string `config:"urls" validate:"required"`
+			}{}
+			err := config.Unpack(&unpacked)
+			if err != nil {
+				return nil, 0, err
+			}
+			c := common.Config{}
+			j, err := createMockJob("test", &c)
+			return j, 1, err
+		}, newPluginCountersRecorder("test", reg)}
 }
 
 func mockPluginsReg() *pluginsReg {
diff --git a/heartbeat/monitors/monitor.go b/heartbeat/monitors/monitor.go
index e106c9ce272..e9fe867e413 100644
--- a/heartbeat/monitors/monitor.go
+++ b/heartbeat/monitors/monitor.go
@@ -21,7 +21,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/elastic/beats/v7/libbeat/cfgfile"
 	"sync"
 
 	"github.com/elastic/beats/v7/heartbeat/monitors/stdfields"
@@ -94,10 +93,6 @@ func (e ErrDuplicateMonitorID) Error() string {
 	return fmt.Sprintf("monitor ID %s is configured for multiple monitors! IDs must be unique values.", e.ID)
 }
 
-type MonitorAlike interface {
-	cfgfile.Runner
-}
-
 // newMonitor Creates a new monitor, without leaking resources in the event of an error.
 func newMonitor(
 	config *common.Config,
@@ -105,7 +100,7 @@ func newMonitor(
 	pipelineConnector beat.PipelineConnector,
 	scheduler *scheduler.Scheduler,
 	allowWatches bool,
-) (MonitorAlike, error) {
+) (*Monitor, error) {
 	m, err := newMonitorUnsafe(config, registrar, pipelineConnector, scheduler, allowWatches)
 	if m != nil && err != nil {
 		m.Stop()
@@ -121,7 +116,7 @@ func newMonitorUnsafe(
 	pipelineConnector beat.PipelineConnector,
 	scheduler *scheduler.Scheduler,
 	allowWatches bool,
-) (MonitorAlike, error) {
+) (*Monitor, error) {
 	// Extract just the Id, Type, and Enabled fields from the config
 	// We'll parse things more precisely later once we know what exact type of
 	// monitor we have
diff --git a/heartbeat/monitors/plugin.go b/heartbeat/monitors/plugin.go
index e4f72a36605..7f972c2f697 100644
--- a/heartbeat/monitors/plugin.go
+++ b/heartbeat/monitors/plugin.go
@@ -94,7 +94,7 @@ func newPluginsReg() *pluginsReg {
 }
 
 // RegisterActive registers a new active (as opposed to passive) monitor.
-func RegisterActive(name string, builder PluginBuilder, aliases ...string,) {
+func RegisterActive(name string, builder PluginBuilder, aliases ...string) {
 	stats := statsForPlugin(name)
 	if err := globalPluginsReg.add(pluginBuilder{name, aliases, builder, stats}); err != nil {
 		panic(err)
diff --git a/heartbeat/monitors/wrappers/monitors.go b/heartbeat/monitors/wrappers/monitors.go
index 2a8a11ae4b1..ce62541813c 100644
--- a/heartbeat/monitors/wrappers/monitors.go
+++ b/heartbeat/monitors/wrappers/monitors.go
@@ -38,27 +38,32 @@ import (
 
 // WrapCommon applies the common wrappers that all monitor jobs get.
 func WrapCommon(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.Job {
-	jobWrappers := []jobs.JobWrapper{
-		addMonitorMeta(stdMonFields, len(js) > 1),
-		addMonitorStatus(stdMonFields.Type),
-	}
-
 	if stdMonFields.Type == "browser" {
-		jobWrappers = append(jobWrappers, addMonitorDuration)
+		return WrapBrowser(js, stdMonFields)
+	} else {
+		return WrapLightweight(js, stdMonFields)
 	}
+}
+
+func WrapLightweight(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.Job {
+	return jobs.WrapAllSeparately(
+		jobs.WrapAll(
+			js,
+			addMonitorMeta(stdMonFields, len(js) > 1),
+			addMonitorStatus(stdMonFields.Type),
+			addMonitorDuration,
+		),
+		func() jobs.JobWrapper {
+			return makeAddSummary(stdMonFields.Type)
+		})
+}
 
-	baseWrapped := jobs.WrapAll(
+func WrapBrowser(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.Job {
+	return jobs.WrapAll(
 		js,
-		jobWrappers...,
+		addMonitorMeta(stdMonFields, len(js) > 1),
+		addMonitorStatus(stdMonFields.Type),
 	)
-	if stdMonFields.Type != "browser" {
-		return jobs.WrapAllSeparately(
-			baseWrapped,
-			func() jobs.JobWrapper {
-				return makeAddSummary(stdMonFields.Type)
-			})
-	}
-	return baseWrapped
 }
 
 // addMonitorMeta adds the id, name, and type fields to the monitor.
@@ -69,7 +74,7 @@ func addMonitorMeta(stdMonFields stdfields.StdMonitorFields, isMulti bool) jobs.
 			cont, e := job(event)
 			thisID := stdMonFields.ID
 			thisName := stdMonFields.Name
-			// Allow jobs to override the ID, useful for suites
+			// Allow jobs to override the ID, useful for browser suites
 			// which do this logic on their own
 			if v, _ := event.GetValue("monitor.id"); v != nil {
 				thisID = v.(string)
@@ -132,13 +137,6 @@ func addMonitorStatus(monitorType string) jobs.JobWrapper {
 		return func(event *beat.Event) ([]jobs.Job, error) {
 			cont, err := origJob(event)
 
-			// Non-summary browser events have no status associated
-			if monitorType == "browser" {
-				if t, _ := event.GetValue("synthetics.type"); t != "heartbeat/summary" {
-					return cont, nil
-				}
-			}
-
 			fields := common.MapStr{
 				"monitor": common.MapStr{
 					"status": look.Status(err),
diff --git a/x-pack/heartbeat/monitors/browser/browser.go b/x-pack/heartbeat/monitors/browser/browser.go
index 104a68d5aec..eb0227e15b1 100644
--- a/x-pack/heartbeat/monitors/browser/browser.go
+++ b/x-pack/heartbeat/monitors/browser/browser.go
@@ -56,7 +56,7 @@ func create(name string, cfg *common.Config) (js []jobs.Job, endpoints int, err
 
 	j, err := synthexec.SuiteJob(context.TODO(), ss.Workdir(), ss.Params())
 	if err != nil {
-			return nil, 0, err
-		}
+		return nil, 0, err
+	}
 	return []jobs.Job{j}, 1, nil
 }
diff --git a/x-pack/heartbeat/monitors/browser/config.go b/x-pack/heartbeat/monitors/browser/config.go
index 9bcbf69bc28..412b7b9c611 100644
--- a/x-pack/heartbeat/monitors/browser/config.go
+++ b/x-pack/heartbeat/monitors/browser/config.go
@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package browser
 
 import (
@@ -6,11 +10,8 @@ import (
 )
 
 type Config struct {
-	Schedule string                 `config:"schedule"`
-	Params   map[string]interface{} `config:"params"`
+	Schedule  string                 `config:"schedule"`
+	Params    map[string]interface{} `config:"params"`
 	RawConfig *common.Config
-	Source *source.Source `config:"source"`
+	Source    *source.Source `config:"source"`
 }
-
-
-
diff --git a/x-pack/heartbeat/monitors/browser/source/github.go b/x-pack/heartbeat/monitors/browser/source/github.go
index 6e264a99430..ae0619d1b67 100644
--- a/x-pack/heartbeat/monitors/browser/source/github.go
+++ b/x-pack/heartbeat/monitors/browser/source/github.go
@@ -1,11 +1,15 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package source
 
 // GithubSource handles configs for github repos, using the API defined here:
 // https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#download-a-repository-archive-tar.
 type GithubSource struct {
-	Owner	string `config:"owner"`
-	Repo string `config:"repo"`
-	Ref string `config:"ref"`
+	Owner   string `config:"owner"`
+	Repo    string `config:"repo"`
+	Ref     string `config:"ref"`
 	UrlBase string `config:"url_base"`
 	PollingSource
 }
diff --git a/x-pack/heartbeat/monitors/browser/source/local.go b/x-pack/heartbeat/monitors/browser/source/local.go
index 4f0029e10ae..3a3e68604df 100644
--- a/x-pack/heartbeat/monitors/browser/source/local.go
+++ b/x-pack/heartbeat/monitors/browser/source/local.go
@@ -1,13 +1,18 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package source
 
 import (
 	"fmt"
 	"io/ioutil"
+
 	"github.com/otiai10/copy"
 )
 
 type LocalSource struct {
-	OrigPath     string                `config:"path"`
+	OrigPath    string `config:"path"`
 	workingPath string
 	BaseSource
 }
diff --git a/x-pack/heartbeat/monitors/browser/source/source.go b/x-pack/heartbeat/monitors/browser/source/source.go
index 4d40eca8e3a..6a13e661a45 100644
--- a/x-pack/heartbeat/monitors/browser/source/source.go
+++ b/x-pack/heartbeat/monitors/browser/source/source.go
@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package source
 
 import (
@@ -28,7 +32,7 @@ func (s *Source) Active() ISource {
 }
 
 func (s *Source) Validate() error {
-	if  s.Active() == nil {
+	if s.Active() == nil {
 		return fmt.Errorf("no valid source specified! Choose one of local, github, zip_url")
 	}
 	return nil
@@ -40,11 +44,10 @@ type ISource interface {
 }
 
 type BaseSource struct {
-	Type	 string					`config:"type"`
+	Type string `config:"type"`
 }
 
 type PollingSource struct {
 	CheckEvery int `config:"check_every"`
 	BaseSource
 }
-
diff --git a/x-pack/heartbeat/monitors/browser/source/zipurl.go b/x-pack/heartbeat/monitors/browser/source/zipurl.go
index fcc1fda1518..d6aa496c40c 100644
--- a/x-pack/heartbeat/monitors/browser/source/zipurl.go
+++ b/x-pack/heartbeat/monitors/browser/source/zipurl.go
@@ -1,7 +1,11 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package source
 
 type ZipURLSource struct {
-	Url string `config:"url"`
+	Url     string            `config:"url"`
 	Headers map[string]string `config:"headers"`
 	PollingSource
 }
diff --git a/x-pack/heartbeat/monitors/browser/suite_runner.go b/x-pack/heartbeat/monitors/browser/suite_runner.go
index 1e15457500f..aba4758cbec 100644
--- a/x-pack/heartbeat/monitors/browser/suite_runner.go
+++ b/x-pack/heartbeat/monitors/browser/suite_runner.go
@@ -1,12 +1,16 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package browser
 
 import (
 	"context"
+
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
-
 type JourneyLister func(ctx context.Context, suitePath string, params common.MapStr) (journeyNames []string, err error)
 
 var journeyListSingleton JourneyLister
@@ -18,7 +22,7 @@ type SyntheticSuite struct {
 
 func NewSuite(rawCfg *common.Config) (*SyntheticSuite, error) {
 	ss := &SyntheticSuite{
-		rawCfg: rawCfg,
+		rawCfg:   rawCfg,
 		suiteCfg: &Config{},
 	}
 	err := rawCfg.Unpack(ss.suiteCfg)
diff --git a/x-pack/heartbeat/monitors/synthexec/enrich.go b/x-pack/heartbeat/monitors/synthexec/enrich.go
index cc001ef8bdf..7f8df6ee487 100644
--- a/x-pack/heartbeat/monitors/synthexec/enrich.go
+++ b/x-pack/heartbeat/monitors/synthexec/enrich.go
@@ -6,15 +6,16 @@ package synthexec
 
 import (
 	"fmt"
-	"github.com/gofrs/uuid"
 	"time"
 
+	"github.com/gofrs/uuid"
+
 	"github.com/elastic/beats/v7/heartbeat/eventext"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
 )
 
-type enricher func (event *beat.Event, se *SynthEvent) error
+type enricher func(event *beat.Event, se *SynthEvent) error
 
 type streamEnricher struct {
 	je *journeyEnricher
@@ -32,7 +33,7 @@ func (e *streamEnricher) enrich(event *beat.Event, se *SynthEvent) error {
 // where relevant to properly enrich *beat.Event instances.
 type journeyEnricher struct {
 	journeyComplete bool
-	journey *Journey
+	journey         *Journey
 	checkGroup      string
 	errorCount      int
 	lastError       error
@@ -82,8 +83,8 @@ func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent) error {
 
 	eventext.MergeEventFields(event, common.MapStr{
 		"monitor": common.MapStr{
-			"id": je.journey.Id,
-			"name": je.journey.Name,
+			"id":          je.journey.Id,
+			"name":        je.journey.Name,
 			"check_group": je.checkGroup,
 		},
 	})
@@ -142,7 +143,7 @@ func (je *journeyEnricher) createSummary(event *beat.Event) error {
 				},
 			},
 			"summary": common.MapStr{
-				"up": up,
+				"up":   up,
 				"down": down,
 			},
 		})

From 44628b0a2c23ca20666b91317c6c4a13a751d351 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Thu, 14 Jan 2021 18:03:57 -0600
Subject: [PATCH 19/41] Suites almost working, inline works

---
 x-pack/heartbeat/monitors/browser/browser.go  | 27 +++++--
 .../monitors/browser/source/github.go         | 23 ------
 .../browser/source/{zipurl.go => inline.go}   | 15 ++--
 .../monitors/browser/source/source.go         |  9 +--
 .../monitors/browser/suite_runner.go          |  7 ++
 .../{ => browser}/synthexec/enrich.go         | 11 ++-
 .../{ => browser}/synthexec/enrich_test.go    |  0
 .../synthexec/execmultiplexer.go              |  0
 .../{ => browser}/synthexec/synthexec.go      | 75 ++-----------------
 .../{ => browser}/synthexec/synthexec_test.go |  0
 .../{ => browser}/synthexec/synthtypes.go     |  0
 .../synthexec/synthtypes_test.go              |  0
 .../{ => browser}/synthexec/testcmd/main.go   |  0
 .../synthexec/testcmd/sample.ndjson           |  0
 .../sample-synthetics-config/heartbeat.yml    | 38 +++++-----
 15 files changed, 72 insertions(+), 133 deletions(-)
 delete mode 100644 x-pack/heartbeat/monitors/browser/source/github.go
 rename x-pack/heartbeat/monitors/browser/source/{zipurl.go => inline.go} (50%)
 rename x-pack/heartbeat/monitors/{ => browser}/synthexec/enrich.go (94%)
 rename x-pack/heartbeat/monitors/{ => browser}/synthexec/enrich_test.go (100%)
 rename x-pack/heartbeat/monitors/{ => browser}/synthexec/execmultiplexer.go (100%)
 rename x-pack/heartbeat/monitors/{ => browser}/synthexec/synthexec.go (81%)
 rename x-pack/heartbeat/monitors/{ => browser}/synthexec/synthexec_test.go (100%)
 rename x-pack/heartbeat/monitors/{ => browser}/synthexec/synthtypes.go (100%)
 rename x-pack/heartbeat/monitors/{ => browser}/synthexec/synthtypes_test.go (100%)
 rename x-pack/heartbeat/monitors/{ => browser}/synthexec/testcmd/main.go (100%)
 rename x-pack/heartbeat/monitors/{ => browser}/synthexec/testcmd/sample.ndjson (100%)

diff --git a/x-pack/heartbeat/monitors/browser/browser.go b/x-pack/heartbeat/monitors/browser/browser.go
index eb0227e15b1..8f143bc107c 100644
--- a/x-pack/heartbeat/monitors/browser/browser.go
+++ b/x-pack/heartbeat/monitors/browser/browser.go
@@ -11,11 +11,13 @@ import (
 	"os/user"
 	"sync"
 
+	"github.com/elastic/beats/v7/libbeat/beat"
+
 	"github.com/elastic/beats/v7/heartbeat/monitors"
 	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
-	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/synthexec"
+	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/synthexec"
 )
 
 func init() {
@@ -51,12 +53,23 @@ func create(name string, cfg *common.Config) (js []jobs.Job, endpoints int, err
 		return nil, 0, err
 	}
 
-	// TODO: Run this before each suite job invocation, not just at startup
-	ss.Fetch()
-
-	j, err := synthexec.SuiteJob(context.TODO(), ss.Workdir(), ss.Params())
-	if err != nil {
-		return nil, 0, err
+	var j jobs.Job
+	if src, ok := ss.InlineSource(); ok {
+		j = synthexec.InlineJourneyJob(context.TODO(), src, ss.Params())
+	} else {
+		j = func(event *beat.Event) ([]jobs.Job, error) {
+			err := ss.Fetch()
+			if err != nil {
+				return nil, fmt.Errorf("could not fetch for suite job: %w", err)
+			}
+			logp.Warn("WORKDIR IS %s", ss.Workdir())
+			sj, err := synthexec.SuiteJob(context.TODO(), ss.Workdir(), ss.Params())
+			if err != nil {
+				return nil, err
+			}
+			return sj(event)
+		}
 	}
+
 	return []jobs.Job{j}, 1, nil
 }
diff --git a/x-pack/heartbeat/monitors/browser/source/github.go b/x-pack/heartbeat/monitors/browser/source/github.go
deleted file mode 100644
index ae0619d1b67..00000000000
--- a/x-pack/heartbeat/monitors/browser/source/github.go
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-package source
-
-// GithubSource handles configs for github repos, using the API defined here:
-// https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#download-a-repository-archive-tar.
-type GithubSource struct {
-	Owner   string `config:"owner"`
-	Repo    string `config:"repo"`
-	Ref     string `config:"ref"`
-	UrlBase string `config:"url_base"`
-	PollingSource
-}
-
-func (g *GithubSource) Fetch() error {
-	panic("implement me")
-}
-
-func (g *GithubSource) Workdir() string {
-	panic("implement me")
-}
diff --git a/x-pack/heartbeat/monitors/browser/source/zipurl.go b/x-pack/heartbeat/monitors/browser/source/inline.go
similarity index 50%
rename from x-pack/heartbeat/monitors/browser/source/zipurl.go
rename to x-pack/heartbeat/monitors/browser/source/inline.go
index d6aa496c40c..5c67b7f32d9 100644
--- a/x-pack/heartbeat/monitors/browser/source/zipurl.go
+++ b/x-pack/heartbeat/monitors/browser/source/inline.go
@@ -4,16 +4,15 @@
 
 package source
 
-type ZipURLSource struct {
-	Url     string            `config:"url"`
-	Headers map[string]string `config:"headers"`
-	PollingSource
+type InlineSource struct {
+	Script string `config:"script"`
+	BaseSource
 }
 
-func (z *ZipURLSource) Fetch() error {
-	panic("implement me")
+func (l *InlineSource) Fetch() (err error) {
+	return nil
 }
 
-func (z *ZipURLSource) Workdir() string {
-	panic("implement me")
+func (l *InlineSource) Workdir() string {
+	return ""
 }
diff --git a/x-pack/heartbeat/monitors/browser/source/source.go b/x-pack/heartbeat/monitors/browser/source/source.go
index 6a13e661a45..9286d162cf3 100644
--- a/x-pack/heartbeat/monitors/browser/source/source.go
+++ b/x-pack/heartbeat/monitors/browser/source/source.go
@@ -10,8 +10,7 @@ import (
 
 type Source struct {
 	Local      *LocalSource  `config:"local"`
-	Github     *GithubSource `config:"github"`
-	ZipURL     *ZipURLSource `config:"zip_url"`
+	Inline     *InlineSource `config:"inline"`
 	ActiveMemo ISource       // cache for selected source
 }
 
@@ -22,10 +21,8 @@ func (s *Source) Active() ISource {
 
 	if s.Local != nil {
 		s.ActiveMemo = s.Local
-	} else if s.Github != nil {
-		s.ActiveMemo = s.Github
-	} else if s.ZipURL != nil {
-		s.ActiveMemo = s.ZipURL
+	} else if s.Inline != nil {
+		s.ActiveMemo = s.Inline
 	}
 
 	return s.ActiveMemo
diff --git a/x-pack/heartbeat/monitors/browser/suite_runner.go b/x-pack/heartbeat/monitors/browser/suite_runner.go
index aba4758cbec..28beb1dd5f3 100644
--- a/x-pack/heartbeat/monitors/browser/suite_runner.go
+++ b/x-pack/heartbeat/monitors/browser/suite_runner.go
@@ -45,6 +45,13 @@ func (s *SyntheticSuite) Workdir() string {
 	return s.suiteCfg.Source.Active().Workdir()
 }
 
+func (s *SyntheticSuite) InlineSource() (string, bool) {
+	if s.suiteCfg.Source.Inline != nil {
+		return s.suiteCfg.Source.Inline.Script, true
+	}
+	return "", false
+}
+
 func (s *SyntheticSuite) Params() map[string]interface{} {
 	return s.suiteCfg.Params
 }
diff --git a/x-pack/heartbeat/monitors/synthexec/enrich.go b/x-pack/heartbeat/monitors/browser/synthexec/enrich.go
similarity index 94%
rename from x-pack/heartbeat/monitors/synthexec/enrich.go
rename to x-pack/heartbeat/monitors/browser/synthexec/enrich.go
index 7f8df6ee487..d6da01d5a86 100644
--- a/x-pack/heartbeat/monitors/synthexec/enrich.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/enrich.go
@@ -83,11 +83,18 @@ func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent) error {
 
 	eventext.MergeEventFields(event, common.MapStr{
 		"monitor": common.MapStr{
-			"id":          je.journey.Id,
-			"name":        je.journey.Name,
 			"check_group": je.checkGroup,
 		},
 	})
+	// Inline jobs have no journey
+	if je.journey != nil {
+		eventext.MergeEventFields(event, common.MapStr{
+			"monitor": common.MapStr{
+				"id":   je.journey.Id,
+				"name": je.journey.Name,
+			},
+		})
+	}
 
 	return je.enrichSynthEvent(event, se)
 }
diff --git a/x-pack/heartbeat/monitors/synthexec/enrich_test.go b/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go
similarity index 100%
rename from x-pack/heartbeat/monitors/synthexec/enrich_test.go
rename to x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go
diff --git a/x-pack/heartbeat/monitors/synthexec/execmultiplexer.go b/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go
similarity index 100%
rename from x-pack/heartbeat/monitors/synthexec/execmultiplexer.go
rename to x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go
diff --git a/x-pack/heartbeat/monitors/synthexec/synthexec.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
similarity index 81%
rename from x-pack/heartbeat/monitors/synthexec/synthexec.go
rename to x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
index 7ebec92a4bb..3f95f42f453 100644
--- a/x-pack/heartbeat/monitors/synthexec/synthexec.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
@@ -10,7 +10,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -23,74 +22,13 @@ import (
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
-
-	"github.com/otiai10/copy"
 )
 
 const debugSelector = "synthexec"
 
-func InitSuite(ctx context.Context, origSuitePath string, params common.MapStr) (journeyNames []string, err error) {
-	dir, err := ioutil.TempDir("/tmp", "elastic-synthetics-")
-	if err != nil {
-		return nil, err
-	}
-
-	err = copy.Copy(origSuitePath, dir)
-	if err != nil {
-		return nil, err
-	}
-
-	return ListJourneys(ctx, dir, params)
-}
-
-// ListJourneys takes the given suite performs a dry run, capturing the Journey names, and returns the list.
-func ListJourneys(ctx context.Context, suitePath string, params common.MapStr) (journeyNames []string, err error) {
-	dir, err := getSuiteDir(suitePath)
-	if err != nil {
-		return nil, err
-	}
-
-	if os.Getenv("ELASTIC_SYNTHETICS_OFFLINE") != "true" {
-		// Ensure all deps installed
-		err = runSimpleCommand(exec.Command("npm", "install"), dir)
-		if err != nil {
-			return nil, err
-		}
-
-		// Update playwright, needs to run separately to ensure post-install hook is run that downloads
-		// chrome. See https://github.com/microsoft/playwright/issues/3712
-		err = runSimpleCommand(exec.Command("npm", "install", "playwright-chromium"), dir)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	cmdFactory, err := suiteCommandFactory(dir, suitePath, "--dry-run")
-	if err != nil {
-		return nil, err
-	}
-
-	mpx, err := runCmd(ctx, cmdFactory(), nil, params)
-Outer:
-	for {
-		select {
-		case se := <-mpx.SynthEvents():
-			if se == nil {
-				break Outer
-			}
-			if se.Type == "journey/register" {
-				journeyNames = append(journeyNames, se.Journey.Name)
-			}
-		}
-	}
-
-	logp.Info("Discovered journeys %#v", journeyNames)
-	return journeyNames, nil
-}
-
 // SuiteJob will run a single journey by name from the given suite.
-func SuiteJob(ctx context.Context, suiteFile string, params common.MapStr) (jobs.Job, error) {
-	newCmd, err := suiteCommandFactory(suiteFile, suiteFile, "--screenshots")
+func SuiteJob(ctx context.Context, suitePath string, params common.MapStr) (jobs.Job, error) {
+	newCmd, err := suiteCommandFactory(suitePath, "--screenshots")
 	if err != nil {
 		return nil, err
 	}
@@ -98,8 +36,8 @@ func SuiteJob(ctx context.Context, suiteFile string, params common.MapStr) (jobs
 	return startCmdJob(ctx, newCmd, nil, params), nil
 }
 
-func suiteCommandFactory(suiteFile string, args ...string) (func() *exec.Cmd, error) {
-	npmRoot, err := getNpmRoot(suiteFile)
+func suiteCommandFactory(suitePath string, args ...string) (func() *exec.Cmd, error) {
+	npmRoot, err := getNpmRoot(suitePath)
 	if err != nil {
 		return nil, err
 	}
@@ -347,6 +285,9 @@ func getNpmRoot(path string) (string, error) {
 // getNpmRootIn does the same as getNpmRoot but remembers the original path for
 // debugging.
 func getNpmRootIn(path, origPath string) (string, error) {
+	if path == "" {
+		return "", fmt.Errorf("cannot check for package.json in empty path: '%s'", origPath)
+	}
 	candidate := filepath.Join(path, "package.json")
 	_, err := os.Lstat(candidate)
 	if err == nil {
@@ -355,7 +296,7 @@ func getNpmRootIn(path, origPath string) (string, error) {
 	// Try again one level up
 	parent := filepath.Dir(path)
 	if len(parent) < 2 {
-		return "", fmt.Errorf("no package.json found in %s", origPath)
+		return "", fmt.Errorf("no package.json found in '%s'", origPath)
 	}
 	return getNpmRootIn(parent, origPath)
 }
diff --git a/x-pack/heartbeat/monitors/synthexec/synthexec_test.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go
similarity index 100%
rename from x-pack/heartbeat/monitors/synthexec/synthexec_test.go
rename to x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go
diff --git a/x-pack/heartbeat/monitors/synthexec/synthtypes.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go
similarity index 100%
rename from x-pack/heartbeat/monitors/synthexec/synthtypes.go
rename to x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go
diff --git a/x-pack/heartbeat/monitors/synthexec/synthtypes_test.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go
similarity index 100%
rename from x-pack/heartbeat/monitors/synthexec/synthtypes_test.go
rename to x-pack/heartbeat/monitors/browser/synthexec/synthtypes_test.go
diff --git a/x-pack/heartbeat/monitors/synthexec/testcmd/main.go b/x-pack/heartbeat/monitors/browser/synthexec/testcmd/main.go
similarity index 100%
rename from x-pack/heartbeat/monitors/synthexec/testcmd/main.go
rename to x-pack/heartbeat/monitors/browser/synthexec/testcmd/main.go
diff --git a/x-pack/heartbeat/monitors/synthexec/testcmd/sample.ndjson b/x-pack/heartbeat/monitors/browser/synthexec/testcmd/sample.ndjson
similarity index 100%
rename from x-pack/heartbeat/monitors/synthexec/testcmd/sample.ndjson
rename to x-pack/heartbeat/monitors/browser/synthexec/testcmd/sample.ndjson
diff --git a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
index 6eb715ffbd5..7aab30acaff 100644
--- a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
+++ b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
@@ -14,30 +14,28 @@ heartbeat.monitors:
       path: "/home/andrewvc/projects/synthetics/examples/todos"
   schedule: '@every 1m'
 - type: http
-  enabled: false
+  enabled: true
   id: SimpleHTTP
   urls: http://www.google.com
   schedule: "@every 15s"
   name: Simple HTTP
-#- type: synthetic
-#  enabled: false
-#  id: my-monitor
-#  name: My Monitor
-#  source:
-#    script:
-#      step("load homepage", async () => {
-#          await page.goto('https://www.elastic.co');
-#      });
-#      step("hover over products menu", async () => {
-#          await page.hover('css=[data-nav-item=products]');
-#      });
-#      step("failme", async () => {
-#          await page.hhover('css=[data-nav-item=products]');
-#      });
-#      step("skip me", async () => {
-#          // noop
-#      });
-#  schedule: "@every 1m"
+- type: browser
+  enabled: false
+  id: my-monitor
+  name: My Monitor
+  source:
+    inline:
+      script:
+        step("load homepage", async () => {
+            await page.goto('https://www.elastic.co');
+        });
+        step("hover over products menu", async () => {
+            await page.hover('css=[data-nav-item=products]');
+        });
+        step("failme", async () => {
+            await page.hhover('css=[data-nav-item=products]');
+        });
+  schedule: "@every 1m"
 
 setup.template.settings:
   index.number_of_shards: 1

From 75250bb148ac1a0a2c399166a257175641510ed1 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Fri, 15 Jan 2021 15:38:10 -0600
Subject: [PATCH 20/41] Checkpoint

---
 x-pack/heartbeat/monitors/browser/browser.go  |  1 -
 .../monitors/browser/synthexec/enrich.go      |  3 +-
 .../monitors/browser/synthexec/enrich_test.go |  6 ++--
 .../browser/synthexec/execmultiplexer.go      | 34 +++++++++++++++----
 .../monitors/browser/synthexec/synthexec.go   |  4 ++-
 .../browser/synthexec/synthexec_test.go       |  2 ++
 .../browser/synthexec/testcmd/main.go         | 33 +++++++++++++-----
 7 files changed, 61 insertions(+), 22 deletions(-)

diff --git a/x-pack/heartbeat/monitors/browser/browser.go b/x-pack/heartbeat/monitors/browser/browser.go
index 8f143bc107c..c2ce7708569 100644
--- a/x-pack/heartbeat/monitors/browser/browser.go
+++ b/x-pack/heartbeat/monitors/browser/browser.go
@@ -62,7 +62,6 @@ func create(name string, cfg *common.Config) (js []jobs.Job, endpoints int, err
 			if err != nil {
 				return nil, fmt.Errorf("could not fetch for suite job: %w", err)
 			}
-			logp.Warn("WORKDIR IS %s", ss.Workdir())
 			sj, err := synthexec.SuiteJob(context.TODO(), ss.Workdir(), ss.Params())
 			if err != nil {
 				return nil, err
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/enrich.go b/x-pack/heartbeat/monitors/browser/synthexec/enrich.go
index d6da01d5a86..cbee774cb3f 100644
--- a/x-pack/heartbeat/monitors/browser/synthexec/enrich.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/enrich.go
@@ -142,7 +142,8 @@ func (je *journeyEnricher) createSummary(event *beat.Event) error {
 		eventext.MergeEventFields(event, common.MapStr{
 			"url": je.urlFields,
 			"synthetics": common.MapStr{
-				"type": "heartbeat/summary",
+				"type":    "heartbeat/summary",
+				"journey": je.journey,
 			},
 			"monitor": common.MapStr{
 				"duration": common.MapStr{
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go b/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go
index 3ea3eb1ec58..cf1cc0dd6cf 100644
--- a/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/enrich_test.go
@@ -75,12 +75,12 @@ func TestJourneyEnricher(t *testing.T) {
 	// We need an expectation for each input
 	// plus a final expectation for the summary which comes
 	// on the nil data.
-	for idx, se := range append(synthEvents, nil) {
+	for idx, se := range synthEvents {
 		e := &beat.Event{}
 		t.Run(fmt.Sprintf("event %d", idx), func(t *testing.T) {
 			enrichErr := je.enrich(e, se)
 
-			if se != nil {
+			if se != nil && se.Type != "journey/end" {
 				// Test that the created event includes the mapped
 				// version of the event
 				testslike.Test(t, lookslike.MustCompile(se.ToMap()), e.Fields)
@@ -89,7 +89,7 @@ func TestJourneyEnricher(t *testing.T) {
 				if se.Error != nil {
 					require.Equal(t, stepError(se.Error), enrichErr)
 				}
-			} else {
+			} else { // journey end gets a summary
 				require.Equal(t, stepError(syntherr), enrichErr)
 
 				u, _ := url.Parse(url1)
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go b/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go
index 29fa9a8f3b6..b1a92aa8247 100644
--- a/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go
@@ -5,13 +5,17 @@
 package synthexec
 
 import (
+	"encoding/json"
+
 	"github.com/elastic/beats/v7/libbeat/common/atomic"
+	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
 type ExecMultiplexer struct {
-	eventCounter *atomic.Int
-	synthEvents  chan *SynthEvent
-	done         chan struct{}
+	currentJourney *atomic.Bool
+	eventCounter   *atomic.Int
+	synthEvents    chan *SynthEvent
+	done           chan struct{}
 }
 
 func (e ExecMultiplexer) Close() {
@@ -22,8 +26,23 @@ func (e ExecMultiplexer) writeSynthEvent(se *SynthEvent) {
 	if se == nil { // we skip writing nil events, since a nil means we're done
 		return
 	}
+
+	if se.Type == "journey/start" {
+		e.currentJourney.Store(true)
+	}
+	hasCurrentJourney := e.currentJourney.Load()
+	if se.Type == "journey/end" {
+		e.currentJourney.Store(false)
+	}
+
+	out, err := json.Marshal(se)
+
 	se.index = e.eventCounter.Inc()
-	e.synthEvents <- se
+	if hasCurrentJourney {
+		e.synthEvents <- se
+	} else {
+		logp.Warn("received output from synthetics outside of journey scope: %s %s", out, err)
+	}
 }
 
 // SynthEvents returns a read only channel for synth events
@@ -43,8 +62,9 @@ func (e ExecMultiplexer) Wait() {
 
 func NewExecMultiplexer() *ExecMultiplexer {
 	return &ExecMultiplexer{
-		eventCounter: atomic.NewInt(-1), // Start from -1 so first call to Inc returns 0
-		synthEvents:  make(chan *SynthEvent),
-		done:         make(chan struct{}),
+		currentJourney: atomic.NewBool(false),
+		eventCounter:   atomic.NewInt(-1), // Start from -1 so first call to Inc returns 0
+		synthEvents:    make(chan *SynthEvent),
+		done:           make(chan struct{}),
 	}
 }
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
index 3f95f42f453..5e67a190b7d 100644
--- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
@@ -28,7 +28,9 @@ const debugSelector = "synthexec"
 
 // SuiteJob will run a single journey by name from the given suite.
 func SuiteJob(ctx context.Context, suitePath string, params common.MapStr) (jobs.Job, error) {
-	newCmd, err := suiteCommandFactory(suitePath, "--screenshots")
+	// Run the command in the given suitePath, use '.' as the first arg since the command runs
+	// in the correct dir
+	newCmd, err := suiteCommandFactory(suitePath, ".", "--screenshots")
 	if err != nil {
 		return nil, err
 	}
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go
index be8b4a8df8b..75a5ff5a497 100644
--- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec_test.go
@@ -130,10 +130,12 @@ Loop:
 
 	t.Run("has echo'd stdin to stdout", func(t *testing.T) {
 		stdoutEvents := eventsWithType("stdout")
+		require.Len(t, stdoutEvents, 1)
 		require.Equal(t, stdinStr, stdoutEvents[0].Payload["message"])
 	})
 	t.Run("has echo'd two lines to stderr", func(t *testing.T) {
 		stdoutEvents := eventsWithType("stderr")
+		require.Len(t, stdoutEvents, 2)
 		require.Equal(t, "Stderr 1", stdoutEvents[0].Payload["message"])
 		require.Equal(t, "Stderr 2", stdoutEvents[1].Payload["message"])
 	})
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/testcmd/main.go b/x-pack/heartbeat/monitors/browser/synthexec/testcmd/main.go
index 57f2a48f2cc..bd70d8d58c5 100644
--- a/x-pack/heartbeat/monitors/browser/synthexec/testcmd/main.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/testcmd/main.go
@@ -8,17 +8,10 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"time"
 )
 
 func main() {
-	// Sample output to test stdout
-	stdin := bufio.NewReader(os.Stdin)
-
-	stdinLine, _ := stdin.ReadString('\n')
-	fmt.Fprintln(os.Stdout, stdinLine)
-	fmt.Fprintln(os.Stderr, "Stderr 1")
-	fmt.Fprintln(os.Stderr, "Stderr 2")
-
 	// For sending JSON results
 	pipe := os.NewFile(3, "pipe")
 
@@ -28,8 +21,30 @@ func main() {
 		os.Exit(1)
 	}
 	scanner := bufio.NewScanner(file)
+	i := 0
 	for scanner.Scan() {
-		fmt.Fprintln(pipe, scanner.Text())
+		// We need to test console out within a journey context
+		// so we wait till the first line, a journey/start is written
+		// we need to make sure the these raw lines are received after
+		// the journey start, so, even though we're careful to use
+		// un-buffered I/O we sleep for a generous 100ms before and after
+		// to make sure these lines are in the right context
+		// otherwise they might get lost.
+		// Note, in the real world lost lines here aren't a big deal
+		// these only are relevant in error situations, and this is a
+		// pathological case
+		if i == 1 {
+			time.Sleep(time.Millisecond * 100)
+			stdin := bufio.NewReader(os.Stdin)
+			stdinLine, _ := stdin.ReadString('\n')
+			os.Stdout.WriteString(stdinLine + "\n")
+			os.Stderr.WriteString("Stderr 1\n")
+			os.Stderr.WriteString("Stderr 2\n")
+			time.Sleep(time.Millisecond * 100)
+		}
+		pipe.WriteString(scanner.Text())
+		pipe.WriteString("\n")
+		i++
 	}
 	if scanner.Err() != nil {
 		fmt.Printf("Scanner error %s", scanner.Err())

From 54eff79407b54502e9f0bed17d9667bf75bbe5ce Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Fri, 15 Jan 2021 15:53:28 -0600
Subject: [PATCH 21/41] npm i + beginnings of cleanup

---
 .../monitors/browser/source/inline.go         |  4 ++
 .../monitors/browser/source/local.go          | 62 +++++++++++++++++++
 .../monitors/browser/source/source.go         |  1 +
 .../monitors/browser/synthexec/synthexec.go   | 25 --------
 4 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/x-pack/heartbeat/monitors/browser/source/inline.go b/x-pack/heartbeat/monitors/browser/source/inline.go
index 5c67b7f32d9..5bcf43dc95a 100644
--- a/x-pack/heartbeat/monitors/browser/source/inline.go
+++ b/x-pack/heartbeat/monitors/browser/source/inline.go
@@ -16,3 +16,7 @@ func (l *InlineSource) Fetch() (err error) {
 func (l *InlineSource) Workdir() string {
 	return ""
 }
+
+func (l *InlineSource) Close() error {
+	return nil
+}
diff --git a/x-pack/heartbeat/monitors/browser/source/local.go b/x-pack/heartbeat/monitors/browser/source/local.go
index 3a3e68604df..b3c7d53022c 100644
--- a/x-pack/heartbeat/monitors/browser/source/local.go
+++ b/x-pack/heartbeat/monitors/browser/source/local.go
@@ -7,6 +7,11 @@ package source
 import (
 	"fmt"
 	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/elastic/beats/v7/libbeat/logp"
 
 	"github.com/otiai10/copy"
 )
@@ -18,6 +23,9 @@ type LocalSource struct {
 }
 
 func (l *LocalSource) Fetch() (err error) {
+	if l.workingPath != "" {
+		return nil
+	}
 	l.workingPath, err = ioutil.TempDir("/tmp", "elastic-synthetics-")
 	if err != nil {
 		return fmt.Errorf("could not create tmp dir: %w", err)
@@ -27,9 +35,63 @@ func (l *LocalSource) Fetch() (err error) {
 	if err != nil {
 		return fmt.Errorf("could not copy suite: %w", err)
 	}
+
+	dir, err := getSuiteDir(l.workingPath)
+	if err != nil {
+		return err
+	}
+
+	if os.Getenv("ELASTIC_SYNTHETICS_OFFLINE") != "true" {
+		// Ensure all deps installed
+		err = runSimpleCommand(exec.Command("npm", "install"), dir)
+		if err != nil {
+			return err
+		}
+
+		// Update playwright, needs to run separately to ensure post-install hook is run that downloads
+		// chrome. See https://github.com/microsoft/playwright/issues/3712
+		err = runSimpleCommand(exec.Command("npm", "install", "playwright-chromium"), dir)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
 func (l *LocalSource) Workdir() string {
 	return l.workingPath
 }
+
+func (l *LocalSource) Close() error {
+	if l.workingPath != "" {
+		return os.RemoveAll(l.workingPath)
+	}
+
+	return nil
+}
+
+func getSuiteDir(suiteFile string) (string, error) {
+	path, err := filepath.Abs(suiteFile)
+	if err != nil {
+		return "", err
+	}
+	stat, err := os.Stat(path)
+	if err != nil {
+		return "", err
+	}
+
+	if stat.IsDir() {
+		return suiteFile, nil
+	}
+
+	return filepath.Dir(suiteFile), nil
+}
+
+func runSimpleCommand(cmd *exec.Cmd, dir string) error {
+	cmd.Dir = dir
+	logp.Info("Running %s in %s", cmd, dir)
+	output, err := cmd.CombinedOutput()
+	logp.Info("Ran %s got %s", cmd, string(output))
+	return err
+}
diff --git a/x-pack/heartbeat/monitors/browser/source/source.go b/x-pack/heartbeat/monitors/browser/source/source.go
index 9286d162cf3..a0f21f1e621 100644
--- a/x-pack/heartbeat/monitors/browser/source/source.go
+++ b/x-pack/heartbeat/monitors/browser/source/source.go
@@ -38,6 +38,7 @@ func (s *Source) Validate() error {
 type ISource interface {
 	Fetch() error
 	Workdir() string
+	Close() error
 }
 
 type BaseSource struct {
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
index 5e67a190b7d..59a44bbe59a 100644
--- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go
@@ -254,31 +254,6 @@ func jsonToSynthEvent(bytes []byte, text string) (res *SynthEvent, err error) {
 	return
 }
 
-func getSuiteDir(suiteFile string) (string, error) {
-	path, err := filepath.Abs(suiteFile)
-	if err != nil {
-		return "", err
-	}
-	stat, err := os.Stat(path)
-	if err != nil {
-		return "", err
-	}
-
-	if stat.IsDir() {
-		return suiteFile, nil
-	}
-
-	return filepath.Dir(suiteFile), nil
-}
-
-func runSimpleCommand(cmd *exec.Cmd, dir string) error {
-	cmd.Dir = dir
-	logp.Info("Running %s in %s", cmd, dir)
-	output, err := cmd.CombinedOutput()
-	logp.Info("Ran %s got %s", cmd, string(output))
-	return err
-}
-
 // getNpmRoot gets the closest ancestor path that contains package.json.
 func getNpmRoot(path string) (string, error) {
 	return getNpmRootIn(path, path)

From c57c81ecf4bec0bb1b483c4a42c73bc1f6ddbbfa Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Fri, 15 Jan 2021 18:04:29 -0600
Subject: [PATCH 22/41] Improve structure of monitor plugins

---
 heartbeat/monitors/active/http/http.go        | 27 +++---
 heartbeat/monitors/active/icmp/icmp.go        | 25 ++---
 heartbeat/monitors/active/icmp/icmp_test.go   |  2 +-
 heartbeat/monitors/active/tcp/tcp.go          | 16 +--
 heartbeat/monitors/factory.go                 |  5 +-
 heartbeat/monitors/mocks_test.go              | 20 ++--
 heartbeat/monitors/monitor.go                 | 48 +++++----
 heartbeat/monitors/{ => plugin}/plugin.go     | 97 +++++++++----------
 heartbeat/monitors/{ => plugin}/regrecord.go  | 40 ++++----
 x-pack/heartbeat/monitors/browser/browser.go  | 17 ++--
 .../monitors/browser/suite_runner.go          |  8 ++
 11 files changed, 163 insertions(+), 142 deletions(-)
 rename heartbeat/monitors/{ => plugin}/plugin.go (59%)
 rename heartbeat/monitors/{ => plugin}/regrecord.go (70%)

diff --git a/heartbeat/monitors/active/http/http.go b/heartbeat/monitors/active/http/http.go
index f14e9c90842..c01b83f44ee 100644
--- a/heartbeat/monitors/active/http/http.go
+++ b/heartbeat/monitors/active/http/http.go
@@ -22,7 +22,8 @@ import (
 	"net/http"
 	"net/url"
 
-	"github.com/elastic/beats/v7/heartbeat/monitors"
+	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
+
 	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
 	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers"
 	"github.com/elastic/beats/v7/libbeat/common"
@@ -32,7 +33,7 @@ import (
 )
 
 func init() {
-	monitors.RegisterActive("http", create, "synthetics/http")
+	plugin.Register("http", create, "synthetics/http")
 }
 
 var debugf = logp.MakeDebug("http")
@@ -41,15 +42,15 @@ var debugf = logp.MakeDebug("http")
 func create(
 	name string,
 	cfg *common.Config,
-) (js []jobs.Job, endpoints int, err error) {
+) (p plugin.Plugin, err error) {
 	config := defaultConfig
 	if err := cfg.Unpack(&config); err != nil {
-		return nil, 0, err
+		return plugin.Plugin{}, err
 	}
 
 	tls, err := tlscommon.LoadTLSConfig(config.TLS)
 	if err != nil {
-		return nil, 0, err
+		return plugin.Plugin{}, err
 	}
 
 	var body []byte
@@ -60,13 +61,13 @@ func create(
 		compression := config.Check.Request.Compression
 		enc, err = getContentEncoder(compression.Type, compression.Level)
 		if err != nil {
-			return nil, 0, err
+			return plugin.Plugin{}, err
 		}
 
 		buf := bytes.NewBuffer(nil)
 		err = enc.Encode(buf, bytes.NewBufferString(config.Check.Request.SendBody))
 		if err != nil {
-			return nil, 0, err
+			return plugin.Plugin{}, err
 		}
 
 		body = buf.Bytes()
@@ -74,7 +75,7 @@ func create(
 
 	validator, err := makeValidateResponse(&config.Check.Response)
 	if err != nil {
-		return nil, 0, err
+		return plugin.Plugin{}, err
 	}
 
 	// Determine whether we're using a proxy or not and then use that to figure out how to
@@ -86,7 +87,7 @@ func create(
 	if config.ProxyURL != "" || config.MaxRedirects > 0 {
 		transport, err := newRoundTripper(&config, tls)
 		if err != nil {
-			return nil, 0, err
+			return plugin.Plugin{}, err
 		}
 
 		makeJob = func(urlStr string) (jobs.Job, error) {
@@ -98,16 +99,16 @@ func create(
 		}
 	}
 
-	js = make([]jobs.Job, len(config.Hosts))
+	js := make([]jobs.Job, len(config.Hosts))
 	for i, urlStr := range config.Hosts {
 		u, _ := url.Parse(urlStr)
 		if err != nil {
-			return nil, 0, err
+			return plugin.Plugin{}, err
 		}
 
 		job, err := makeJob(urlStr)
 		if err != nil {
-			return nil, 0, err
+			return plugin.Plugin{}, err
 		}
 
 		// Assign any execution errors to the error field and
@@ -115,7 +116,7 @@ func create(
 		js[i] = wrappers.WithURLField(u, job)
 	}
 
-	return js, len(config.Hosts), nil
+	return plugin.Plugin{js, nil, len(config.Hosts)}, nil
 }
 
 func newRoundTripper(config *Config, tls *tlscommon.TLSConfig) (*http.Transport, error) {
diff --git a/heartbeat/monitors/active/icmp/icmp.go b/heartbeat/monitors/active/icmp/icmp.go
index 7feb76e6274..4b57c103deb 100644
--- a/heartbeat/monitors/active/icmp/icmp.go
+++ b/heartbeat/monitors/active/icmp/icmp.go
@@ -22,6 +22,8 @@ import (
 	"net"
 	"net/url"
 
+	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
+
 	"github.com/elastic/beats/v7/heartbeat/eventext"
 	"github.com/elastic/beats/v7/heartbeat/look"
 	"github.com/elastic/beats/v7/heartbeat/monitors"
@@ -35,29 +37,29 @@ import (
 var debugf = logp.MakeDebug("icmp")
 
 func init() {
-	monitors.RegisterActive("icmp", create, "synthetics/icmp")
+	plugin.Register("icmp", create, "synthetics/icmp")
 }
 
 func create(
 	name string,
 	commonConfig *common.Config,
-) (jobs []jobs.Job, endpoints int, err error) {
+) (p plugin.Plugin, err error) {
 	loop, err := getStdLoop()
 	if err != nil {
 		logp.Warn("Failed to initialize ICMP loop %v", err)
-		return nil, 0, err
+		return plugin.Plugin{}, err
 	}
 
 	config := DefaultConfig
 	if err := commonConfig.Unpack(&config); err != nil {
-		return nil, 0, err
+		return plugin.Plugin{}, err
 	}
 
 	jf, err := newJobFactory(config, monitors.NewStdResolver(), loop)
 	if err != nil {
-		return nil, 0, err
+		return plugin.Plugin{}, err
 	}
-	return jf.makeJobs()
+	return jf.makePlugin()
 
 }
 
@@ -88,29 +90,30 @@ func (jf *jobFactory) checkConfig() error {
 	return nil
 }
 
-func (jf *jobFactory) makeJobs() (j []jobs.Job, endpoints int, err error) {
+func (jf *jobFactory) makePlugin() (plugin2 plugin.Plugin, err error) {
 	if err := jf.loop.checkNetworkMode(jf.ipVersion); err != nil {
-		return nil, 0, err
+		return plugin.Plugin{}, err
 	}
 
 	pingFactory := jf.pingIPFactory(&jf.config)
 
+	var j []jobs.Job
 	for _, host := range jf.config.Hosts {
 		job, err := monitors.MakeByHostJob(host, jf.config.Mode, monitors.NewStdResolver(), pingFactory)
 
 		if err != nil {
-			return nil, 0, err
+			return plugin.Plugin{}, err
 		}
 
 		u, err := url.Parse(fmt.Sprintf("icmp://%s", host))
 		if err != nil {
-			return nil, 0, err
+			return plugin.Plugin{}, err
 		}
 
 		j = append(j, wrappers.WithURLField(u, job))
 	}
 
-	return j, len(jf.config.Hosts), nil
+	return plugin.Plugin{j, nil, len(jf.config.Hosts)}, nil
 }
 
 func (jf *jobFactory) pingIPFactory(config *Config) func(*net.IPAddr) jobs.Job {
diff --git a/heartbeat/monitors/active/icmp/icmp_test.go b/heartbeat/monitors/active/icmp/icmp_test.go
index 955520b81ba..776990e5499 100644
--- a/heartbeat/monitors/active/icmp/icmp_test.go
+++ b/heartbeat/monitors/active/icmp/icmp_test.go
@@ -65,7 +65,7 @@ func execTestICMPCheck(t *testing.T, cfg Config) (mockLoop, *beat.Event) {
 	tl := mockLoop{pingRtt: time.Microsecond * 1000, pingRequests: 1}
 	jf, err := newJobFactory(cfg, monitors.NewStdResolver(), tl)
 	require.NoError(t, err)
-	j, endpoints, err := jf.makeJobs()
+	j, endpoints, err := jf.makePlugin()
 	require.Len(t, j, 1)
 	require.Equal(t, 1, endpoints)
 	e := &beat.Event{}
diff --git a/heartbeat/monitors/active/tcp/tcp.go b/heartbeat/monitors/active/tcp/tcp.go
index f9d3499b681..dda96b85088 100644
--- a/heartbeat/monitors/active/tcp/tcp.go
+++ b/heartbeat/monitors/active/tcp/tcp.go
@@ -23,6 +23,8 @@ import (
 	"net/url"
 	"time"
 
+	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
+
 	"github.com/elastic/beats/v7/heartbeat/eventext"
 	"github.com/elastic/beats/v7/heartbeat/look"
 	"github.com/elastic/beats/v7/heartbeat/monitors"
@@ -39,7 +41,7 @@ import (
 )
 
 func init() {
-	monitors.RegisterActive("tcp", create, "synthetics/tcp")
+	plugin.Register("tcp", create, "synthetics/tcp")
 }
 
 var debugf = logp.MakeDebug("tcp")
@@ -47,7 +49,7 @@ var debugf = logp.MakeDebug("tcp")
 func create(
 	name string,
 	cfg *common.Config,
-) (jobs []jobs.Job, endpoints int, err error) {
+) (p plugin.Plugin, err error) {
 	return createWithResolver(cfg, monitors.NewStdResolver())
 }
 
@@ -56,18 +58,18 @@ func create(
 func createWithResolver(
 	cfg *common.Config,
 	resolver monitors.Resolver,
-) (jobs []jobs.Job, endpoints int, err error) {
+) (p plugin.Plugin, err error) {
 	jc, err := newJobFactory(cfg, resolver)
 	if err != nil {
-		return nil, 0, err
+		return plugin.Plugin{}, err
 	}
 
-	jobs, err = jc.makeJobs()
+	js, err := jc.makeJobs()
 	if err != nil {
-		return nil, 0, err
+		return plugin.Plugin{}, err
 	}
 
-	return jobs, len(jc.endpoints), nil
+	return plugin.Plugin{js, nil, len(jc.endpoints)}, nil
 }
 
 // jobFactory is where most of the logic here lives. It provides a common context around
diff --git a/heartbeat/monitors/factory.go b/heartbeat/monitors/factory.go
index 10d039d0830..8212f0002bd 100644
--- a/heartbeat/monitors/factory.go
+++ b/heartbeat/monitors/factory.go
@@ -18,6 +18,7 @@
 package monitors
 
 import (
+	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
 	"github.com/elastic/beats/v7/heartbeat/scheduler"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/cfgfile"
@@ -68,13 +69,13 @@ func (f *RunnerFactory) Create(p beat.Pipeline, c *common.Config) (cfgfile.Runne
 	}
 
 	p = pipetool.WithClientConfigEdit(p, configEditor)
-	monitor, err := newMonitor(c, globalPluginsReg, p, f.sched, f.allowWatches)
+	monitor, err := newMonitor(c, plugin.GlobalPluginsReg, p, f.sched, f.allowWatches)
 	return monitor, err
 }
 
 // CheckConfig checks to see if the given monitor config is valid.
 func (f *RunnerFactory) CheckConfig(config *common.Config) error {
-	return checkMonitorConfig(config, globalPluginsReg, f.allowWatches)
+	return checkMonitorConfig(config, plugin.GlobalPluginsReg, f.allowWatches)
 }
 
 func newCommonPublishConfigs(info beat.Info, cfg *common.Config) (pipetool.ConfigEditor, error) {
diff --git a/heartbeat/monitors/mocks_test.go b/heartbeat/monitors/mocks_test.go
index 970fdd01f12..16fd5cd15fe 100644
--- a/heartbeat/monitors/mocks_test.go
+++ b/heartbeat/monitors/mocks_test.go
@@ -23,6 +23,8 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
+
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/v7/heartbeat/eventext"
@@ -134,30 +136,30 @@ func createMockJob(name string, cfg *common.Config) ([]jobs.Job, error) {
 	return []jobs.Job{j}, nil
 }
 
-func mockPluginBuilder() pluginBuilder {
+func mockPluginBuilder() plugin.PluginFactory {
 	reg := monitoring.NewRegistry()
 
-	return pluginBuilder{
+	return plugin.PluginFactory{
 		"test",
 		[]string{"testAlias"},
-		func(s string, config *common.Config) ([]jobs.Job, int, error) {
+		func(s string, config *common.Config) (plugin.Plugin, error) {
 			// Declare a real config block with a required attr so we can see what happens when it doesn't work
 			unpacked := struct {
 				URLs []string `config:"urls" validate:"required"`
 			}{}
 			err := config.Unpack(&unpacked)
 			if err != nil {
-				return nil, 0, err
+				return plugin.Plugin{}, err
 			}
 			c := common.Config{}
 			j, err := createMockJob("test", &c)
-			return j, 1, err
-		}, newPluginCountersRecorder("test", reg)}
+			return plugin.Plugin{j, nil, 1}, err
+		}, plugin.NewPluginCountersRecorder("test", reg)}
 }
 
-func mockPluginsReg() *pluginsReg {
-	reg := newPluginsReg()
-	reg.add(mockPluginBuilder())
+func mockPluginsReg() *plugin.PluginsReg {
+	reg := plugin.NewPluginsReg()
+	reg.Add(mockPluginBuilder())
 	return reg
 }
 
diff --git a/heartbeat/monitors/monitor.go b/heartbeat/monitors/monitor.go
index e9fe867e413..e17518975aa 100644
--- a/heartbeat/monitors/monitor.go
+++ b/heartbeat/monitors/monitor.go
@@ -23,6 +23,8 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
+
 	"github.com/elastic/beats/v7/heartbeat/monitors/stdfields"
 
 	"github.com/mitchellh/hashstructure"
@@ -43,7 +45,7 @@ type Monitor struct {
 	stdFields      stdfields.StdMonitorFields
 	pluginName     string
 	config         *common.Config
-	registrar      *pluginsReg
+	registrar      *plugin.PluginsReg
 	uniqueName     string
 	scheduler      *scheduler.Scheduler
 	configuredJobs []*configuredJob
@@ -53,6 +55,7 @@ type Monitor struct {
 	// internalsMtx is used to synchronize access to critical
 	// internal datastructures
 	internalsMtx sync.Mutex
+	close        func() error
 
 	// Watch related fields
 	watchPollTasks []*configuredJob
@@ -62,7 +65,7 @@ type Monitor struct {
 
 	// stats is the countersRecorder used to record lifecycle events
 	// for global metrics + telemetry
-	stats registryRecorder
+	stats plugin.RegistryRecorder
 }
 
 // String prints a description of the monitor in a threadsafe way. It is important that this use threadsafe
@@ -71,7 +74,7 @@ func (m *Monitor) String() string {
 	return fmt.Sprintf("Monitor<pluginName: %s, enabled: %t>", m.stdFields.Name, m.enabled)
 }
 
-func checkMonitorConfig(config *common.Config, registrar *pluginsReg, allowWatches bool) error {
+func checkMonitorConfig(config *common.Config, registrar *plugin.PluginsReg, allowWatches bool) error {
 	m, err := newMonitor(config, registrar, nil, nil, allowWatches)
 	if m != nil {
 		m.Stop() // Stop the monitor to free up the ID from uniqueness checks
@@ -96,7 +99,7 @@ func (e ErrDuplicateMonitorID) Error() string {
 // newMonitor Creates a new monitor, without leaking resources in the event of an error.
 func newMonitor(
 	config *common.Config,
-	registrar *pluginsReg,
+	registrar *plugin.PluginsReg,
 	pipelineConnector beat.PipelineConnector,
 	scheduler *scheduler.Scheduler,
 	allowWatches bool,
@@ -112,7 +115,7 @@ func newMonitor(
 // error without freeing monitor resources. m.Stop() must always be called on a non-nil monitor to free resources.
 func newMonitorUnsafe(
 	config *common.Config,
-	registrar *pluginsReg,
+	registrar *plugin.PluginsReg,
 	pipelineConnector beat.PipelineConnector,
 	scheduler *scheduler.Scheduler,
 	allowWatches bool,
@@ -125,21 +128,21 @@ func newMonitorUnsafe(
 		return nil, err
 	}
 
-	monitorPlugin, found := registrar.get(standardFields.Type)
+	pluginFactory, found := registrar.Get(standardFields.Type)
 	if !found {
-		return nil, fmt.Errorf("monitor type %v does not exist, valid types are %v", standardFields.Type, registrar.monitorNames())
+		return nil, fmt.Errorf("monitor type %v does not exist, valid types are %v", standardFields.Type, registrar.MonitorNames())
 	}
 
 	m := &Monitor{
 		stdFields:         standardFields,
-		pluginName:        monitorPlugin.name,
+		pluginName:        pluginFactory.Name,
 		scheduler:         scheduler,
 		configuredJobs:    []*configuredJob{},
 		pipelineConnector: pipelineConnector,
 		watchPollTasks:    []*configuredJob{},
 		internalsMtx:      sync.Mutex{},
 		config:            config,
-		stats:             monitorPlugin.stats,
+		stats:             pluginFactory.Stats,
 	}
 
 	if m.stdFields.ID != "" {
@@ -156,9 +159,10 @@ func newMonitorUnsafe(
 		m.stdFields.ID = fmt.Sprintf("auto-%s-%#X", m.stdFields.Type, hash)
 	}
 
-	rawJobs, endpoints, err := monitorPlugin.create(config)
-	wrappedJobs := wrappers.WrapCommon(rawJobs, m.stdFields)
-	m.endpoints = endpoints
+	p, err := pluginFactory.Create(config)
+	m.close = p.Close
+	wrappedJobs := wrappers.WrapCommon(p.Jobs, m.stdFields)
+	m.endpoints = p.Endpoints
 
 	if err != nil {
 		return m, fmt.Errorf("job err %v", err)
@@ -169,7 +173,7 @@ func newMonitorUnsafe(
 		return m, err
 	}
 
-	err = m.makeWatchTasks(monitorPlugin)
+	err = m.makeWatchTasks(pluginFactory)
 	if err != nil {
 		return m, err
 	}
@@ -225,7 +229,7 @@ func (m *Monitor) makeTasks(config *common.Config, jobs []jobs.Job) ([]*configur
 	return mTasks, nil
 }
 
-func (m *Monitor) makeWatchTasks(monitorPlugin pluginBuilder) error {
+func (m *Monitor) makeWatchTasks(pluginFactory plugin.PluginFactory) error {
 	watchCfg := watcher.DefaultWatchConfig
 	err := m.config.Unpack(&watchCfg)
 	if err != nil {
@@ -257,13 +261,14 @@ func (m *Monitor) makeWatchTasks(monitorPlugin pluginBuilder) error {
 					return
 				}
 
-				watchJobs, endpoints, err := monitorPlugin.create(merged)
-				m.endpoints = endpoints
+				p, err := pluginFactory.Create(merged)
+				m.close = p.Close
+				m.endpoints = p.Endpoints
 				if err != nil {
 					logp.Err("Could not create job from watch file: %v", err)
 				}
 
-				watchTasks, err := m.makeTasks(merged, watchJobs)
+				watchTasks, err := m.makeTasks(merged, p.Jobs)
 				if err != nil {
 					logp.Err("Could not make configuredJob for config: %v", err)
 					return
@@ -305,7 +310,7 @@ func (m *Monitor) Start() {
 		t.Start()
 	}
 
-	m.stats.startMonitor(int64(m.endpoints))
+	m.stats.StartMonitor(int64(m.endpoints))
 }
 
 // Stop stops the Monitor's execution in its configured scheduler.
@@ -323,7 +328,12 @@ func (m *Monitor) Stop() {
 		t.Stop()
 	}
 
-	m.stats.stopMonitor(int64(m.endpoints))
+	err := m.close()
+	if err != nil {
+		logp.Error(fmt.Errorf("error closing monitor %s: %w", m.String(), err))
+	}
+
+	m.stats.StopMonitor(int64(m.endpoints))
 }
 
 func (m *Monitor) freeID() {
diff --git a/heartbeat/monitors/plugin.go b/heartbeat/monitors/plugin/plugin.go
similarity index 59%
rename from heartbeat/monitors/plugin.go
rename to heartbeat/monitors/plugin/plugin.go
index 7f972c2f697..73335ca5e29 100644
--- a/heartbeat/monitors/plugin.go
+++ b/heartbeat/monitors/plugin/plugin.go
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package monitors
+package plugin
 
 import (
 	"errors"
@@ -29,11 +29,19 @@ import (
 	"github.com/elastic/beats/v7/libbeat/plugin"
 )
 
-type pluginBuilder struct {
-	name    string
-	aliases []string
-	builder PluginBuilder
-	stats   registryRecorder
+type PluginFactory struct {
+	Name    string
+	Aliases []string
+	Builder PluginFactoryCreate
+	Stats   RegistryRecorder
+}
+
+type PluginFactoryCreate func(string, *common.Config) (p Plugin, err error)
+
+type Plugin struct {
+	Jobs      []jobs.Job
+	Close     func() error
+	Endpoints int
 }
 
 var pluginKey = "heartbeat.monitor"
@@ -41,13 +49,13 @@ var pluginKey = "heartbeat.monitor"
 // stateGlobalRecorder records statistics across all plugin types
 var stateGlobalRecorder = newRootGaugeRecorder(hbregistry.TelemetryRegistry)
 
-func statsForPlugin(pluginName string) registryRecorder {
-	return multiRegistryRecorder{
-		recorders: []registryRecorder{
+func statsForPlugin(pluginName string) RegistryRecorder {
+	return MultiRegistryRecorder{
+		recorders: []RegistryRecorder{
 			// state (telemetry)
 			newPluginGaugeRecorder(pluginName, hbregistry.TelemetryRegistry),
 			// Record global monitors / endpoints count
-			newPluginCountersRecorder(pluginName, hbregistry.StatsRegistry),
+			NewPluginCountersRecorder(pluginName, hbregistry.StatsRegistry),
 			// When stats for this plugin are updated, update the global stats as well
 			stateGlobalRecorder,
 		},
@@ -56,20 +64,16 @@ func statsForPlugin(pluginName string) registryRecorder {
 
 func init() {
 	plugin.MustRegisterLoader(pluginKey, func(ifc interface{}) error {
-		p, ok := ifc.(pluginBuilder)
+		p, ok := ifc.(PluginFactory)
 		if !ok {
 			return errors.New("plugin does not match monitor plugin type")
 		}
 
-		stats := statsForPlugin(p.name)
-		return globalPluginsReg.register(pluginBuilder{p.name, p.aliases, p.builder, stats})
+		stats := statsForPlugin(p.Name)
+		return GlobalPluginsReg.Register(PluginFactory{p.Name, p.Aliases, p.Builder, stats})
 	})
 }
 
-// PluginBuilder is the signature of functions used to build active
-// monitorStarts
-type PluginBuilder func(string, *common.Config) (jobs []jobs.Job, endpoints int, err error)
-
 // Type represents whether a plugin is active or passive.
 type Type uint8
 
@@ -81,40 +85,40 @@ const (
 )
 
 // globalPluginsReg maintains the canonical list of valid Heartbeat monitorStarts at runtime.
-var globalPluginsReg = newPluginsReg()
+var GlobalPluginsReg = NewPluginsReg()
 
-type pluginsReg struct {
-	monitors map[string]pluginBuilder
+type PluginsReg struct {
+	monitors map[string]PluginFactory
 }
 
-func newPluginsReg() *pluginsReg {
-	return &pluginsReg{
-		monitors: map[string]pluginBuilder{},
+func NewPluginsReg() *PluginsReg {
+	return &PluginsReg{
+		monitors: map[string]PluginFactory{},
 	}
 }
 
-// RegisterActive registers a new active (as opposed to passive) monitor.
-func RegisterActive(name string, builder PluginBuilder, aliases ...string) {
+// Register registers a new active (as opposed to passive) monitor.
+func Register(name string, builder PluginFactoryCreate, aliases ...string) {
 	stats := statsForPlugin(name)
-	if err := globalPluginsReg.add(pluginBuilder{name, aliases, builder, stats}); err != nil {
+	if err := GlobalPluginsReg.Add(PluginFactory{name, aliases, builder, stats}); err != nil {
 		panic(err)
 	}
 }
 
 // ErrPluginAlreadyExists is returned when there is an attempt to register two plugins
 // with the same pluginName.
-type ErrPluginAlreadyExists pluginBuilder
+type ErrPluginAlreadyExists PluginFactory
 
 func (m ErrPluginAlreadyExists) Error() string {
-	return fmt.Sprintf("monitor plugin named '%s' with aliases %v already exists", m.name, m.aliases)
+	return fmt.Sprintf("monitor plugin named '%s' with Aliases %v already exists", m.Name, m.Aliases)
 }
 
-func (r *pluginsReg) add(plugin pluginBuilder) error {
-	if _, exists := r.monitors[plugin.name]; exists {
+func (r *PluginsReg) Add(plugin PluginFactory) error {
+	if _, exists := r.monitors[plugin.Name]; exists {
 		return ErrPluginAlreadyExists(plugin)
 	}
-	r.monitors[plugin.name] = plugin
-	for _, alias := range plugin.aliases {
+	r.monitors[plugin.Name] = plugin
+	for _, alias := range plugin.Aliases {
 		if _, exists := r.monitors[alias]; exists {
 			return ErrPluginAlreadyExists(plugin)
 		}
@@ -123,22 +127,22 @@ func (r *pluginsReg) add(plugin pluginBuilder) error {
 	return nil
 }
 
-func (r *pluginsReg) register(plugin pluginBuilder) error {
-	if _, found := r.monitors[plugin.name]; found {
-		return fmt.Errorf("monitor type %v already exists", plugin.name)
+func (r *PluginsReg) Register(plugin PluginFactory) error {
+	if _, found := r.monitors[plugin.Name]; found {
+		return fmt.Errorf("monitor type %v already exists", plugin.Name)
 	}
 
-	r.monitors[plugin.name] = plugin
+	r.monitors[plugin.Name] = plugin
 
 	return nil
 }
 
-func (r *pluginsReg) get(name string) (pluginBuilder, bool) {
+func (r *PluginsReg) Get(name string) (PluginFactory, bool) {
 	e, found := r.monitors[name]
 	return e, found
 }
 
-func (r *pluginsReg) String() string {
+func (r *PluginsReg) String() string {
 	var monitors []string
 	for m := range r.monitors {
 		monitors = append(monitors, m)
@@ -148,7 +152,7 @@ func (r *pluginsReg) String() string {
 	return fmt.Sprintf("globalPluginsReg, monitor: %v",
 		strings.Join(monitors, ", "))
 }
-func (r *pluginsReg) monitorNames() []string {
+func (r *PluginsReg) MonitorNames() []string {
 	names := make([]string, 0, len(r.monitors))
 	for k := range r.monitors {
 		names = append(names, k)
@@ -156,17 +160,6 @@ func (r *pluginsReg) monitorNames() []string {
 	return names
 }
 
-func (e *pluginBuilder) create(cfg *common.Config) (jobs []jobs.Job, endpoints int, err error) {
-	return e.builder(e.name, cfg)
-}
-
-func (t Type) String() string {
-	switch t {
-	case ActiveMonitor:
-		return "active"
-	case PassiveMonitor:
-		return "passive"
-	default:
-		return "unknown type"
-	}
+func (e *PluginFactory) Create(cfg *common.Config) (p Plugin, err error) {
+	return e.Builder(e.Name, cfg)
 }
diff --git a/heartbeat/monitors/regrecord.go b/heartbeat/monitors/plugin/regrecord.go
similarity index 70%
rename from heartbeat/monitors/regrecord.go
rename to heartbeat/monitors/plugin/regrecord.go
index a7c630deaa1..49d327c812b 100644
--- a/heartbeat/monitors/regrecord.go
+++ b/heartbeat/monitors/plugin/regrecord.go
@@ -15,46 +15,46 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package monitors
+package plugin
 
 import (
 	"github.com/elastic/beats/v7/libbeat/monitoring"
 )
 
-type registryRecorder interface {
-	startMonitor(endpoints int64)
-	stopMonitor(endpoints int64)
+type RegistryRecorder interface {
+	StartMonitor(endpoints int64)
+	StopMonitor(endpoints int64)
 }
 
-// multiRegistryRecorder composes multiple statsRecorders.
-type multiRegistryRecorder struct {
-	recorders []registryRecorder
+// MultiRegistryRecorder composes multiple statsRecorders.
+type MultiRegistryRecorder struct {
+	recorders []RegistryRecorder
 }
 
-func (mr multiRegistryRecorder) startMonitor(endpoints int64) {
+func (mr MultiRegistryRecorder) StartMonitor(endpoints int64) {
 	for _, recorder := range mr.recorders {
-		recorder.startMonitor(endpoints)
+		recorder.StartMonitor(endpoints)
 	}
 }
 
-func (mr multiRegistryRecorder) stopMonitor(endpoints int64) {
+func (mr MultiRegistryRecorder) StopMonitor(endpoints int64) {
 	for _, recorder := range mr.recorders {
-		recorder.stopMonitor(endpoints)
+		recorder.StopMonitor(endpoints)
 	}
 }
 
 // countersRecorder is used to record start/stop events for a single monitor/plugin
 // to a single registry as counters.
-type countersRecorder struct {
+type CountersRecorder struct {
 	monitorStarts  *monitoring.Int
 	monitorStops   *monitoring.Int
 	endpointStarts *monitoring.Int
 	endpointStops  *monitoring.Int
 }
 
-func newPluginCountersRecorder(pluginName string, rootRegistry *monitoring.Registry) registryRecorder {
+func NewPluginCountersRecorder(pluginName string, rootRegistry *monitoring.Registry) RegistryRecorder {
 	pluginRegistry := rootRegistry.NewRegistry(pluginName)
-	return countersRecorder{
+	return CountersRecorder{
 		monitoring.NewInt(pluginRegistry, "monitor_starts"),
 		monitoring.NewInt(pluginRegistry, "monitor_stops"),
 		monitoring.NewInt(pluginRegistry, "endpoint_starts"),
@@ -62,12 +62,12 @@ func newPluginCountersRecorder(pluginName string, rootRegistry *monitoring.Regis
 	}
 }
 
-func (r countersRecorder) startMonitor(endpoints int64) {
+func (r CountersRecorder) StartMonitor(endpoints int64) {
 	r.monitorStarts.Inc()
 	r.endpointStarts.Add(endpoints)
 }
 
-func (r countersRecorder) stopMonitor(endpoints int64) {
+func (r CountersRecorder) StopMonitor(endpoints int64) {
 	r.monitorStops.Inc()
 	r.endpointStops.Add(endpoints)
 }
@@ -79,24 +79,24 @@ type gaugeRecorder struct {
 	endpoints *monitoring.Int
 }
 
-func newRootGaugeRecorder(r *monitoring.Registry) registryRecorder {
+func newRootGaugeRecorder(r *monitoring.Registry) RegistryRecorder {
 	return gaugeRecorder{
 		monitoring.NewInt(r, "monitors"),
 		monitoring.NewInt(r, "endpoints"),
 	}
 }
 
-func newPluginGaugeRecorder(pluginName string, rootRegistry *monitoring.Registry) registryRecorder {
+func newPluginGaugeRecorder(pluginName string, rootRegistry *monitoring.Registry) RegistryRecorder {
 	pluginRegistry := rootRegistry.NewRegistry(pluginName)
 	return newRootGaugeRecorder(pluginRegistry)
 }
 
-func (r gaugeRecorder) startMonitor(endpoints int64) {
+func (r gaugeRecorder) StartMonitor(endpoints int64) {
 	r.monitors.Inc()
 	r.endpoints.Add(endpoints)
 }
 
-func (r gaugeRecorder) stopMonitor(endpoints int64) {
+func (r gaugeRecorder) StopMonitor(endpoints int64) {
 	r.monitors.Dec()
 	r.endpoints.Sub(endpoints)
 }
diff --git a/x-pack/heartbeat/monitors/browser/browser.go b/x-pack/heartbeat/monitors/browser/browser.go
index c2ce7708569..756760bd4a1 100644
--- a/x-pack/heartbeat/monitors/browser/browser.go
+++ b/x-pack/heartbeat/monitors/browser/browser.go
@@ -11,9 +11,10 @@ import (
 	"os/user"
 	"sync"
 
+	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
+
 	"github.com/elastic/beats/v7/libbeat/beat"
 
-	"github.com/elastic/beats/v7/heartbeat/monitors"
 	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
@@ -21,19 +22,19 @@ import (
 )
 
 func init() {
-	monitors.RegisterActive("browser", create, "synthetic", "synthetics/synthetic")
+	plugin.Register("browser", create, "synthetic", "synthetics/synthetic")
 }
 
 var showExperimentalOnce = sync.Once{}
 
 var NotSyntheticsCapableError = fmt.Errorf("synthetic monitors cannot be created outside the official elastic docker image")
 
-func create(name string, cfg *common.Config) (js []jobs.Job, endpoints int, err error) {
+func create(name string, cfg *common.Config) (p plugin.Plugin, err error) {
 	// We don't want users running synthetics in environments that don't have the required GUI libraries etc, so we check
 	// this flag. When we're ready to support the many possible configurations of systems outside the docker environment
 	// we can remove this check.
 	if os.Getenv("ELASTIC_SYNTHETICS_CAPABLE") != "true" {
-		return nil, 0, NotSyntheticsCapableError
+		return plugin.Plugin{}, NotSyntheticsCapableError
 	}
 
 	showExperimentalOnce.Do(func() {
@@ -42,15 +43,15 @@ func create(name string, cfg *common.Config) (js []jobs.Job, endpoints int, err
 
 	curUser, err := user.Current()
 	if err != nil {
-		return nil, 0, fmt.Errorf("could not determine current user for script monitor %w: ", err)
+		return plugin.Plugin{}, fmt.Errorf("could not determine current user for script monitor %w: ", err)
 	}
 	if curUser.Uid == "0" {
-		return nil, 0, fmt.Errorf("script monitors cannot be run as root! Current UID is %s", curUser.Uid)
+		return plugin.Plugin{}, fmt.Errorf("script monitors cannot be run as root! Current UID is %s", curUser.Uid)
 	}
 
 	ss, err := NewSuite(cfg)
 	if err != nil {
-		return nil, 0, err
+		return plugin.Plugin{}, err
 	}
 
 	var j jobs.Job
@@ -70,5 +71,5 @@ func create(name string, cfg *common.Config) (js []jobs.Job, endpoints int, err
 		}
 	}
 
-	return []jobs.Job{j}, 1, nil
+	return plugin.Plugin{[]jobs.Job{j}, ss.Close, 1}, nil
 }
diff --git a/x-pack/heartbeat/monitors/browser/suite_runner.go b/x-pack/heartbeat/monitors/browser/suite_runner.go
index 28beb1dd5f3..17308cdc7ba 100644
--- a/x-pack/heartbeat/monitors/browser/suite_runner.go
+++ b/x-pack/heartbeat/monitors/browser/suite_runner.go
@@ -55,3 +55,11 @@ func (s *SyntheticSuite) InlineSource() (string, bool) {
 func (s *SyntheticSuite) Params() map[string]interface{} {
 	return s.suiteCfg.Params
 }
+
+func (s *SyntheticSuite) Close() error {
+	if s.suiteCfg.Source.ActiveMemo != nil {
+		s.suiteCfg.Source.ActiveMemo.Close()
+	}
+
+	return nil
+}

From 9b9f6b16ed2d5f8ff6f142cc473ad7b9bd5f0c4a Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Fri, 15 Jan 2021 18:23:28 -0600
Subject: [PATCH 23/41] More updates

---
 heartbeat/beater/heartbeat.go                   | 17 +++++++++++++----
 heartbeat/monitors/monitor.go                   |  8 +++++---
 .../heartbeat/monitors/browser/suite_runner.go  |  1 +
 .../sample-synthetics-config/heartbeat.yml      |  2 +-
 4 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go
index e944798887e..e7ecc0adfe8 100644
--- a/heartbeat/beater/heartbeat.go
+++ b/heartbeat/beater/heartbeat.go
@@ -80,10 +80,11 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
 func (bt *Heartbeat) Run(b *beat.Beat) error {
 	logp.Info("heartbeat is running! Hit CTRL-C to stop it.")
 
-	err := bt.RunStaticMonitors(b)
+	stopStaticMonitors, err := bt.RunStaticMonitors(b)
 	if err != nil {
 		return err
 	}
+	defer stopStaticMonitors()
 
 	if b.Manager.Enabled() {
 		bt.RunCentralMgmtMonitors(b)
@@ -121,9 +122,10 @@ func (bt *Heartbeat) Run(b *beat.Beat) error {
 }
 
 // RunStaticMonitors runs the `heartbeat.monitors` portion of the yaml config if present.
-func (bt *Heartbeat) RunStaticMonitors(b *beat.Beat) error {
+func (bt *Heartbeat) RunStaticMonitors(b *beat.Beat) (stop func(), err error) {
 	factory := monitors.NewFactory(b.Info, bt.scheduler, true)
 
+	var runners []cfgfile.Runner
 	for _, cfg := range bt.config.Monitors {
 		created, err := factory.Create(b.Publisher, cfg)
 		if err != nil {
@@ -131,12 +133,19 @@ func (bt *Heartbeat) RunStaticMonitors(b *beat.Beat) error {
 				continue // don't stop loading monitors just because they're disabled
 			}
 
-			return errors.Wrap(err, "could not create monitor")
+			return nil, errors.Wrap(err, "could not create monitor")
 		}
 
 		created.Start()
+		runners = append(runners, created)
 	}
-	return nil
+
+	stop = func() {
+		for _, runner := range runners {
+			runner.Stop()
+		}
+	}
+	return stop, nil
 }
 
 // RunCentralMgmtMonitors loads any central management configured configs.
diff --git a/heartbeat/monitors/monitor.go b/heartbeat/monitors/monitor.go
index e17518975aa..263d85ca862 100644
--- a/heartbeat/monitors/monitor.go
+++ b/heartbeat/monitors/monitor.go
@@ -328,9 +328,11 @@ func (m *Monitor) Stop() {
 		t.Stop()
 	}
 
-	err := m.close()
-	if err != nil {
-		logp.Error(fmt.Errorf("error closing monitor %s: %w", m.String(), err))
+	if m.close != nil {
+		err := m.close()
+		if err != nil {
+			logp.Error(fmt.Errorf("error closing monitor %s: %w", m.String(), err))
+		}
 	}
 
 	m.stats.StopMonitor(int64(m.endpoints))
diff --git a/x-pack/heartbeat/monitors/browser/suite_runner.go b/x-pack/heartbeat/monitors/browser/suite_runner.go
index 17308cdc7ba..61ca86fac40 100644
--- a/x-pack/heartbeat/monitors/browser/suite_runner.go
+++ b/x-pack/heartbeat/monitors/browser/suite_runner.go
@@ -57,6 +57,7 @@ func (s *SyntheticSuite) Params() map[string]interface{} {
 }
 
 func (s *SyntheticSuite) Close() error {
+	logp.Warn("INTERNAL CLOSE!!!")
 	if s.suiteCfg.Source.ActiveMemo != nil {
 		s.suiteCfg.Source.ActiveMemo.Close()
 	}
diff --git a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
index 7aab30acaff..a076851f5be 100644
--- a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
+++ b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
@@ -20,7 +20,7 @@ heartbeat.monitors:
   schedule: "@every 15s"
   name: Simple HTTP
 - type: browser
-  enabled: false
+  enabled: true
   id: my-monitor
   name: My Monitor
   source:

From 1c10922a43c29bcb9eb3a234f9189b4a3d927111 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Sat, 16 Jan 2021 07:58:06 -0600
Subject: [PATCH 24/41] Add some basic validation to sources

---
 .../monitors/browser/source/inline.go         | 19 ++++++++++++++++---
 .../monitors/browser/source/local.go          | 17 +++++++++++++++++
 .../monitors/browser/suite_runner.go          |  1 -
 .../sample-synthetics-config/heartbeat.yml    |  2 +-
 4 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/x-pack/heartbeat/monitors/browser/source/inline.go b/x-pack/heartbeat/monitors/browser/source/inline.go
index 5bcf43dc95a..d3ef2c452e5 100644
--- a/x-pack/heartbeat/monitors/browser/source/inline.go
+++ b/x-pack/heartbeat/monitors/browser/source/inline.go
@@ -4,19 +4,32 @@
 
 package source
 
+import (
+	"fmt"
+	"regexp"
+)
+
 type InlineSource struct {
 	Script string `config:"script"`
 	BaseSource
 }
 
-func (l *InlineSource) Fetch() (err error) {
+func (s *InlineSource) Validate() error {
+	if !regexp.MustCompile("\\S").MatchString(s.Script) {
+		return fmt.Errorf("no 'script' value specified for inline source")
+	}
+
+	return nil
+}
+
+func (s *InlineSource) Fetch() (err error) {
 	return nil
 }
 
-func (l *InlineSource) Workdir() string {
+func (s *InlineSource) Workdir() string {
 	return ""
 }
 
-func (l *InlineSource) Close() error {
+func (s *InlineSource) Close() error {
 	return nil
 }
diff --git a/x-pack/heartbeat/monitors/browser/source/local.go b/x-pack/heartbeat/monitors/browser/source/local.go
index b3c7d53022c..c1a95d31a6e 100644
--- a/x-pack/heartbeat/monitors/browser/source/local.go
+++ b/x-pack/heartbeat/monitors/browser/source/local.go
@@ -22,6 +22,23 @@ type LocalSource struct {
 	BaseSource
 }
 
+func (l *LocalSource) Validate() error {
+	if l.OrigPath == "" {
+		return fmt.Errorf("local source defined with no path specified")
+	}
+
+	s, err := os.Stat(l.OrigPath)
+	base := fmt.Sprintf("local source has invalid path '%s'", l.OrigPath)
+	if err != nil {
+		return fmt.Errorf("%s: %w", base, err)
+	}
+	if !s.IsDir() {
+		return fmt.Errorf("%s: path points to a non-directory", base)
+	}
+
+	return nil
+}
+
 func (l *LocalSource) Fetch() (err error) {
 	if l.workingPath != "" {
 		return nil
diff --git a/x-pack/heartbeat/monitors/browser/suite_runner.go b/x-pack/heartbeat/monitors/browser/suite_runner.go
index 61ca86fac40..17308cdc7ba 100644
--- a/x-pack/heartbeat/monitors/browser/suite_runner.go
+++ b/x-pack/heartbeat/monitors/browser/suite_runner.go
@@ -57,7 +57,6 @@ func (s *SyntheticSuite) Params() map[string]interface{} {
 }
 
 func (s *SyntheticSuite) Close() error {
-	logp.Warn("INTERNAL CLOSE!!!")
 	if s.suiteCfg.Source.ActiveMemo != nil {
 		s.suiteCfg.Source.ActiveMemo.Close()
 	}
diff --git a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
index a076851f5be..bebebdb4673 100644
--- a/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
+++ b/x-pack/heartbeat/sample-synthetics-config/heartbeat.yml
@@ -11,7 +11,7 @@ heartbeat.monitors:
   name: Todos Suite
   source:
     local:
-      path: "/home/andrewvc/projects/synthetics/examples/todos"
+      path: "/home/andrewvc/projects/synthetics/examples/todos/"
   schedule: '@every 1m'
 - type: http
   enabled: true

From faac3c9ba7a9506fe4b8ce22de7ec0865a3d8d25 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Sat, 16 Jan 2021 08:27:45 -0600
Subject: [PATCH 25/41] Test fixes

---
 heartbeat/monitors/active/http/http_test.go    | 18 +++++++++---------
 heartbeat/monitors/active/icmp/icmp_test.go    |  8 ++++----
 heartbeat/monitors/active/tcp/helpers_test.go  |  6 +++---
 heartbeat/monitors/active/tcp/tls_test.go      |  6 +++---
 .../monitors/browser/synthexec/enrich.go       | 11 ++++++-----
 5 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/heartbeat/monitors/active/http/http_test.go b/heartbeat/monitors/active/http/http_test.go
index 2e5a43656ad..1f813adb06c 100644
--- a/heartbeat/monitors/active/http/http_test.go
+++ b/heartbeat/monitors/active/http/http_test.go
@@ -78,17 +78,17 @@ func sendTLSRequest(t *testing.T, testURL string, useUrls bool, extraConfig map[
 	config, err := common.NewConfigFrom(configSrc)
 	require.NoError(t, err)
 
-	jobs, endpoints, err := create("tls", config)
+	p, err := create("tls", config)
 	require.NoError(t, err)
 
 	sched := schedule.MustParse("@every 1s")
-	job := wrappers.WrapCommon(jobs, stdfields.StdMonitorFields{ID: "tls", Type: "http", Schedule: sched, Timeout: 1})[0]
+	job := wrappers.WrapCommon(p.Jobs, stdfields.StdMonitorFields{ID: "tls", Type: "http", Schedule: sched, Timeout: 1})[0]
 
 	event := &beat.Event{}
 	_, err = job(event)
 	require.NoError(t, err)
 
-	require.Equal(t, 1, endpoints)
+	require.Equal(t, 1, p.Endpoints)
 
 	return event
 }
@@ -318,11 +318,11 @@ func TestLargeResponse(t *testing.T) {
 	config, err := common.NewConfigFrom(configSrc)
 	require.NoError(t, err)
 
-	jobs, _, err := create("largeresp", config)
+	p, err := create("largeresp", config)
 	require.NoError(t, err)
 
 	sched, _ := schedule.Parse("@every 1s")
-	job := wrappers.WrapCommon(jobs, stdfields.StdMonitorFields{ID: "test", Type: "http", Schedule: sched, Timeout: 1})[0]
+	job := wrappers.WrapCommon(p.Jobs, stdfields.StdMonitorFields{ID: "test", Type: "http", Schedule: sched, Timeout: 1})[0]
 
 	event := &beat.Event{}
 	_, err = job(event)
@@ -532,11 +532,11 @@ func TestRedirect(t *testing.T) {
 	config, err := common.NewConfigFrom(configSrc)
 	require.NoError(t, err)
 
-	jobs, _, err := create("redirect", config)
+	p, err := create("redirect", config)
 	require.NoError(t, err)
 
 	sched, _ := schedule.Parse("@every 1s")
-	job := wrappers.WrapCommon(jobs, stdfields.StdMonitorFields{ID: "test", Type: "http", Schedule: sched, Timeout: 1})[0]
+	job := wrappers.WrapCommon(p.Jobs, stdfields.StdMonitorFields{ID: "test", Type: "http", Schedule: sched, Timeout: 1})[0]
 
 	// Run this test multiple times since in the past we had an issue where the redirects
 	// list was added onto by each request. See https://github.com/elastic/beats/pull/15944
@@ -579,11 +579,11 @@ func TestNoHeaders(t *testing.T) {
 	config, err := common.NewConfigFrom(configSrc)
 	require.NoError(t, err)
 
-	jobs, _, err := create("http", config)
+	p, err := create("http", config)
 	require.NoError(t, err)
 
 	sched, _ := schedule.Parse("@every 1s")
-	job := wrappers.WrapCommon(jobs, stdfields.StdMonitorFields{ID: "test", Type: "http", Schedule: sched, Timeout: 1})[0]
+	job := wrappers.WrapCommon(p.Jobs, stdfields.StdMonitorFields{ID: "test", Type: "http", Schedule: sched, Timeout: 1})[0]
 
 	event := &beat.Event{}
 	_, err = job(event)
diff --git a/heartbeat/monitors/active/icmp/icmp_test.go b/heartbeat/monitors/active/icmp/icmp_test.go
index 776990e5499..2fd654648db 100644
--- a/heartbeat/monitors/active/icmp/icmp_test.go
+++ b/heartbeat/monitors/active/icmp/icmp_test.go
@@ -65,12 +65,12 @@ func execTestICMPCheck(t *testing.T, cfg Config) (mockLoop, *beat.Event) {
 	tl := mockLoop{pingRtt: time.Microsecond * 1000, pingRequests: 1}
 	jf, err := newJobFactory(cfg, monitors.NewStdResolver(), tl)
 	require.NoError(t, err)
-	j, endpoints, err := jf.makePlugin()
-	require.Len(t, j, 1)
-	require.Equal(t, 1, endpoints)
+	p, err := jf.makePlugin()
+	require.Len(t, p.Jobs, 1)
+	require.Equal(t, 1, p.Endpoints)
 	e := &beat.Event{}
 	sched, _ := schedule.Parse("@every 1s")
-	wrapped := wrappers.WrapCommon(j, stdfields.StdMonitorFields{ID: "test", Type: "icmp", Schedule: sched, Timeout: 1})
+	wrapped := wrappers.WrapCommon(p.Jobs, stdfields.StdMonitorFields{ID: "test", Type: "icmp", Schedule: sched, Timeout: 1})
 	wrapped[0](e)
 	return tl, e
 }
diff --git a/heartbeat/monitors/active/tcp/helpers_test.go b/heartbeat/monitors/active/tcp/helpers_test.go
index ea3a22b2888..b5c7aa077f3 100644
--- a/heartbeat/monitors/active/tcp/helpers_test.go
+++ b/heartbeat/monitors/active/tcp/helpers_test.go
@@ -38,17 +38,17 @@ func testTCPConfigCheck(t *testing.T, configMap common.MapStr, host string, port
 	config, err := common.NewConfigFrom(configMap)
 	require.NoError(t, err)
 
-	jobs, endpoints, err := create("tcp", config)
+	p, err := create("tcp", config)
 	require.NoError(t, err)
 
 	sched := schedule.MustParse("@every 1s")
-	job := wrappers.WrapCommon(jobs, stdfields.StdMonitorFields{ID: "test", Type: "tcp", Schedule: sched, Timeout: 1})[0]
+	job := wrappers.WrapCommon(p.Jobs, stdfields.StdMonitorFields{ID: "test", Type: "tcp", Schedule: sched, Timeout: 1})[0]
 
 	event := &beat.Event{}
 	_, err = job(event)
 	require.NoError(t, err)
 
-	require.Equal(t, 1, endpoints)
+	require.Equal(t, 1, p.Endpoints)
 
 	return event
 }
diff --git a/heartbeat/monitors/active/tcp/tls_test.go b/heartbeat/monitors/active/tcp/tls_test.go
index 88c539ee7e7..62477ed2f56 100644
--- a/heartbeat/monitors/active/tcp/tls_test.go
+++ b/heartbeat/monitors/active/tcp/tls_test.go
@@ -184,17 +184,17 @@ func testTLSTCPCheck(t *testing.T, host string, port uint16, certFileName string
 	})
 	require.NoError(t, err)
 
-	jobs, endpoints, err := createWithResolver(config, resolver)
+	p, err := createWithResolver(config, resolver)
 	require.NoError(t, err)
 
 	sched := schedule.MustParse("@every 1s")
-	job := wrappers.WrapCommon(jobs, stdfields.StdMonitorFields{ID: "test", Type: "tcp", Schedule: sched, Timeout: 1})[0]
+	job := wrappers.WrapCommon(p.Jobs, stdfields.StdMonitorFields{ID: "test", Type: "tcp", Schedule: sched, Timeout: 1})[0]
 
 	event := &beat.Event{}
 	_, err = job(event)
 	require.NoError(t, err)
 
-	require.Equal(t, 1, endpoints)
+	require.Equal(t, 1, p.Endpoints)
 
 	return event
 }
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/enrich.go b/x-pack/heartbeat/monitors/browser/synthexec/enrich.go
index cbee774cb3f..a246fd41243 100644
--- a/x-pack/heartbeat/monitors/browser/synthexec/enrich.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/enrich.go
@@ -60,7 +60,11 @@ func makeUuid() string {
 }
 
 func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent) error {
-	if se != nil && !se.Timestamp().IsZero() {
+	if se == nil {
+		return nil
+	}
+
+	if !se.Timestamp().IsZero() {
 		event.Timestamp = se.Timestamp()
 		// Record start and end so we can calculate journey duration accurately later
 		switch se.Type {
@@ -76,10 +80,7 @@ func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent) error {
 		event.Timestamp = time.Now()
 	}
 
-	// No more synthEvents? In this case this is the summary event
-	if se == nil {
-		return nil
-	}
+
 
 	eventext.MergeEventFields(event, common.MapStr{
 		"monitor": common.MapStr{

From 100dead853c235aeb7a3ea86c7c7e0c6c8e4319e Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Sat, 16 Jan 2021 08:30:41 -0600
Subject: [PATCH 26/41] Test fixes

---
 heartbeat/monitors/wrappers/monitors.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/heartbeat/monitors/wrappers/monitors.go b/heartbeat/monitors/wrappers/monitors.go
index ce62541813c..f7a2881aa9a 100644
--- a/heartbeat/monitors/wrappers/monitors.go
+++ b/heartbeat/monitors/wrappers/monitors.go
@@ -202,10 +202,6 @@ func makeAddSummary(monitorType string) jobs.JobWrapper {
 
 	return func(job jobs.Job) jobs.Job {
 		return func(event *beat.Event) ([]jobs.Job, error) {
-			if v, _ := event.GetValue("monitor.id"); v != state.monitorId {
-				resetState()
-			}
-
 			cont, jobErr := job(event)
 			state.mtx.Lock()
 			defer state.mtx.Unlock()

From 5c7d9818fb997ee6f170828386f9ab6a22a90d14 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Wed, 20 Jan 2021 16:42:00 -0600
Subject: [PATCH 27/41] Improve monitor tests

---
 heartbeat/monitors/mocks_test.go              | 56 ++++++++++++-------
 heartbeat/monitors/monitor.go                 |  6 +-
 heartbeat/monitors/monitor_test.go            | 31 ++++++----
 heartbeat/monitors/stdfields/stdfields.go     |  2 -
 .../monitors/browser/synthexec/enrich.go      |  2 -
 5 files changed, 58 insertions(+), 39 deletions(-)

diff --git a/heartbeat/monitors/mocks_test.go b/heartbeat/monitors/mocks_test.go
index 16fd5cd15fe..4cfd3b372ed 100644
--- a/heartbeat/monitors/mocks_test.go
+++ b/heartbeat/monitors/mocks_test.go
@@ -23,6 +23,8 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/elastic/beats/v7/libbeat/common/atomic"
+
 	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
 
 	"github.com/stretchr/testify/require"
@@ -136,31 +138,43 @@ func createMockJob(name string, cfg *common.Config) ([]jobs.Job, error) {
 	return []jobs.Job{j}, nil
 }
 
-func mockPluginBuilder() plugin.PluginFactory {
+func mockPluginBuilder() (p plugin.PluginFactory, built *atomic.Int, closed *atomic.Int) {
 	reg := monitoring.NewRegistry()
 
+	built = atomic.NewInt(0)
+	closed = atomic.NewInt(0)
+
 	return plugin.PluginFactory{
-		"test",
-		[]string{"testAlias"},
-		func(s string, config *common.Config) (plugin.Plugin, error) {
-			// Declare a real config block with a required attr so we can see what happens when it doesn't work
-			unpacked := struct {
-				URLs []string `config:"urls" validate:"required"`
-			}{}
-			err := config.Unpack(&unpacked)
-			if err != nil {
-				return plugin.Plugin{}, err
-			}
-			c := common.Config{}
-			j, err := createMockJob("test", &c)
-			return plugin.Plugin{j, nil, 1}, err
-		}, plugin.NewPluginCountersRecorder("test", reg)}
-}
-
-func mockPluginsReg() *plugin.PluginsReg {
+			"test",
+			[]string{"testAlias"},
+			func(s string, config *common.Config) (plugin.Plugin, error) {
+				built.Inc()
+				// Declare a real config block with a required attr so we can see what happens when it doesn't work
+				unpacked := struct {
+					URLs []string `config:"urls" validate:"required"`
+				}{}
+				err := config.Unpack(&unpacked)
+				if err != nil {
+					return plugin.Plugin{}, err
+				}
+				c := common.Config{}
+				j, err := createMockJob("test", &c)
+				close := func() error {
+					closed.Inc()
+					return nil
+				}
+				return plugin.Plugin{j, close, 1}, err
+			},
+			plugin.NewPluginCountersRecorder("test", reg)},
+		built,
+		closed
+}
+
+func mockPluginsReg() (p *plugin.PluginsReg, built *atomic.Int, closed *atomic.Int) {
 	reg := plugin.NewPluginsReg()
-	reg.Add(mockPluginBuilder())
-	return reg
+	builder, built, closed := mockPluginBuilder()
+	reg.Add(builder)
+	return reg, built, closed
 }
 
 func mockPluginConf(t *testing.T, id string, schedule string, url string) *common.Config {
diff --git a/heartbeat/monitors/monitor.go b/heartbeat/monitors/monitor.go
index 263d85ca862..4003cacdf42 100644
--- a/heartbeat/monitors/monitor.go
+++ b/heartbeat/monitors/monitor.go
@@ -23,14 +23,12 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
-
-	"github.com/elastic/beats/v7/heartbeat/monitors/stdfields"
-
 	"github.com/mitchellh/hashstructure"
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
+	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
+	"github.com/elastic/beats/v7/heartbeat/monitors/stdfields"
 	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers"
 	"github.com/elastic/beats/v7/heartbeat/scheduler"
 	"github.com/elastic/beats/v7/heartbeat/watcher"
diff --git a/heartbeat/monitors/monitor_test.go b/heartbeat/monitors/monitor_test.go
index 341839e382d..b0d02f819f6 100644
--- a/heartbeat/monitors/monitor_test.go
+++ b/heartbeat/monitors/monitor_test.go
@@ -21,19 +21,17 @@ import (
 	"testing"
 	"time"
 
-	"github.com/elastic/beats/v7/libbeat/monitoring"
-
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/elastic/go-lookslike/testslike"
-
 	"github.com/elastic/beats/v7/heartbeat/scheduler"
+	"github.com/elastic/beats/v7/libbeat/monitoring"
+	"github.com/elastic/go-lookslike/testslike"
 )
 
 func TestMonitor(t *testing.T) {
 	serverMonConf := mockPluginConf(t, "", "@every 1ms", "http://example.net")
-	reg := mockPluginsReg()
+	reg, built, closed := mockPluginsReg()
 	pipelineConnector := &MockPipelineConnector{}
 
 	sched := scheduler.New(1, monitoring.NewRegistry())
@@ -57,7 +55,6 @@ func TestMonitor(t *testing.T) {
 		if count >= 1 {
 			success = true
 
-			mon.Stop()
 			pcClient.Close()
 
 			for _, event := range pcClient.Publishes() {
@@ -74,14 +71,17 @@ func TestMonitor(t *testing.T) {
 		t.Fatalf("No publishes detected!")
 	}
 
+	assert.Equal(t, 1, built.Load())
 	mon.Stop()
+
+	assert.Equal(t, 1, closed.Load())
 	assert.Equal(t, true, pcClient.closed)
 }
 
 func TestDuplicateMonitorIDs(t *testing.T) {
 	serverMonConf := mockPluginConf(t, "custom", "@every 1ms", "http://example.net")
 	badConf := mockBadPluginConf(t, "custom", "@every 1ms")
-	reg := mockPluginsReg()
+	reg, built, closed := mockPluginsReg()
 	pipelineConnector := &MockPipelineConnector{}
 
 	sched := scheduler.New(1, monitoring.NewRegistry())
@@ -102,15 +102,22 @@ func TestDuplicateMonitorIDs(t *testing.T) {
 	require.NoError(t, m1Err)
 	_, m2Err := makeTestMon()
 	require.Error(t, m2Err)
-
 	m1.Stop()
-	_, m3Err := makeTestMon()
+	m3, m3Err := makeTestMon()
+	require.NoError(t, m3Err)
+	m3.Stop()
+
+	// We count 3 because built doesn't count successful builds,
+	// just attempted creations of monitors
+	require.Equal(t, 3, built.Load())
+	// Only one stops because the others errored on create
+	require.Equal(t, 2, closed.Load())
 	require.NoError(t, m3Err)
 }
 
 func TestCheckInvalidConfig(t *testing.T) {
 	serverMonConf := mockInvalidPluginConf(t)
-	reg := mockPluginsReg()
+	reg, built, closed := mockPluginsReg()
 	pipelineConnector := &MockPipelineConnector{}
 
 	sched := scheduler.New(1, monitoring.NewRegistry())
@@ -122,5 +129,9 @@ func TestCheckInvalidConfig(t *testing.T) {
 	// This could change if we decide the contract for newMonitor should always return a monitor
 	require.Nil(t, m, "For this test to work we need a nil value for the monitor.")
 
+	// These counters are both zero since this fails at config parse time
+	require.Equal(t, 0, built.Load())
+	require.Equal(t, 0, closed.Load())
+
 	require.Error(t, checkMonitorConfig(serverMonConf, reg, false))
 }
diff --git a/heartbeat/monitors/stdfields/stdfields.go b/heartbeat/monitors/stdfields/stdfields.go
index 433f6223863..26b28628fc3 100644
--- a/heartbeat/monitors/stdfields/stdfields.go
+++ b/heartbeat/monitors/stdfields/stdfields.go
@@ -18,7 +18,6 @@
 package stdfields
 
 import (
-	"fmt"
 	"time"
 
 	"github.com/pkg/errors"
@@ -50,7 +49,6 @@ func ConfigToStdMonitorFields(config *common.Config) (StdMonitorFields, error) {
 	mpi := StdMonitorFields{Enabled: true}
 
 	if err := config.Unpack(&mpi); err != nil {
-		fmt.Printf("HIER %s", err)
 		return mpi, errors.Wrap(err, "error unpacking monitor plugin config")
 	}
 
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/enrich.go b/x-pack/heartbeat/monitors/browser/synthexec/enrich.go
index a246fd41243..0df01cf278a 100644
--- a/x-pack/heartbeat/monitors/browser/synthexec/enrich.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/enrich.go
@@ -80,8 +80,6 @@ func (je *journeyEnricher) enrich(event *beat.Event, se *SynthEvent) error {
 		event.Timestamp = time.Now()
 	}
 
-
-
 	eventext.MergeEventFields(event, common.MapStr{
 		"monitor": common.MapStr{
 			"check_group": je.checkGroup,

From 0956aff6a60b06e8c2f22b5ac2985c44b00973b3 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Wed, 20 Jan 2021 20:50:38 -0600
Subject: [PATCH 28/41] Refactor wrappers/monitors for greater testability

---
 heartbeat/monitors/mocks_test.go        |  6 +-
 heartbeat/monitors/wrappers/monitors.go | 81 ++++++++++++++-----------
 2 files changed, 48 insertions(+), 39 deletions(-)

diff --git a/heartbeat/monitors/mocks_test.go b/heartbeat/monitors/mocks_test.go
index 4cfd3b372ed..b818234720d 100644
--- a/heartbeat/monitors/mocks_test.go
+++ b/heartbeat/monitors/mocks_test.go
@@ -23,18 +23,16 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/elastic/beats/v7/libbeat/common/atomic"
-
-	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
-
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/v7/heartbeat/eventext"
 	"github.com/elastic/beats/v7/heartbeat/hbtest"
 	"github.com/elastic/beats/v7/heartbeat/hbtestllext"
 	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
+	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/common/atomic"
 	"github.com/elastic/beats/v7/libbeat/monitoring"
 	"github.com/elastic/go-lookslike"
 	"github.com/elastic/go-lookslike/isdef"
diff --git a/heartbeat/monitors/wrappers/monitors.go b/heartbeat/monitors/wrappers/monitors.go
index f7a2881aa9a..2bf8d7017e2 100644
--- a/heartbeat/monitors/wrappers/monitors.go
+++ b/heartbeat/monitors/wrappers/monitors.go
@@ -45,6 +45,7 @@ func WrapCommon(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.J
 	}
 }
 
+// WrapLightweight applies to http/tcp/icmp, everything but journeys involving node
 func WrapLightweight(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.Job {
 	return jobs.WrapAllSeparately(
 		jobs.WrapAll(
@@ -58,6 +59,9 @@ func WrapLightweight(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []j
 		})
 }
 
+// WrapBrowser is pretty minimal in terms of fields added. The browser monitor
+// type handles most of the fields directly, since it runs multiple jobs in a single
+// run it needs to take this task on in a unique way.
 func WrapBrowser(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.Job {
 	return jobs.WrapAll(
 		js,
@@ -70,48 +74,55 @@ func WrapBrowser(js []jobs.Job, stdMonFields stdfields.StdMonitorFields) []jobs.
 func addMonitorMeta(stdMonFields stdfields.StdMonitorFields, isMulti bool) jobs.JobWrapper {
 	return func(job jobs.Job) jobs.Job {
 		return func(event *beat.Event) ([]jobs.Job, error) {
-			started := time.Now()
 			cont, e := job(event)
-			thisID := stdMonFields.ID
-			thisName := stdMonFields.Name
-			// Allow jobs to override the ID, useful for browser suites
-			// which do this logic on their own
-			if v, _ := event.GetValue("monitor.id"); v != nil {
-				thisID = v.(string)
-			}
-			if v, _ := event.GetValue("monitor.name"); v != nil {
-				thisName = v.(string)
-			}
-			if isMulti {
-				url, err := event.GetValue("url.full")
-				if err != nil {
-					logp.Error(errors.Wrap(err, "Mandatory url.full key missing!"))
-					url = "n/a"
-				}
-				urlHash, _ := hashstructure.Hash(url, nil)
-				thisID = fmt.Sprintf("%s-%x", stdMonFields.ID, urlHash)
-			}
+			addMonitorMetaFields(event, time.Now(), stdMonFields, isMulti)
+			return cont, e
+		}
+	}
+}
 
-			fieldsToMerge := common.MapStr{
-				"monitor": common.MapStr{
-					"id":       thisID,
-					"name":     thisName,
-					"type":     stdMonFields.Type,
-					"timespan": timespan(started, stdMonFields.Schedule, stdMonFields.Timeout),
-				},
-			}
+func addMonitorMetaFields(event *beat.Event, started time.Time, sf stdfields.StdMonitorFields, isMulti bool) {
+	id := sf.ID
+	name := sf.Name
 
-			if stdMonFields.Service.Name != "" {
-				fieldsToMerge["service"] = common.MapStr{
-					"name": stdMonFields.Service.Name,
-				}
-			}
+	// If multiple jobs are listed for this monitor, we can't have a single ID, so we hash the
+	// unique URLs to create unique suffixes for the monitor.
+	if isMulti {
+		url, err := event.GetValue("url.full")
+		if err != nil {
+			logp.Error(errors.Wrap(err, "Mandatory url.full key missing!"))
+			url = "n/a"
+		}
+		urlHash, _ := hashstructure.Hash(url, nil)
+		id = fmt.Sprintf("%s-%x", sf.ID, urlHash)
+	}
 
-			eventext.MergeEventFields(event, fieldsToMerge)
+	// Allow jobs to override the ID, useful for browser suites
+	// which do this logic on their own
+	if v, _ := event.GetValue("monitor.id"); v != nil {
+		id = v.(string)
+	}
+	if v, _ := event.GetValue("monitor.name"); v != nil {
+		name = v.(string)
+	}
 
-			return cont, e
+	fieldsToMerge := common.MapStr{
+		"monitor": common.MapStr{
+			"id":       id,
+			"name":     name,
+			"type":     sf.Type,
+			"timespan": timespan(started, sf.Schedule, sf.Timeout),
+		},
+	}
+
+	// Add service.name for APM interop
+	if sf.Service.Name != "" {
+		fieldsToMerge["service"] = common.MapStr{
+			"name": sf.Service.Name,
 		}
 	}
+
+	eventext.MergeEventFields(event, fieldsToMerge)
 }
 
 func timespan(started time.Time, sched *schedule.Schedule, timeout time.Duration) common.MapStr {

From 85543706f072ee77a7da6bd36e14a5a7ffdb2464 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Mon, 25 Jan 2021 13:03:09 -0600
Subject: [PATCH 29/41] checkpoint

---
 heartbeat/monitors/wrappers/monitors.go      |  4 +--
 heartbeat/monitors/wrappers/monitors_test.go | 38 ++++++++++++++++++++
 x-pack/heartbeat/monitors/browser/config.go  | 16 +++++++++
 3 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/heartbeat/monitors/wrappers/monitors.go b/heartbeat/monitors/wrappers/monitors.go
index 2bf8d7017e2..7dfbd529373 100644
--- a/heartbeat/monitors/wrappers/monitors.go
+++ b/heartbeat/monitors/wrappers/monitors.go
@@ -100,10 +100,10 @@ func addMonitorMetaFields(event *beat.Event, started time.Time, sf stdfields.Std
 	// Allow jobs to override the ID, useful for browser suites
 	// which do this logic on their own
 	if v, _ := event.GetValue("monitor.id"); v != nil {
-		id = v.(string)
+		id = fmt.Sprintf("%s - %s", sf.ID, v.(string))
 	}
 	if v, _ := event.GetValue("monitor.name"); v != nil {
-		name = v.(string)
+		name = fmt.Sprintf("%s - %s", sf.Name, v.(string))
 	}
 
 	fieldsToMerge := common.MapStr{
diff --git a/heartbeat/monitors/wrappers/monitors_test.go b/heartbeat/monitors/wrappers/monitors_test.go
index d8ff497225e..74a780982f7 100644
--- a/heartbeat/monitors/wrappers/monitors_test.go
+++ b/heartbeat/monitors/wrappers/monitors_test.go
@@ -387,3 +387,41 @@ func TestTimespan(t *testing.T) {
 		})
 	}
 }
+
+func makeInlineBrowserJob(t *testing.T, u string) jobs.Job {
+	parsed, err := url.Parse(u)
+	require.NoError(t, err)
+	return func(event *beat.Event) (i []jobs.Job, e error) {
+		eventext.MergeEventFields(event, common.MapStr{"url": URLFields(parsed)})
+		return nil, nil
+	}
+}
+
+// Inline browser jobs function very similarly to lightweight jobs
+// in that they don't override the ID.
+// They do not, however, get a summary field added, nor duration.
+func TestInlineBrowserJob(t *testing.T) {
+	fields := testMonFields
+	testCommonWrap(t, testDef{
+		"simple",
+		fields,
+		[]jobs.Job{makeInlineBrowserJob(t, "http://foo.com")},
+		[]validator.Validator{
+			lookslike.Compose(
+				urlValidator(t, "http://foo.com"),
+				lookslike.MustCompile(map[string]interface{}{
+					"monitor": map[string]interface{}{
+						"duration.us": isdef.IsDuration,
+						"id":          testMonFields.ID,
+						"name":        testMonFields.Name,
+						"type":        testMonFields.Type,
+						"status":      "up",
+						"check_group": isdef.IsString,
+					},
+				}),
+				hbtestllext.MonitorTimespanValidator,
+				summaryValidator(1, 0),
+			)},
+		nil,
+	})
+}
diff --git a/x-pack/heartbeat/monitors/browser/config.go b/x-pack/heartbeat/monitors/browser/config.go
index 412b7b9c611..14132a0f714 100644
--- a/x-pack/heartbeat/monitors/browser/config.go
+++ b/x-pack/heartbeat/monitors/browser/config.go
@@ -5,6 +5,7 @@
 package browser
 
 import (
+	"fmt"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/source"
 )
@@ -14,4 +15,19 @@ type Config struct {
 	Params    map[string]interface{} `config:"params"`
 	RawConfig *common.Config
 	Source    *source.Source `config:"source"`
+	// Name is optional for lightweight checks but required for browsers
+	Name string `config:"id"`
+	// Id is optional for lightweight checks but required for browsers
+	Id string `config:"name"`
+}
+
+func (c *Config) Validate() error {
+	if c.Name == "" {
+		return fmt.Errorf("config 'name' must be specified for this moniotr")
+	}
+	if c.Id == "" {
+		return fmt.Errorf("config 'id' must be specified for this monitor")
+	}
+
+	return nil
 }

From 8081b1c7c66bfc944bd8bba61e6d7e7ff1bff275 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Fri, 29 Jan 2021 16:06:51 -0600
Subject: [PATCH 30/41] Add tests for new wrapper behavior

---
 heartbeat/monitors/wrappers/monitors.go      |   2 +-
 heartbeat/monitors/wrappers/monitors_test.go | 102 ++++++++++++++++---
 x-pack/heartbeat/monitors/browser/config.go  |   1 +
 3 files changed, 89 insertions(+), 16 deletions(-)

diff --git a/heartbeat/monitors/wrappers/monitors.go b/heartbeat/monitors/wrappers/monitors.go
index 7dfbd529373..dca3a8b70de 100644
--- a/heartbeat/monitors/wrappers/monitors.go
+++ b/heartbeat/monitors/wrappers/monitors.go
@@ -100,7 +100,7 @@ func addMonitorMetaFields(event *beat.Event, started time.Time, sf stdfields.Std
 	// Allow jobs to override the ID, useful for browser suites
 	// which do this logic on their own
 	if v, _ := event.GetValue("monitor.id"); v != nil {
-		id = fmt.Sprintf("%s - %s", sf.ID, v.(string))
+		id = fmt.Sprintf("%s-%s", sf.ID, v.(string))
 	}
 	if v, _ := event.GetValue("monitor.name"); v != nil {
 		name = fmt.Sprintf("%s - %s", sf.Name, v.(string))
diff --git a/heartbeat/monitors/wrappers/monitors_test.go b/heartbeat/monitors/wrappers/monitors_test.go
index 74a780982f7..88e6fd76997 100644
--- a/heartbeat/monitors/wrappers/monitors_test.go
+++ b/heartbeat/monitors/wrappers/monitors_test.go
@@ -56,6 +56,14 @@ var testMonFields = stdfields.StdMonitorFields{
 	Timeout:  1,
 }
 
+var testBrowserMonFields = stdfields.StdMonitorFields{
+	ID:       "myid",
+	Name:     "myname",
+	Type:     "browser",
+	Schedule: schedule.MustParse("@every 1s"),
+	Timeout:  1,
+}
+
 func testCommonWrap(t *testing.T, tt testDef) {
 	t.Run(tt.name, func(t *testing.T) {
 		wrapped := WrapCommon(tt.jobs, tt.stdFields)
@@ -392,7 +400,12 @@ func makeInlineBrowserJob(t *testing.T, u string) jobs.Job {
 	parsed, err := url.Parse(u)
 	require.NoError(t, err)
 	return func(event *beat.Event) (i []jobs.Job, e error) {
-		eventext.MergeEventFields(event, common.MapStr{"url": URLFields(parsed)})
+		eventext.MergeEventFields(event, common.MapStr{
+			"url": URLFields(parsed),
+			"monitor": common.MapStr{
+				"check_group": "inline-check-group",
+			},
+		})
 		return nil, nil
 	}
 }
@@ -401,26 +414,85 @@ func makeInlineBrowserJob(t *testing.T, u string) jobs.Job {
 // in that they don't override the ID.
 // They do not, however, get a summary field added, nor duration.
 func TestInlineBrowserJob(t *testing.T) {
-	fields := testMonFields
+	fields := testBrowserMonFields
 	testCommonWrap(t, testDef{
 		"simple",
 		fields,
 		[]jobs.Job{makeInlineBrowserJob(t, "http://foo.com")},
+		[]validator.Validator{
+			lookslike.Strict(
+				lookslike.Compose(
+					urlValidator(t, "http://foo.com"),
+					lookslike.MustCompile(map[string]interface{}{
+						"monitor": map[string]interface{}{
+							"id":          testMonFields.ID,
+							"name":        testMonFields.Name,
+							"type":        fields.Type,
+							"status":      "up",
+							"check_group": "inline-check-group",
+						},
+					}),
+					hbtestllext.MonitorTimespanValidator,
+				),
+			),
+		},
+		nil,
+	})
+}
+
+var suiteBrowserJobValues = struct {
+	id         string
+	name       string
+	checkGroup string
+}{
+	id:         "journey_1",
+	name:       "Journey 1",
+	checkGroup: "journey-1-check-group",
+}
+
+func makeSuiteBrowserJob(t *testing.T, u string) jobs.Job {
+	parsed, err := url.Parse(u)
+	require.NoError(t, err)
+	return func(event *beat.Event) (i []jobs.Job, e error) {
+		eventext.MergeEventFields(event, common.MapStr{
+			"url": URLFields(parsed),
+			"monitor": common.MapStr{
+				"id":          suiteBrowserJobValues.id,
+				"name":        suiteBrowserJobValues.name,
+				"check_group": suiteBrowserJobValues.checkGroup,
+			},
+		})
+		return nil, nil
+	}
+}
+
+func TestSuiteBrowserJob(t *testing.T) {
+	fields := testBrowserMonFields
+	urlStr := "http://foo.com"
+	urlU, _ := url.Parse(urlStr)
+	testCommonWrap(t, testDef{
+		"simple",
+		fields,
+		[]jobs.Job{makeSuiteBrowserJob(t, urlStr)},
 		[]validator.Validator{
 			lookslike.Compose(
-				urlValidator(t, "http://foo.com"),
-				lookslike.MustCompile(map[string]interface{}{
-					"monitor": map[string]interface{}{
-						"duration.us": isdef.IsDuration,
-						"id":          testMonFields.ID,
-						"name":        testMonFields.Name,
-						"type":        testMonFields.Type,
-						"status":      "up",
-						"check_group": isdef.IsString,
-					},
-				}),
-				hbtestllext.MonitorTimespanValidator,
-				summaryValidator(1, 0),
+				urlValidator(t, urlStr),
+				lookslike.Strict(
+					lookslike.MustCompile(map[string]interface{}{
+						"monitor": map[string]interface{}{
+							"id":          fmt.Sprintf("%s-%s", testMonFields.ID, suiteBrowserJobValues.id),
+							"name":        fmt.Sprintf("%s - %s", testMonFields.Name, suiteBrowserJobValues.name),
+							"type":        fields.Type,
+							"check_group": suiteBrowserJobValues.checkGroup,
+							"status":      "up",
+							"timespan": common.MapStr{
+								"gte": hbtestllext.IsTime,
+								"lt":  hbtestllext.IsTime,
+							},
+						},
+						"url": URLFields(urlU),
+					}),
+				),
 			)},
 		nil,
 	})
diff --git a/x-pack/heartbeat/monitors/browser/config.go b/x-pack/heartbeat/monitors/browser/config.go
index 14132a0f714..02559fe9481 100644
--- a/x-pack/heartbeat/monitors/browser/config.go
+++ b/x-pack/heartbeat/monitors/browser/config.go
@@ -6,6 +6,7 @@ package browser
 
 import (
 	"fmt"
+
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/source"
 )

From 2332b1389a66172a57a4de2b80b44d4bf83e7cce Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Fri, 29 Jan 2021 17:35:54 -0600
Subject: [PATCH 31/41] Additional tests

---
 x-pack/heartbeat/monitors/browser/browser.go  | 10 ++-
 x-pack/heartbeat/monitors/browser/config.go   | 11 ++-
 .../monitors/browser/source/source.go         |  9 +--
 .../monitors/browser/suite_runner.go          | 10 ++-
 .../monitors/browser/suite_runner_test.go     | 76 +++++++++++++++++++
 5 files changed, 99 insertions(+), 17 deletions(-)
 create mode 100644 x-pack/heartbeat/monitors/browser/suite_runner_test.go

diff --git a/x-pack/heartbeat/monitors/browser/browser.go b/x-pack/heartbeat/monitors/browser/browser.go
index 756760bd4a1..76e02f1ff28 100644
--- a/x-pack/heartbeat/monitors/browser/browser.go
+++ b/x-pack/heartbeat/monitors/browser/browser.go
@@ -11,11 +11,9 @@ import (
 	"os/user"
 	"sync"
 
+	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
 	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
-
 	"github.com/elastic/beats/v7/libbeat/beat"
-
-	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/synthexec"
@@ -71,5 +69,9 @@ func create(name string, cfg *common.Config) (p plugin.Plugin, err error) {
 		}
 	}
 
-	return plugin.Plugin{[]jobs.Job{j}, ss.Close, 1}, nil
+	return plugin.Plugin{
+		Jobs:      []jobs.Job{j},
+		Close:     ss.Close,
+		Endpoints: 1,
+	}, nil
 }
diff --git a/x-pack/heartbeat/monitors/browser/config.go b/x-pack/heartbeat/monitors/browser/config.go
index 02559fe9481..0cbb699da88 100644
--- a/x-pack/heartbeat/monitors/browser/config.go
+++ b/x-pack/heartbeat/monitors/browser/config.go
@@ -17,17 +17,20 @@ type Config struct {
 	RawConfig *common.Config
 	Source    *source.Source `config:"source"`
 	// Name is optional for lightweight checks but required for browsers
-	Name string `config:"id"`
+	Name string `config:"name"`
 	// Id is optional for lightweight checks but required for browsers
-	Id string `config:"name"`
+	Id string `config:"id"`
 }
 
+var ErrNameRequired = fmt.Errorf("config 'name' must be specified for this monitor")
+var ErrIdRequired = fmt.Errorf("config 'id' must be specified for this monitor")
+
 func (c *Config) Validate() error {
 	if c.Name == "" {
-		return fmt.Errorf("config 'name' must be specified for this moniotr")
+		return ErrNameRequired
 	}
 	if c.Id == "" {
-		return fmt.Errorf("config 'id' must be specified for this monitor")
+		return ErrIdRequired
 	}
 
 	return nil
diff --git a/x-pack/heartbeat/monitors/browser/source/source.go b/x-pack/heartbeat/monitors/browser/source/source.go
index a0f21f1e621..62c7ce03e9e 100644
--- a/x-pack/heartbeat/monitors/browser/source/source.go
+++ b/x-pack/heartbeat/monitors/browser/source/source.go
@@ -28,9 +28,11 @@ func (s *Source) Active() ISource {
 	return s.ActiveMemo
 }
 
+var ErrInvalidSource = fmt.Errorf("no or unknown source type specified for synthetic monitor")
+
 func (s *Source) Validate() error {
 	if s.Active() == nil {
-		return fmt.Errorf("no valid source specified! Choose one of local, github, zip_url")
+		return ErrInvalidSource
 	}
 	return nil
 }
@@ -44,8 +46,3 @@ type ISource interface {
 type BaseSource struct {
 	Type string `config:"type"`
 }
-
-type PollingSource struct {
-	CheckEvery int `config:"check_every"`
-	BaseSource
-}
diff --git a/x-pack/heartbeat/monitors/browser/suite_runner.go b/x-pack/heartbeat/monitors/browser/suite_runner.go
index 17308cdc7ba..73b597766ff 100644
--- a/x-pack/heartbeat/monitors/browser/suite_runner.go
+++ b/x-pack/heartbeat/monitors/browser/suite_runner.go
@@ -6,9 +6,9 @@ package browser
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
 type JourneyLister func(ctx context.Context, suitePath string, params common.MapStr) (journeyNames []string, err error)
@@ -27,10 +27,14 @@ func NewSuite(rawCfg *common.Config) (*SyntheticSuite, error) {
 	}
 	err := rawCfg.Unpack(ss.suiteCfg)
 	if err != nil {
-		logp.Err("could not parse suite config: %s", err)
+		return nil, ErrBadConfig(err)
 	}
 
-	return ss, err
+	return ss, nil
+}
+
+func ErrBadConfig(err error) error {
+	return fmt.Errorf("could not parse suite config: %w", err)
 }
 
 func (s *SyntheticSuite) String() string {
diff --git a/x-pack/heartbeat/monitors/browser/suite_runner_test.go b/x-pack/heartbeat/monitors/browser/suite_runner_test.go
new file mode 100644
index 00000000000..d99e3c22056
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/suite_runner_test.go
@@ -0,0 +1,76 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package browser
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/source"
+)
+
+func TestValidInline(t *testing.T) {
+	script := "a script"
+	testParams := map[string]interface{}{
+		"key1": "value1",
+		"key2": "value2",
+	}
+	cfg := common.MustNewConfigFrom(common.MapStr{
+		"name":   "My Name",
+		"id":     "myId",
+		"params": testParams,
+		"source": common.MapStr{
+			"inline": common.MapStr{
+				"script": script,
+			},
+		},
+	})
+	s, e := NewSuite(cfg)
+	require.NoError(t, e)
+	require.NotNil(t, s)
+	sSrc, ok := s.InlineSource()
+	require.True(t, ok)
+	require.Equal(t, script, script, sSrc)
+	require.Equal(t, "", s.Workdir())
+	require.Equal(t, testParams, s.Params())
+}
+
+func TestNameRequired(t *testing.T) {
+	cfg := common.MustNewConfigFrom(common.MapStr{
+		"id": "myId",
+		"source": common.MapStr{
+			"inline": common.MapStr{
+				"script": "a script",
+			},
+		},
+	})
+	_, e := NewSuite(cfg)
+	require.Regexp(t, ErrNameRequired, e)
+}
+
+func TestIDRequired(t *testing.T) {
+	cfg := common.MustNewConfigFrom(common.MapStr{
+		"name": "My Name",
+		"source": common.MapStr{
+			"inline": common.MapStr{
+				"script": "a script",
+			},
+		},
+	})
+	_, e := NewSuite(cfg)
+	require.Regexp(t, ErrIdRequired, e)
+}
+
+func TestEmptySource(t *testing.T) {
+	cfg := common.MustNewConfigFrom(common.MapStr{
+		"source": common.MapStr{},
+	})
+	s, e := NewSuite(cfg)
+
+	require.Regexp(t, ErrBadConfig(source.ErrInvalidSource), e)
+	require.Nil(t, s)
+}

From fde576532016ba417d29d7cb61b2602599b29ced Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Fri, 29 Jan 2021 17:57:29 -0600
Subject: [PATCH 32/41] Add basic validations for local source

---
 .../monitors/browser/source/local.go          | 10 ++++--
 .../monitors/browser/source/local_test.go     | 34 +++++++++++++++++++
 .../monitors/browser/suite_runner_test.go     |  3 ++
 3 files changed, 45 insertions(+), 2 deletions(-)
 create mode 100644 x-pack/heartbeat/monitors/browser/source/local_test.go

diff --git a/x-pack/heartbeat/monitors/browser/source/local.go b/x-pack/heartbeat/monitors/browser/source/local.go
index c1a95d31a6e..376f7b20605 100644
--- a/x-pack/heartbeat/monitors/browser/source/local.go
+++ b/x-pack/heartbeat/monitors/browser/source/local.go
@@ -22,13 +22,19 @@ type LocalSource struct {
 	BaseSource
 }
 
+var ErrNoPath = fmt.Errorf("local source defined with no path specified")
+
+func ErrInvalidPath(path string) error {
+	return fmt.Errorf("local source has invalid path '%s'", path)
+}
+
 func (l *LocalSource) Validate() error {
 	if l.OrigPath == "" {
-		return fmt.Errorf("local source defined with no path specified")
+		return ErrNoPath
 	}
 
 	s, err := os.Stat(l.OrigPath)
-	base := fmt.Sprintf("local source has invalid path '%s'", l.OrigPath)
+	base := ErrInvalidPath(l.OrigPath)
 	if err != nil {
 		return fmt.Errorf("%s: %w", base, err)
 	}
diff --git a/x-pack/heartbeat/monitors/browser/source/local_test.go b/x-pack/heartbeat/monitors/browser/source/local_test.go
new file mode 100644
index 00000000000..76a4279a178
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/source/local_test.go
@@ -0,0 +1,34 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package source
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLocalSourceValidate(t *testing.T) {
+	tests := []struct {
+		name     string
+		OrigPath string
+		err      error
+	}{
+		{"valid", "./", nil},
+		{"invalid", "/not/a/path", ErrInvalidPath("/not/a/path")},
+		{"nopath", "", ErrNoPath},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			l := &LocalSource{OrigPath: tt.OrigPath}
+			err := l.Validate()
+			if tt.err == nil {
+				require.NoError(t, err)
+			} else {
+				require.Regexp(t, tt.err, err)
+			}
+		})
+	}
+}
diff --git a/x-pack/heartbeat/monitors/browser/suite_runner_test.go b/x-pack/heartbeat/monitors/browser/suite_runner_test.go
index d99e3c22056..0287b5276d9 100644
--- a/x-pack/heartbeat/monitors/browser/suite_runner_test.go
+++ b/x-pack/heartbeat/monitors/browser/suite_runner_test.go
@@ -37,6 +37,9 @@ func TestValidInline(t *testing.T) {
 	require.Equal(t, script, script, sSrc)
 	require.Equal(t, "", s.Workdir())
 	require.Equal(t, testParams, s.Params())
+
+	e = s.Close()
+	require.NoError(t, e)
 }
 
 func TestNameRequired(t *testing.T) {

From c82c15594b04ca3f75f3825fe51b13c18929d5a6 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Sat, 30 Jan 2021 09:31:53 -0600
Subject: [PATCH 33/41] Add tests for local source

---
 .../browser/source/fixtures/todos/.npmrc      |  1 +
 .../browser/source/fixtures/todos/Dockerfile  | 30 +++++++++++
 .../browser/source/fixtures/todos/README.md   | 30 +++++++++++
 .../fixtures/todos/add-remove.journey.ts      | 38 +++++++++++++
 .../source/fixtures/todos/basics.journey.ts   | 30 +++++++++++
 .../todos/build-offline-dockerfile.sh         |  7 +++
 .../fixtures/todos/heartbeat.docker.yml       | 12 +++++
 .../browser/source/fixtures/todos/helpers.ts  | 54 +++++++++++++++++++
 .../source/fixtures/todos/package.json        | 11 ++++
 .../monitors/browser/source/local.go          | 13 ++++-
 .../monitors/browser/source/local_test.go     | 41 ++++++++++++++
 11 files changed, 266 insertions(+), 1 deletion(-)
 create mode 100644 x-pack/heartbeat/monitors/browser/source/fixtures/todos/.npmrc
 create mode 100644 x-pack/heartbeat/monitors/browser/source/fixtures/todos/Dockerfile
 create mode 100644 x-pack/heartbeat/monitors/browser/source/fixtures/todos/README.md
 create mode 100644 x-pack/heartbeat/monitors/browser/source/fixtures/todos/add-remove.journey.ts
 create mode 100644 x-pack/heartbeat/monitors/browser/source/fixtures/todos/basics.journey.ts
 create mode 100755 x-pack/heartbeat/monitors/browser/source/fixtures/todos/build-offline-dockerfile.sh
 create mode 100644 x-pack/heartbeat/monitors/browser/source/fixtures/todos/heartbeat.docker.yml
 create mode 100644 x-pack/heartbeat/monitors/browser/source/fixtures/todos/helpers.ts
 create mode 100644 x-pack/heartbeat/monitors/browser/source/fixtures/todos/package.json

diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/.npmrc b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/.npmrc
new file mode 100644
index 00000000000..43c97e719a5
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/.npmrc
@@ -0,0 +1 @@
+package-lock=false
diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/Dockerfile b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/Dockerfile
new file mode 100644
index 00000000000..4b3c80291dd
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/Dockerfile
@@ -0,0 +1,30 @@
+# This Dockerfile illustrates the usage of synthetics in an air gapped environment, where 
+# public internet access is not available. In this situation you'll want to create
+# a custom image using our official docker image as a base.
+
+# Use our synthetics image as a base.
+ARG STACK_VERSION=latest
+FROM docker.elastic.co/experimental/synthetics:${STACK_VERSION}-synthetics
+# Use the line below if you're using a custom base image, usually only for
+# developers
+#FROM heartbeat-synthetics-local
+
+# This flag variable will prevent heartbeat from running `npm i` or
+# similar commands that depend on an internet connection.
+# We'll have to do that work now, when we bake the image.
+ENV ELASTIC_SYNTHETICS_OFFLINE=true
+
+# Copy our heartbeat config directly to the image.
+# This could be done as a shared mount instead, but if we're
+# baking an image anyway, this may be something that may be easier
+# to do in this manner.
+COPY heartbeat.docker.yml /usr/share/heartbeat/heartbeat.yml
+
+RUN mkdir -p $SUITES_DIR/todos
+# Copy your custom synthetics tests into a folder on the image
+COPY . $SUITES_DIR/todos/
+
+# Install NPM deps locally on this image
+# Please note that it's important to run both `npm install` AND `npm install playwright`
+# for more see this issue: https://github.com/microsoft/playwright/issues/3712
+RUN cd $SUITES_DIR/todos && npm install
diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/README.md b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/README.md
new file mode 100644
index 00000000000..de30b63da28
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/README.md
@@ -0,0 +1,30 @@
+# Test Vue.js Examples
+
+This suite tests the examples that ship with the open source Vue.js project.
+
+You can run the test suites in two ways
+
+## Running via `@elastic/synthetics`
+
+We can invoke the Synthetics runner from the CLI using the below steps
+
+```sh
+// Install the dependencies
+npm install
+
+// Invoke the runner and show test results
+npx @elastic/synthetics .
+
+```
+
+## Running via `Heartbeat`
+
+Invoke the synthetic test suites using heartbeat.
+
+```sh
+// Run the below command inside /examples/docker directory
+
+// run heartbeat which is already configured to run the todo app, you
+// can check `heartbeat.docker.yml`
+./run-build-local.sh -E output.console={}
+```
diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/add-remove.journey.ts b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/add-remove.journey.ts
new file mode 100644
index 00000000000..bb80e031cec
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/add-remove.journey.ts
@@ -0,0 +1,38 @@
+import { journey } from '@elastic/synthetics';
+import {
+  loadAppStep,
+  addTaskStep,
+  assertTaskListSizeStep,
+  checkForTaskStep,
+  destroyTaskStep,
+} from './helpers';
+
+journey('basic addition and completion of single task', async ({ page }) => {
+  const testText = "Don't put salt in your eyes";
+
+  loadAppStep(page);
+  addTaskStep(page, testText);
+  assertTaskListSizeStep(page, 1);
+  checkForTaskStep(page, testText);
+  destroyTaskStep(page, testText);
+  assertTaskListSizeStep(page, 0);
+});
+
+journey('adding and removing a few tasks', async ({ page }) => {
+  const testTasks = ['Task 1', 'Task 2', 'Task 3'];
+
+  loadAppStep(page);
+  testTasks.forEach(t => {
+    addTaskStep(page, t);
+  });
+
+  assertTaskListSizeStep(page, 3);
+
+  // remove the middle task and check that it worked
+  destroyTaskStep(page, testTasks[1]);
+  assertTaskListSizeStep(page, 2);
+
+  // add a new task and check it exists
+  addTaskStep(page, 'Task 4');
+  assertTaskListSizeStep(page, 3);
+});
diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/basics.journey.ts b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/basics.journey.ts
new file mode 100644
index 00000000000..f8f8f522635
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/basics.journey.ts
@@ -0,0 +1,30 @@
+import { journey, step } from '@elastic/synthetics';
+import { deepStrictEqual } from 'assert';
+import { join } from 'path';
+
+journey('check that title is present', async ({ page }) => {
+  step('go to app', async () => {
+    const path = 'file://' + join(__dirname, 'app', 'index.html');
+    await page.goto(path);
+  });
+
+  step('check title is present', async () => {
+    const header = await page.$('h1');
+    deepStrictEqual(await header.textContent(), 'todos');
+  });
+});
+
+journey('check that input placeholder is correct', async ({ page }) => {
+  step('go to app', async () => {
+    const path = 'file://' + join(__dirname, 'app', 'index.html');
+    await page.goto(path);
+  });
+
+  step('check title is present', async () => {
+    const input = await page.$('input.new-todo');
+    deepStrictEqual(
+      await input.getAttribute('placeholder'),
+      'What nneeds to be done?'
+    );
+  });
+});
diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/build-offline-dockerfile.sh b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/build-offline-dockerfile.sh
new file mode 100755
index 00000000000..25b57e46544
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/build-offline-dockerfile.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+# This demonstrates building and tagging a custom offline docker heartbeat image with synthetics
+# with all dependencies pre-bundled.
+
+# You'll want to run this in an environment with internet access so that NPM deps can be installed,
+# then, take the resultant image and transfer that to your air gapped network.
+docker build --build-arg STACK_VERSION=7.10.0 -t my-custom-heartbeat .
diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/heartbeat.docker.yml b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/heartbeat.docker.yml
new file mode 100644
index 00000000000..48c52f9da7e
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/heartbeat.docker.yml
@@ -0,0 +1,12 @@
+---
+seccomp.enabled: false
+heartbeat.config.monitors:
+  path: "${path.config}/monitors.d/*.yml"
+  reload.enabled: false
+  reload.period: 5s
+
+heartbeat.synthetic_suites:
+- name: Todos
+  # SUITES_DIR is an environment var provided by the docker container
+  path: "${SUITES_DIR}/todos"
+  schedule: "@every 1m"
\ No newline at end of file
diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/helpers.ts b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/helpers.ts
new file mode 100644
index 00000000000..bb69aeddbad
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/helpers.ts
@@ -0,0 +1,54 @@
+import { step } from '@elastic/synthetics';
+import * as assert from 'assert';
+import { join } from 'path';
+import { Page } from 'playwright-core';
+
+export const loadAppStep = (page: Page) => {
+  step('go to app', async () => {
+    const path = 'file://' + join(__dirname, 'app', 'index.html');
+    await page.goto(path);
+  });
+};
+
+export const addTaskStep = (page: Page, task: string) => {
+  step(`add task ${task}`, async () => {
+    const input = await page.$('input.new-todo');
+    await input.type(task);
+    await input.press('Enter');
+  });
+};
+
+const todosSelector = 'ul.todo-list li.todo';
+
+export const findTask = async (page: Page, task: string) => {
+  return await page.waitForSelector(`${todosSelector} >> text="${task}"`);
+};
+
+export const assertTaskListSizeStep = async (page: Page, size: number) => {
+  step(`check that task list has exactly ${size} elements`, async () => {
+    assert.deepEqual((await page.$$(todosSelector)).length, size);
+  });
+};
+
+export const checkForTaskStep = async (page: Page, task: string) => {
+  step(`check for task '${task}' in list`, async () => {
+    return findTask(page, task);
+  });
+};
+
+export const destroyTaskStep = async (page: Page, task: string) => {
+  step(`destroy task '${task}'`, async () => {
+    const label = await findTask(page, task);
+    // xpath indexes arrays starting at 1!!! Easy to forget!
+    const li = await label.$('xpath=ancestor::li[1]');
+    const destroyButton = await li.$('button');
+
+    // The destroy button is not visible until hovered. Setup a click test which
+    // will wait up to 30s for the button to be visible.
+    const clickFuture = destroyButton.click();
+    // now hover, making the destroy button clickable
+    li.hover();
+    // now we are done
+    await clickFuture;
+  });
+};
diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/package.json b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/package.json
new file mode 100644
index 00000000000..4de5c6a1414
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/package.json
@@ -0,0 +1,11 @@
+{
+  "name": "todos",
+  "private": true,
+  "description": "This suite tests the examples that ship with the open source Vue.js project.",
+  "scripts": {},
+  "license": "MIT",
+  "dependencies": {
+    "@elastic/synthetics": "*",
+    "playwright-core": "=1.6.2"
+  }
+}
diff --git a/x-pack/heartbeat/monitors/browser/source/local.go b/x-pack/heartbeat/monitors/browser/source/local.go
index 376f7b20605..d697f7b970f 100644
--- a/x-pack/heartbeat/monitors/browser/source/local.go
+++ b/x-pack/heartbeat/monitors/browser/source/local.go
@@ -45,6 +45,12 @@ func (l *LocalSource) Validate() error {
 	return nil
 }
 
+var offlineEnvVar = "ELASTIC_SYNTHETICS_OFFLINE"
+
+func offline() bool {
+	return os.Getenv(offlineEnvVar) == "true"
+}
+
 func (l *LocalSource) Fetch() (err error) {
 	if l.workingPath != "" {
 		return nil
@@ -53,6 +59,11 @@ func (l *LocalSource) Fetch() (err error) {
 	if err != nil {
 		return fmt.Errorf("could not create tmp dir: %w", err)
 	}
+	defer func() {
+		if err != nil {
+			l.Close() // cleanup the dir if this function returns an err
+		}
+	}()
 
 	err = copy.Copy(l.OrigPath, l.workingPath)
 	if err != nil {
@@ -64,7 +75,7 @@ func (l *LocalSource) Fetch() (err error) {
 		return err
 	}
 
-	if os.Getenv("ELASTIC_SYNTHETICS_OFFLINE") != "true" {
+	if !offline() {
 		// Ensure all deps installed
 		err = runSimpleCommand(exec.Command("npm", "install"), dir)
 		if err != nil {
diff --git a/x-pack/heartbeat/monitors/browser/source/local_test.go b/x-pack/heartbeat/monitors/browser/source/local_test.go
index 76a4279a178..b7e6d3be290 100644
--- a/x-pack/heartbeat/monitors/browser/source/local_test.go
+++ b/x-pack/heartbeat/monitors/browser/source/local_test.go
@@ -5,8 +5,13 @@
 package source
 
 import (
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -32,3 +37,39 @@ func TestLocalSourceValidate(t *testing.T) {
 		})
 	}
 }
+
+func TestLocalSourceLifeCycle(t *testing.T) {
+	_, filename, _, _ := runtime.Caller(0)
+	origPath := path.Join(filepath.Dir(filename), "fixtures/todos")
+	ls := LocalSource{OrigPath: origPath}
+	require.NoError(t, ls.Validate())
+
+	// Don't run the NPM commands in unit tests
+	// We can leave that for E2E tests
+	origOffline := os.Getenv(offlineEnvVar)
+	require.NoError(t, os.Setenv(offlineEnvVar, "true"))
+	defer require.NoError(t, os.Setenv(offlineEnvVar, origOffline))
+	require.NoError(t, ls.Fetch())
+
+	require.NotEmpty(t, ls.workingPath)
+	expected := []string{
+		"Dockerfile",
+		"package.json",
+		"build-offline-dockerfile.sh",
+		".npmrc",
+		"heartbeat.docker.yml",
+		"README.md",
+		"helpers.ts",
+		"add-remove.journey.ts",
+		"basics.journey.ts",
+	}
+	for _, file := range expected {
+		_, err := os.Stat(path.Join(ls.Workdir(), file))
+		// assert, not require, because we want to proceed to the close bit
+		assert.NoError(t, err)
+	}
+
+	require.NoError(t, ls.Close())
+	_, err := os.Stat(ls.Workdir())
+	require.True(t, os.IsNotExist(err), "Workdir %s should have been deleted", ls.Workdir())
+}

From 24ced05b3f9a583cbb51236bc190ea7163e27f97 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Sat, 30 Jan 2021 09:33:07 -0600
Subject: [PATCH 34/41] Minimize fixtures

---
 .../browser/source/fixtures/todos/.npmrc      |  1 -
 .../browser/source/fixtures/todos/Dockerfile  | 30 -------------------
 .../browser/source/fixtures/todos/README.md   | 30 -------------------
 .../todos/build-offline-dockerfile.sh         |  7 -----
 .../fixtures/todos/heartbeat.docker.yml       | 12 --------
 .../monitors/browser/source/local_test.go     |  9 ++----
 6 files changed, 2 insertions(+), 87 deletions(-)
 delete mode 100644 x-pack/heartbeat/monitors/browser/source/fixtures/todos/.npmrc
 delete mode 100644 x-pack/heartbeat/monitors/browser/source/fixtures/todos/Dockerfile
 delete mode 100644 x-pack/heartbeat/monitors/browser/source/fixtures/todos/README.md
 delete mode 100755 x-pack/heartbeat/monitors/browser/source/fixtures/todos/build-offline-dockerfile.sh
 delete mode 100644 x-pack/heartbeat/monitors/browser/source/fixtures/todos/heartbeat.docker.yml

diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/.npmrc b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/.npmrc
deleted file mode 100644
index 43c97e719a5..00000000000
--- a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/.npmrc
+++ /dev/null
@@ -1 +0,0 @@
-package-lock=false
diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/Dockerfile b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/Dockerfile
deleted file mode 100644
index 4b3c80291dd..00000000000
--- a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/Dockerfile
+++ /dev/null
@@ -1,30 +0,0 @@
-# This Dockerfile illustrates the usage of synthetics in an air gapped environment, where 
-# public internet access is not available. In this situation you'll want to create
-# a custom image using our official docker image as a base.
-
-# Use our synthetics image as a base.
-ARG STACK_VERSION=latest
-FROM docker.elastic.co/experimental/synthetics:${STACK_VERSION}-synthetics
-# Use the line below if you're using a custom base image, usually only for
-# developers
-#FROM heartbeat-synthetics-local
-
-# This flag variable will prevent heartbeat from running `npm i` or
-# similar commands that depend on an internet connection.
-# We'll have to do that work now, when we bake the image.
-ENV ELASTIC_SYNTHETICS_OFFLINE=true
-
-# Copy our heartbeat config directly to the image.
-# This could be done as a shared mount instead, but if we're
-# baking an image anyway, this may be something that may be easier
-# to do in this manner.
-COPY heartbeat.docker.yml /usr/share/heartbeat/heartbeat.yml
-
-RUN mkdir -p $SUITES_DIR/todos
-# Copy your custom synthetics tests into a folder on the image
-COPY . $SUITES_DIR/todos/
-
-# Install NPM deps locally on this image
-# Please note that it's important to run both `npm install` AND `npm install playwright`
-# for more see this issue: https://github.com/microsoft/playwright/issues/3712
-RUN cd $SUITES_DIR/todos && npm install
diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/README.md b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/README.md
deleted file mode 100644
index de30b63da28..00000000000
--- a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/README.md
+++ /dev/null
@@ -1,30 +0,0 @@
-# Test Vue.js Examples
-
-This suite tests the examples that ship with the open source Vue.js project.
-
-You can run the test suites in two ways
-
-## Running via `@elastic/synthetics`
-
-We can invoke the Synthetics runner from the CLI using the below steps
-
-```sh
-// Install the dependencies
-npm install
-
-// Invoke the runner and show test results
-npx @elastic/synthetics .
-
-```
-
-## Running via `Heartbeat`
-
-Invoke the synthetic test suites using heartbeat.
-
-```sh
-// Run the below command inside /examples/docker directory
-
-// run heartbeat which is already configured to run the todo app, you
-// can check `heartbeat.docker.yml`
-./run-build-local.sh -E output.console={}
-```
diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/build-offline-dockerfile.sh b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/build-offline-dockerfile.sh
deleted file mode 100755
index 25b57e46544..00000000000
--- a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/build-offline-dockerfile.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/sh
-# This demonstrates building and tagging a custom offline docker heartbeat image with synthetics
-# with all dependencies pre-bundled.
-
-# You'll want to run this in an environment with internet access so that NPM deps can be installed,
-# then, take the resultant image and transfer that to your air gapped network.
-docker build --build-arg STACK_VERSION=7.10.0 -t my-custom-heartbeat .
diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/heartbeat.docker.yml b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/heartbeat.docker.yml
deleted file mode 100644
index 48c52f9da7e..00000000000
--- a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/heartbeat.docker.yml
+++ /dev/null
@@ -1,12 +0,0 @@
----
-seccomp.enabled: false
-heartbeat.config.monitors:
-  path: "${path.config}/monitors.d/*.yml"
-  reload.enabled: false
-  reload.period: 5s
-
-heartbeat.synthetic_suites:
-- name: Todos
-  # SUITES_DIR is an environment var provided by the docker container
-  path: "${SUITES_DIR}/todos"
-  schedule: "@every 1m"
\ No newline at end of file
diff --git a/x-pack/heartbeat/monitors/browser/source/local_test.go b/x-pack/heartbeat/monitors/browser/source/local_test.go
index b7e6d3be290..8e66f6e619d 100644
--- a/x-pack/heartbeat/monitors/browser/source/local_test.go
+++ b/x-pack/heartbeat/monitors/browser/source/local_test.go
@@ -47,18 +47,13 @@ func TestLocalSourceLifeCycle(t *testing.T) {
 	// Don't run the NPM commands in unit tests
 	// We can leave that for E2E tests
 	origOffline := os.Getenv(offlineEnvVar)
-	require.NoError(t, os.Setenv(offlineEnvVar, "true"))
-	defer require.NoError(t, os.Setenv(offlineEnvVar, origOffline))
+	os.Setenv(offlineEnvVar, "true")
+	defer os.Setenv(offlineEnvVar, origOffline)
 	require.NoError(t, ls.Fetch())
 
 	require.NotEmpty(t, ls.workingPath)
 	expected := []string{
-		"Dockerfile",
 		"package.json",
-		"build-offline-dockerfile.sh",
-		".npmrc",
-		"heartbeat.docker.yml",
-		"README.md",
 		"helpers.ts",
 		"add-remove.journey.ts",
 		"basics.journey.ts",

From ebcfb0329effe65562224d317ba278bf80750957 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Sat, 30 Jan 2021 09:37:02 -0600
Subject: [PATCH 35/41] Update go.sum

---
 go.sum | 2 --
 1 file changed, 2 deletions(-)

diff --git a/go.sum b/go.sum
index 8c08c87c2ed..d23ec2ecaf6 100644
--- a/go.sum
+++ b/go.sum
@@ -247,8 +247,6 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 h1:DW6WrARxK5J+o8uAKCiACi5wy9EK1UzrsCpGBPsKHAA=
 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
-github.com/elastic/beats v1.3.1 h1:hHzUBHCo3HJHxnRVwa0XlfZoxmP8Rxp7GQ0ZVELGY4A=
-github.com/elastic/beats v7.6.2+incompatible h1:jHdLv83KURaqWUC6f55iMyVP6LYZrgElfeqxKWcskVE=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
 github.com/elastic/ecs v1.6.0 h1:8NmgfnsjmKXh9hVsK3H2tZtfUptepNc3msJOAynhtmc=

From 34469bf748f6a022ac2fe973ccfa0e9d78da514c Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Sat, 30 Jan 2021 16:11:39 -0600
Subject: [PATCH 36/41] Fix linter errors

---
 heartbeat/monitors/active/http/http.go |  2 +-
 heartbeat/monitors/active/icmp/icmp.go |  2 +-
 heartbeat/monitors/active/tcp/tcp.go   |  2 +-
 heartbeat/monitors/mocks_test.go       | 17 ++++++++---------
 4 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/heartbeat/monitors/active/http/http.go b/heartbeat/monitors/active/http/http.go
index c01b83f44ee..214ed8519d8 100644
--- a/heartbeat/monitors/active/http/http.go
+++ b/heartbeat/monitors/active/http/http.go
@@ -116,7 +116,7 @@ func create(
 		js[i] = wrappers.WithURLField(u, job)
 	}
 
-	return plugin.Plugin{js, nil, len(config.Hosts)}, nil
+	return plugin.Plugin{Jobs: js, Close: nil, Endpoints: len(config.Hosts)}, nil
 }
 
 func newRoundTripper(config *Config, tls *tlscommon.TLSConfig) (*http.Transport, error) {
diff --git a/heartbeat/monitors/active/icmp/icmp.go b/heartbeat/monitors/active/icmp/icmp.go
index 4b57c103deb..1315a1dddf0 100644
--- a/heartbeat/monitors/active/icmp/icmp.go
+++ b/heartbeat/monitors/active/icmp/icmp.go
@@ -113,7 +113,7 @@ func (jf *jobFactory) makePlugin() (plugin2 plugin.Plugin, err error) {
 		j = append(j, wrappers.WithURLField(u, job))
 	}
 
-	return plugin.Plugin{j, nil, len(jf.config.Hosts)}, nil
+	return plugin.Plugin{Jobs: j, Close: nil, Endpoints: len(jf.config.Hosts)}, nil
 }
 
 func (jf *jobFactory) pingIPFactory(config *Config) func(*net.IPAddr) jobs.Job {
diff --git a/heartbeat/monitors/active/tcp/tcp.go b/heartbeat/monitors/active/tcp/tcp.go
index dda96b85088..d0c6fd2b8a2 100644
--- a/heartbeat/monitors/active/tcp/tcp.go
+++ b/heartbeat/monitors/active/tcp/tcp.go
@@ -69,7 +69,7 @@ func createWithResolver(
 		return plugin.Plugin{}, err
 	}
 
-	return plugin.Plugin{js, nil, len(jc.endpoints)}, nil
+	return plugin.Plugin{Jobs: js, Close: nil, Endpoints: len(jc.endpoints)}, nil
 }
 
 // jobFactory is where most of the logic here lives. It provides a common context around
diff --git a/heartbeat/monitors/mocks_test.go b/heartbeat/monitors/mocks_test.go
index b818234720d..3e51387d28e 100644
--- a/heartbeat/monitors/mocks_test.go
+++ b/heartbeat/monitors/mocks_test.go
@@ -127,7 +127,7 @@ func mockEventCustomFields() map[string]interface{} {
 	return common.MapStr{"foo": "bar"}
 }
 
-func createMockJob(name string, cfg *common.Config) ([]jobs.Job, error) {
+func createMockJob() ([]jobs.Job, error) {
 	j := jobs.MakeSimpleJob(func(event *beat.Event) error {
 		eventext.MergeEventFields(event, mockEventCustomFields())
 		return nil
@@ -143,9 +143,9 @@ func mockPluginBuilder() (p plugin.PluginFactory, built *atomic.Int, closed *ato
 	closed = atomic.NewInt(0)
 
 	return plugin.PluginFactory{
-			"test",
-			[]string{"testAlias"},
-			func(s string, config *common.Config) (plugin.Plugin, error) {
+			Name:    "test",
+			Aliases: []string{"testAlias"},
+			Builder: func(s string, config *common.Config) (plugin.Plugin, error) {
 				built.Inc()
 				// Declare a real config block with a required attr so we can see what happens when it doesn't work
 				unpacked := struct {
@@ -155,15 +155,14 @@ func mockPluginBuilder() (p plugin.PluginFactory, built *atomic.Int, closed *ato
 				if err != nil {
 					return plugin.Plugin{}, err
 				}
-				c := common.Config{}
-				j, err := createMockJob("test", &c)
-				close := func() error {
+				j, err := createMockJob()
+				closer := func() error {
 					closed.Inc()
 					return nil
 				}
-				return plugin.Plugin{j, close, 1}, err
+				return plugin.Plugin{Jobs: j, Close: closer, Endpoints: 1}, err
 			},
-			plugin.NewPluginCountersRecorder("test", reg)},
+			Stats: plugin.NewPluginCountersRecorder("test", reg)},
 		built,
 		closed
 }

From 8ed3ad1ce2e62b66ba99bf064962c271385e2c1d Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Tue, 2 Feb 2021 14:58:41 -0600
Subject: [PATCH 37/41] Fix local source behavior to remove node_modules

---
 .../fixtures/todos/node_modules/test.json     |  0
 .../monitors/browser/source/local.go          | 47 +++++++++++++------
 .../monitors/browser/source/local_test.go     |  1 +
 3 files changed, 33 insertions(+), 15 deletions(-)
 create mode 100644 x-pack/heartbeat/monitors/browser/source/fixtures/todos/node_modules/test.json

diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/node_modules/test.json b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/node_modules/test.json
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/x-pack/heartbeat/monitors/browser/source/local.go b/x-pack/heartbeat/monitors/browser/source/local.go
index d697f7b970f..71e9f192462 100644
--- a/x-pack/heartbeat/monitors/browser/source/local.go
+++ b/x-pack/heartbeat/monitors/browser/source/local.go
@@ -9,6 +9,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 
 	"github.com/elastic/beats/v7/libbeat/logp"
@@ -61,7 +62,10 @@ func (l *LocalSource) Fetch() (err error) {
 	}
 	defer func() {
 		if err != nil {
-			l.Close() // cleanup the dir if this function returns an err
+			err := l.Close() // cleanup the dir if this function returns an err
+			if err != nil {
+				logp.Warn("could not cleanup dir: %s", err)
+			}
 		}
 	}()
 
@@ -70,27 +74,40 @@ func (l *LocalSource) Fetch() (err error) {
 		return fmt.Errorf("could not copy suite: %w", err)
 	}
 
-	dir, err := getSuiteDir(l.workingPath)
+	dir, err := getAbsoluteSuiteDir(l.workingPath)
 	if err != nil {
 		return err
 	}
 
 	if !offline() {
-		// Ensure all deps installed
-		err = runSimpleCommand(exec.Command("npm", "install"), dir)
-		if err != nil {
-			return err
-		}
+		err = setupOnlineDir(dir)
+		return err
+	}
+
+	return nil
+}
 
-		// Update playwright, needs to run separately to ensure post-install hook is run that downloads
-		// chrome. See https://github.com/microsoft/playwright/issues/3712
-		err = runSimpleCommand(exec.Command("npm", "install", "playwright-chromium"), dir)
+// setupOnlineDir is run in environments with internet access and attempts to make sure the node env
+// is setup correctly.
+func setupOnlineDir(dir string) (err error) {
+	// If we're not offline remove the node_modules folder so we can do a fresh install, this minimizes
+	// issues with dependencies being broken.
+	modDir := path.Join(dir, "node_modules")
+	_, statErr := os.Stat(modDir)
+	if os.IsExist(statErr) {
+		err := os.RemoveAll(modDir)
 		if err != nil {
-			return err
+			return fmt.Errorf("could not remove node_modules from '%s': %w", dir, err)
 		}
 	}
 
-	return nil
+	// Ensure all deps installed
+	err = runSimpleCommand(exec.Command("npm", "install"), dir)
+	if err != nil {
+		return err
+	}
+
+	return err
 }
 
 func (l *LocalSource) Workdir() string {
@@ -105,12 +122,12 @@ func (l *LocalSource) Close() error {
 	return nil
 }
 
-func getSuiteDir(suiteFile string) (string, error) {
-	path, err := filepath.Abs(suiteFile)
+func getAbsoluteSuiteDir(suiteFile string) (string, error) {
+	absPath, err := filepath.Abs(suiteFile)
 	if err != nil {
 		return "", err
 	}
-	stat, err := os.Stat(path)
+	stat, err := os.Stat(absPath)
 	if err != nil {
 		return "", err
 	}
diff --git a/x-pack/heartbeat/monitors/browser/source/local_test.go b/x-pack/heartbeat/monitors/browser/source/local_test.go
index 8e66f6e619d..137cb001aae 100644
--- a/x-pack/heartbeat/monitors/browser/source/local_test.go
+++ b/x-pack/heartbeat/monitors/browser/source/local_test.go
@@ -53,6 +53,7 @@ func TestLocalSourceLifeCycle(t *testing.T) {
 
 	require.NotEmpty(t, ls.workingPath)
 	expected := []string{
+		"node_modules",
 		"package.json",
 		"helpers.ts",
 		"add-remove.journey.ts",

From 400275538b2eeb726a40cef4aae70260f95f1da2 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Tue, 2 Feb 2021 16:22:17 -0600
Subject: [PATCH 38/41] Improve test coverage

---
 heartbeat/monitors/active/tcp/tcp.go          |  3 +-
 .../source/fixtures/todos/package.json        |  1 -
 .../monitors/browser/source/local.go          |  8 +-
 .../monitors/browser/source/local_test.go     |  5 +-
 .../monitors/browser/source/offline.go        | 31 +++++++
 .../monitors/browser/suite_runner_test.go     | 39 ++++++++-
 .../browser/synthexec/execmultiplexer.go      |  1 +
 .../browser/synthexec/execmultiplexer_test.go | 85 +++++++++++++++++++
 .../monitors/browser/synthexec/synthtypes.go  |  1 +
 9 files changed, 160 insertions(+), 14 deletions(-)
 create mode 100644 x-pack/heartbeat/monitors/browser/source/offline.go
 create mode 100644 x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer_test.go

diff --git a/heartbeat/monitors/active/tcp/tcp.go b/heartbeat/monitors/active/tcp/tcp.go
index d0c6fd2b8a2..aeaebe79a55 100644
--- a/heartbeat/monitors/active/tcp/tcp.go
+++ b/heartbeat/monitors/active/tcp/tcp.go
@@ -23,14 +23,13 @@ import (
 	"net/url"
 	"time"
 
-	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
-
 	"github.com/elastic/beats/v7/heartbeat/eventext"
 	"github.com/elastic/beats/v7/heartbeat/look"
 	"github.com/elastic/beats/v7/heartbeat/monitors"
 	"github.com/elastic/beats/v7/heartbeat/monitors/active/dialchain"
 	"github.com/elastic/beats/v7/heartbeat/monitors/active/dialchain/tlsmeta"
 	"github.com/elastic/beats/v7/heartbeat/monitors/jobs"
+	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
 	"github.com/elastic/beats/v7/heartbeat/monitors/wrappers"
 	"github.com/elastic/beats/v7/heartbeat/reason"
 	"github.com/elastic/beats/v7/libbeat/beat"
diff --git a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/package.json b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/package.json
index 4de5c6a1414..5972cf95b98 100644
--- a/x-pack/heartbeat/monitors/browser/source/fixtures/todos/package.json
+++ b/x-pack/heartbeat/monitors/browser/source/fixtures/todos/package.json
@@ -3,7 +3,6 @@
   "private": true,
   "description": "This suite tests the examples that ship with the open source Vue.js project.",
   "scripts": {},
-  "license": "MIT",
   "dependencies": {
     "@elastic/synthetics": "*",
     "playwright-core": "=1.6.2"
diff --git a/x-pack/heartbeat/monitors/browser/source/local.go b/x-pack/heartbeat/monitors/browser/source/local.go
index 71e9f192462..61ec0a1d255 100644
--- a/x-pack/heartbeat/monitors/browser/source/local.go
+++ b/x-pack/heartbeat/monitors/browser/source/local.go
@@ -46,12 +46,6 @@ func (l *LocalSource) Validate() error {
 	return nil
 }
 
-var offlineEnvVar = "ELASTIC_SYNTHETICS_OFFLINE"
-
-func offline() bool {
-	return os.Getenv(offlineEnvVar) == "true"
-}
-
 func (l *LocalSource) Fetch() (err error) {
 	if l.workingPath != "" {
 		return nil
@@ -79,7 +73,7 @@ func (l *LocalSource) Fetch() (err error) {
 		return err
 	}
 
-	if !offline() {
+	if !Offline() {
 		err = setupOnlineDir(dir)
 		return err
 	}
diff --git a/x-pack/heartbeat/monitors/browser/source/local_test.go b/x-pack/heartbeat/monitors/browser/source/local_test.go
index 137cb001aae..7deeb7bbf25 100644
--- a/x-pack/heartbeat/monitors/browser/source/local_test.go
+++ b/x-pack/heartbeat/monitors/browser/source/local_test.go
@@ -46,9 +46,8 @@ func TestLocalSourceLifeCycle(t *testing.T) {
 
 	// Don't run the NPM commands in unit tests
 	// We can leave that for E2E tests
-	origOffline := os.Getenv(offlineEnvVar)
-	os.Setenv(offlineEnvVar, "true")
-	defer os.Setenv(offlineEnvVar, origOffline)
+	GoOffline()
+	defer GoOnline()
 	require.NoError(t, ls.Fetch())
 
 	require.NotEmpty(t, ls.workingPath)
diff --git a/x-pack/heartbeat/monitors/browser/source/offline.go b/x-pack/heartbeat/monitors/browser/source/offline.go
new file mode 100644
index 00000000000..0347958a812
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/source/offline.go
@@ -0,0 +1,31 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package source
+
+import "os"
+
+var offlineEnvVar = "ELASTIC_SYNTHETICS_OFFLINE"
+
+// Offline checks whether sources should act in offline mode, where
+// calls to NPM are forbidden.
+func Offline() bool {
+	return os.Getenv(offlineEnvVar) == "true"
+}
+
+// GoOffline switches our current state to offline. Primarily for tests.
+func GoOffline() {
+	e := os.Setenv(offlineEnvVar, "true")
+	if e != nil {
+		panic("could not set offline env var!")
+	}
+}
+
+// GoOffline switches our current state to offline. Primarily for tests.
+func GoOnline() {
+	e := os.Setenv(offlineEnvVar, "false")
+	if e != nil {
+		panic("could not set offline env var!")
+	}
+}
diff --git a/x-pack/heartbeat/monitors/browser/suite_runner_test.go b/x-pack/heartbeat/monitors/browser/suite_runner_test.go
index 0287b5276d9..9f9448440cc 100644
--- a/x-pack/heartbeat/monitors/browser/suite_runner_test.go
+++ b/x-pack/heartbeat/monitors/browser/suite_runner_test.go
@@ -5,6 +5,9 @@
 package browser
 
 import (
+	"path"
+	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -13,6 +16,40 @@ import (
 	"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/source"
 )
 
+func TestValidLocal(t *testing.T) {
+	_, filename, _, _ := runtime.Caller(0)
+	path := path.Join(filepath.Dir(filename), "source/fixtures/todos")
+	testParams := map[string]interface{}{
+		"key1": "value1",
+		"key2": "value2",
+	}
+	cfg := common.MustNewConfigFrom(common.MapStr{
+		"name":   "My Name",
+		"id":     "myId",
+		"params": testParams,
+		"source": common.MapStr{
+			"local": common.MapStr{
+				"path": path,
+			},
+		},
+	})
+	s, e := NewSuite(cfg)
+	require.NoError(t, e)
+	require.NotNil(t, s)
+	_, ok := s.InlineSource()
+	require.False(t, ok)
+
+	source.GoOffline()
+	defer source.GoOnline()
+	require.NoError(t, s.Fetch())
+	defer require.NoError(t, s.Close())
+	require.Regexp(t, "\\w{1,}", s.Workdir())
+	require.Equal(t, testParams, s.Params())
+
+	e = s.Close()
+	require.NoError(t, e)
+}
+
 func TestValidInline(t *testing.T) {
 	script := "a script"
 	testParams := map[string]interface{}{
@@ -34,7 +71,7 @@ func TestValidInline(t *testing.T) {
 	require.NotNil(t, s)
 	sSrc, ok := s.InlineSource()
 	require.True(t, ok)
-	require.Equal(t, script, script, sSrc)
+	require.Equal(t, script, sSrc)
 	require.Equal(t, "", s.Workdir())
 	require.Equal(t, testParams, s.Params())
 
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go b/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go
index b1a92aa8247..68e627b1cf7 100644
--- a/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer.go
@@ -29,6 +29,7 @@ func (e ExecMultiplexer) writeSynthEvent(se *SynthEvent) {
 
 	if se.Type == "journey/start" {
 		e.currentJourney.Store(true)
+		e.eventCounter.Store(-1)
 	}
 	hasCurrentJourney := e.currentJourney.Load()
 	if se.Type == "journey/end" {
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer_test.go b/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer_test.go
new file mode 100644
index 00000000000..6cbc34b6889
--- /dev/null
+++ b/x-pack/heartbeat/monitors/browser/synthexec/execmultiplexer_test.go
@@ -0,0 +1,85 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package synthexec
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestExecMultiplexer(t *testing.T) {
+	em := NewExecMultiplexer()
+
+	// Generate three fake journeys with three fake steps
+	var testJourneys []*Journey
+	var testEvents []*SynthEvent
+	time := float64(0)
+	for jIdx := 0; jIdx < 3; jIdx++ {
+		time++ // fake time to make events seem spaced out
+		journey := &Journey{
+			Name: fmt.Sprintf("J%d", jIdx),
+			Id:   fmt.Sprintf("j-%d", jIdx),
+		}
+		testJourneys = append(testJourneys, journey)
+		testEvents = append(testEvents, &SynthEvent{
+			Journey:              journey,
+			Type:                 "journey/start",
+			TimestampEpochMicros: time,
+		})
+
+		for sIdx := 0; sIdx < 3; sIdx++ {
+			step := &Step{
+				Name:  fmt.Sprintf("S%d", sIdx),
+				Index: sIdx,
+			}
+
+			testEvents = append(testEvents, &SynthEvent{
+				Journey:              journey,
+				Step:                 step,
+				TimestampEpochMicros: time,
+			})
+		}
+
+		testEvents = append(testEvents, &SynthEvent{
+			Journey:              journey,
+			Type:                 "journey/end",
+			TimestampEpochMicros: time,
+		})
+	}
+
+	// Write the test events in another go routine since writes block
+	var results []*SynthEvent
+	go func() {
+		for _, se := range testEvents {
+			em.writeSynthEvent(se)
+		}
+		em.Close()
+	}()
+
+	// Wait for all results
+Loop:
+	for {
+		select {
+		case result := <-em.synthEvents:
+			if result == nil {
+				break Loop
+			}
+			results = append(results, result)
+		}
+	}
+
+	require.Len(t, results, len(testEvents))
+	i := 0 // counter for index, resets on journey change
+	for _, se := range results {
+		require.Equal(t, i, se.index)
+		if se.Type == "journey/end" {
+			i = 0
+		} else {
+			i++
+		}
+	}
+}
diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go
index de0c76e1401..d612529844e 100644
--- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go
+++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go
@@ -37,6 +37,7 @@ func (se SynthEvent) ToMap() (m common.MapStr) {
 			"type":            se.Type,
 			"package_version": se.PackageVersion,
 			"payload":         se.Payload,
+			"index":           se.index,
 		},
 	}
 	if se.Blob != "" {

From b7c8d192050e5b76f4887999f771505cefddb133 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Tue, 2 Feb 2021 16:27:04 -0600
Subject: [PATCH 39/41] Add changelog

---
 CHANGELOG.next.asciidoc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 237a9c75e57..0e00c3c6539 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -104,6 +104,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Heartbeat*
 - Adds negative body match. {pull}20728[20728]
+- Refactor synthetics configuration to new syntax. {pull}23467[23467]
 
 *Journalbeat*
 

From 6e1a49b7c61ddd398cc990f3193f1e0fba8d7451 Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Wed, 3 Feb 2021 08:29:41 -0600
Subject: [PATCH 40/41] Update changelog

---
 CHANGELOG.next.asciidoc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 0e00c3c6539..a4a21478967 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -834,7 +834,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Heartbeat*
 
 - Add mime type detection for http responses. {pull}22976[22976]
-- Copy suite directories for synthetic checks to tmp dir before running. {pull}23347[23347]
 - Bundle synthetics deps with heartbeat docker image. {pull}23274[23274]
 
 *Journalbeat*

From 8248900352987841a78e9c9d30e7cd22ee3c3dad Mon Sep 17 00:00:00 2001
From: Andrew Cholakian <andrew@andrewvc.com>
Date: Wed, 17 Feb 2021 07:47:16 -0600
Subject: [PATCH 41/41] Incorporate PR feedback

---
 heartbeat/monitors/factory.go    | 3 ++-
 heartbeat/monitors/mocks_test.go | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/heartbeat/monitors/factory.go b/heartbeat/monitors/factory.go
index cdd903f0759..7e4da00f82d 100644
--- a/heartbeat/monitors/factory.go
+++ b/heartbeat/monitors/factory.go
@@ -18,8 +18,9 @@
 package monitors
 
 import (
-	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
 	"fmt"
+
+	"github.com/elastic/beats/v7/heartbeat/monitors/plugin"
 	"github.com/elastic/beats/v7/heartbeat/scheduler"
 	"github.com/elastic/beats/v7/libbeat/beat"
 	"github.com/elastic/beats/v7/libbeat/cfgfile"
diff --git a/heartbeat/monitors/mocks_test.go b/heartbeat/monitors/mocks_test.go
index 3e51387d28e..fecf8a53f59 100644
--- a/heartbeat/monitors/mocks_test.go
+++ b/heartbeat/monitors/mocks_test.go
@@ -136,11 +136,11 @@ func createMockJob() ([]jobs.Job, error) {
 	return []jobs.Job{j}, nil
 }
 
-func mockPluginBuilder() (p plugin.PluginFactory, built *atomic.Int, closed *atomic.Int) {
+func mockPluginBuilder() (plugin.PluginFactory, *atomic.Int, *atomic.Int) {
 	reg := monitoring.NewRegistry()
 
-	built = atomic.NewInt(0)
-	closed = atomic.NewInt(0)
+	built := atomic.NewInt(0)
+	closed := atomic.NewInt(0)
 
 	return plugin.PluginFactory{
 			Name:    "test",