feat: support running k8s autodiscover suite for Beats PRs and local …

…repositories (#1115)

* chore: add license

* chore: initialise configurations before test suite

* chore: use timeout_factor from env

* fix: tell kind to skip pulling beats images

* chore: add a method to load images into kind

* feat: support running k8s autodiscover for Beats PRs or local filesystem

* chore: add license header

* chore: expose logger and use it, simplifying initialisation

* fix: only run APM services for local APM environment

* Revert "chore: expose logger and use it, simplifying initialisation"

This reverts commit a89325c.

* chore: log scenario name

* fix: always cache beat version for podName

* chore: reduce log level

Co-authored-by: Adam Stokes <51892+adam-stokes@users.noreply.github.com>

e2e/_suites/helm/helm_charts_test.go

@@ -675,7 +675,8 @@ func InitializeHelmChartTestSuite(ctx *godog.TestSuiteContext) {
 		suiteContext = apm.ContextWithSpan(suiteContext, suiteParentSpan)
 		defer suiteParentSpan.End()
 
-		if elasticAPMActive {
+		elasticAPMEnvironment := shell.GetEnv("ELASTIC_APM_ENVIRONMENT", "ci")
+		if elasticAPMActive && elasticAPMEnvironment == "local" {
 			serviceManager := compose.NewServiceManager()
 
 			env := map[string]string{

e2e/_suites/kubernetes-autodiscover/autodiscover_test.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package main
 
 import (
@@ -18,14 +22,28 @@ import (
 	messages "github.com/cucumber/messages-go/v10"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/elastic/e2e-testing/cli/config"
+	"github.com/elastic/e2e-testing/internal/common"
+	"github.com/elastic/e2e-testing/internal/docker"
 	"github.com/elastic/e2e-testing/internal/kubernetes"
 	"github.com/elastic/e2e-testing/internal/shell"
 	"github.com/elastic/e2e-testing/internal/utils"
 )
 
+var beatVersions = map[string]string{}
+
 const defaultBeatVersion = "8.0.0-SNAPSHOT"
-const defaultEventsWaitTimeout = 120 * time.Second
-const defaultDeployWaitTimeout = 120 * time.Second
+
+var defaultEventsWaitTimeout = 60 * time.Second
+var defaultDeployWaitTimeout = 60 * time.Second
+
+func init() {
+	// initialise timeout factor
+	common.TimeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", common.TimeoutFactor)
+
+	defaultEventsWaitTimeout = defaultEventsWaitTimeout * time.Duration(common.TimeoutFactor)
+	defaultDeployWaitTimeout = defaultDeployWaitTimeout * time.Duration(common.TimeoutFactor)
+}
 
 type podsManager struct {
 	kubectl kubernetes.Control
@@ -35,6 +53,11 @@ type podsManager struct {
 func (m *podsManager) executeTemplateFor(podName string, writer io.Writer, options []string) error {
 	path := filepath.Join("testdata/templates", sanitizeName(podName)+".yml.tmpl")
 
+	err := m.configureDockerImage(podName)
+	if err != nil {
+		return err
+	}
+
 	usedOptions := make(map[string]bool)
 	funcs := template.FuncMap{
 		"option": func(o string) bool {
@@ -50,7 +73,7 @@ func (m *podsManager) executeTemplateFor(podName string, writer io.Writer, optio
 			return utils.GetDockerNamespaceEnvVar("beats")
 		},
 		"beats_version": func() string {
-			return shell.GetEnv("GITHUB_CHECK_SHA1", shell.GetEnv("BEAT_VERSION", defaultBeatVersion))
+			return beatVersions[podName]
 		},
 		"namespace": func() string {
 			return m.kubectl.Namespace
@@ -86,6 +109,64 @@ func (m *podsManager) executeTemplateFor(podName string, writer io.Writer, optio
 	return nil
 }
 
+func (m *podsManager) configureDockerImage(podName string) error {
+	if podName != "filebeat" && podName != "heartbeat" && podName != "metricbeat" {
+		log.Debugf("Not processing custom binaries for pod: %s. Only [filebeat, heartbeat, metricbeat] will be processed", podName)
+		return nil
+	}
+
+	// we are caching the versions by pod to avoid downloading and loading/tagging the Docker image multiple times
+	if beatVersions[podName] != "" {
+		log.Tracef("The beat version was already loaded: %s", beatVersions[podName])
+		return nil
+	}
+
+	beatVersion := shell.GetEnv("BEAT_VERSION", defaultBeatVersion)
+
+	useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS")
+	beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "")
+	if useCISnapshots || beatsLocalPath != "" {
+		log.Debugf("Configuring Docker image for %s", podName)
+
+		// this method will detect if the GITHUB_CHECK_SHA1 variable is set
+		artifactName := utils.BuildArtifactName(podName, beatVersion, defaultBeatVersion, "linux", "amd64", "tar.gz", true)
+
+		imagePath, err := utils.FetchBeatsBinary(artifactName, podName, beatVersion, defaultBeatVersion, common.TimeoutFactor, true)
+		if err != nil {
+			return err
+		}
+
+		// load the TAR file into the docker host as a Docker image
+		err = docker.LoadImage(imagePath)
+		if err != nil {
+			return err
+		}
+
+		beatVersion = beatVersion + "-amd64"
+
+		// tag the image with the proper docker tag, including platform
+		err = docker.TagImage(
+			"docker.elastic.co/beats/"+podName+":"+defaultBeatVersion,
+			"docker.elastic.co/observability-ci/"+podName+":"+beatVersion,
+		)
+		if err != nil {
+			return err
+		}
+
+		// load PR image into kind
+		err = cluster.LoadImage(m.ctx, "docker.elastic.co/observability-ci/"+podName+":"+beatVersion)
+		if err != nil {
+			return err
+		}
+
+	}
+
+	log.Tracef("Caching beat version '%s' for %s", beatVersion, podName)
+	beatVersions[podName] = beatVersion
+
+	return nil
+}
+
 func (m *podsManager) isDeleted(podName string, options []string) error {
 	var buf bytes.Buffer
 	err := m.executeTemplateFor(podName, &buf, options)
@@ -388,6 +469,9 @@ func InitializeTestSuite(ctx *godog.TestSuiteContext) {
 	log.DeferExitHandler(cancel)
 
 	ctx.BeforeSuite(func() {
+		// init logger
+		config.Init()
+
 		err := cluster.Initialize(suiteContext, "testdata/kind.yml")
 		if err != nil {
 			log.WithError(err).Fatal("Failed to initialize cluster")
@@ -409,10 +493,10 @@ func InitializeScenario(ctx *godog.ScenarioContext) {
 
 	var kubectl kubernetes.Control
 	var pods podsManager
-	ctx.BeforeScenario(func(*messages.Pickle) {
+	ctx.BeforeScenario(func(p *messages.Pickle) {
 		kubectl = cluster.Kubectl().WithNamespace(scenarioCtx, "")
 		if kubectl.Namespace != "" {
-			log.Debugf("Running scenario in namespace: %s", kubectl.Namespace)
+			log.Debugf("Running scenario %s in namespace: %s", p.Name, kubectl.Namespace)
 		}
 		pods.kubectl = kubectl
 		pods.ctx = scenarioCtx

e2e/_suites/kubernetes-autodiscover/testdata/templates/filebeat.yml.tmpl

@@ -45,6 +45,7 @@ spec:
       containers:
       - name: filebeat
         image: docker.elastic.co/{{ beats_namespace }}/filebeat:{{ beats_version }}
+        imagePullPolicy: IfNotPresent
         args: [
           "-c", "/etc/filebeat.yml",
           "-e",

e2e/_suites/kubernetes-autodiscover/testdata/templates/heartbeat.yml.tmpl

@@ -64,6 +64,7 @@ spec:
       containers:
       - name: heartbeat
         image: docker.elastic.co/{{ beats_namespace }}/heartbeat:{{ beats_version }}
+        imagePullPolicy: IfNotPresent
         args: [
           "-c", "/etc/heartbeat.yml",
           "-e",

e2e/_suites/kubernetes-autodiscover/testdata/templates/metricbeat.yml.tmpl

@@ -41,6 +41,7 @@ spec:
       containers:
       - name: metricbeat
         image: docker.elastic.co/{{ beats_namespace }}/metricbeat:{{ beats_version }}
+        imagePullPolicy: IfNotPresent
         args: [
           "-c", "/etc/metricbeat.yml",
           "-e",

e2e/_suites/metricbeat/metricbeat_test.go

@@ -281,7 +281,8 @@ func InitializeMetricbeatTestSuite(ctx *godog.TestSuiteContext) {
 			}).Fatal("The Elasticsearch cluster could not get the healthy status")
 		}
 
-		if elasticAPMActive {
+		elasticAPMEnvironment := shell.GetEnv("ELASTIC_APM_ENVIRONMENT", "ci")
+		if elasticAPMActive && elasticAPMEnvironment == "local" {
 			steps.AddAPMServicesForInstrumentation(suiteContext, "metricbeat", stackVersion, true, env)
 		}
 	})

internal/kubernetes/kubernetes.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package kubernetes
 
 import (
@@ -183,3 +187,27 @@ func (c *Cluster) Cleanup(ctx context.Context) {
 		}
 	}
 }
+
+// LoadImage loads a Docker image into Kind runtime, using it fully qualified name.
+// It does not check cluster availability because a pull error could be present in the pod,
+// which will need the load of the requested image, causing a chicken-egg error.
+func (c *Cluster) LoadImage(ctx context.Context, image string) error {
+	shell.CheckInstalledSoftware("kind")
+
+	loadArgs := []string{"load", "docker-image", image}
+	// default cluster name is equals to 'kind'
+	if c.kindName != "" {
+		loadArgs = append(loadArgs, "--name", c.kindName)
+	}
+
+	result, err := shell.Execute(ctx, ".", "kind", loadArgs...)
+	if err != nil {
+		log.WithError(err).Fatal("Failed to load archive into kind")
+	}
+	log.WithFields(log.Fields{
+		"image":  image,
+		"result": result,
+	}).Info("Image has been loaded into Kind runtime")
+
+	return nil
+}