From f2584148fac8ef4bfb7d919cd2f35abcb6ed75d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 3 May 2021 16:05:14 +0200 Subject: [PATCH] feat: support running k8s autodiscover suite for Beats PRs and local repositories (#1115) * chore: add license * chore: initialise configurations before test suite * chore: use timeout_factor from env * fix: tell kind to skip pulling beats images * chore: add a method to load images into kind * feat: support running k8s autodiscover for Beats PRs or local filesystem * chore: add license header * chore: expose logger and use it, simplifying initialisation * fix: only run APM services for local APM environment * Revert "chore: expose logger and use it, simplifying initialisation" This reverts commit a89325c41b0063446154459772949291d8086367. * chore: log scenario name * fix: always cache beat version for podName * chore: reduce log level Co-authored-by: Adam Stokes <51892+adam-stokes@users.noreply.github.com> --- e2e/_suites/helm/helm_charts_test.go | 3 +- .../autodiscover_test.go | 94 ++++++++++++++++++- .../testdata/templates/filebeat.yml.tmpl | 1 + .../testdata/templates/heartbeat.yml.tmpl | 1 + .../testdata/templates/metricbeat.yml.tmpl | 1 + e2e/_suites/metricbeat/metricbeat_test.go | 3 +- internal/kubernetes/kubernetes.go | 28 ++++++ 7 files changed, 124 insertions(+), 7 deletions(-) diff --git a/e2e/_suites/helm/helm_charts_test.go b/e2e/_suites/helm/helm_charts_test.go index b0d81b35c6..38ce603d5d 100644 --- a/e2e/_suites/helm/helm_charts_test.go +++ b/e2e/_suites/helm/helm_charts_test.go @@ -675,7 +675,8 @@ func InitializeHelmChartTestSuite(ctx *godog.TestSuiteContext) { suiteContext = apm.ContextWithSpan(suiteContext, suiteParentSpan) defer suiteParentSpan.End() - if elasticAPMActive { + elasticAPMEnvironment := shell.GetEnv("ELASTIC_APM_ENVIRONMENT", "ci") + if elasticAPMActive && elasticAPMEnvironment == "local" { serviceManager := compose.NewServiceManager() env := map[string]string{ diff --git a/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go b/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go index f32d636637..667db8be18 100644 --- a/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go +++ b/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package main import ( @@ -18,14 +22,28 @@ import ( messages "github.com/cucumber/messages-go/v10" log "github.com/sirupsen/logrus" + "github.com/elastic/e2e-testing/cli/config" + "github.com/elastic/e2e-testing/internal/common" + "github.com/elastic/e2e-testing/internal/docker" "github.com/elastic/e2e-testing/internal/kubernetes" "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" ) +var beatVersions = map[string]string{} + const defaultBeatVersion = "8.0.0-SNAPSHOT" -const defaultEventsWaitTimeout = 120 * time.Second -const defaultDeployWaitTimeout = 120 * time.Second + +var defaultEventsWaitTimeout = 60 * time.Second +var defaultDeployWaitTimeout = 60 * time.Second + +func init() { + // initialise timeout factor + common.TimeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", common.TimeoutFactor) + + defaultEventsWaitTimeout = defaultEventsWaitTimeout * time.Duration(common.TimeoutFactor) + defaultDeployWaitTimeout = defaultDeployWaitTimeout * time.Duration(common.TimeoutFactor) +} type podsManager struct { kubectl kubernetes.Control @@ -35,6 +53,11 @@ type podsManager struct { func (m *podsManager) executeTemplateFor(podName string, writer io.Writer, options []string) error { path := filepath.Join("testdata/templates", sanitizeName(podName)+".yml.tmpl") + err := m.configureDockerImage(podName) + if err != nil { + return err + } + usedOptions := make(map[string]bool) funcs := template.FuncMap{ "option": func(o string) bool { @@ -50,7 +73,7 @@ func (m *podsManager) executeTemplateFor(podName string, writer io.Writer, optio return utils.GetDockerNamespaceEnvVar("beats") }, "beats_version": func() string { - return shell.GetEnv("GITHUB_CHECK_SHA1", shell.GetEnv("BEAT_VERSION", defaultBeatVersion)) + return beatVersions[podName] }, "namespace": func() string { return m.kubectl.Namespace @@ -86,6 +109,64 @@ func (m *podsManager) executeTemplateFor(podName string, writer io.Writer, optio return nil } +func (m *podsManager) configureDockerImage(podName string) error { + if podName != "filebeat" && podName != "heartbeat" && podName != "metricbeat" { + log.Debugf("Not processing custom binaries for pod: %s. Only [filebeat, heartbeat, metricbeat] will be processed", podName) + return nil + } + + // we are caching the versions by pod to avoid downloading and loading/tagging the Docker image multiple times + if beatVersions[podName] != "" { + log.Tracef("The beat version was already loaded: %s", beatVersions[podName]) + return nil + } + + beatVersion := shell.GetEnv("BEAT_VERSION", defaultBeatVersion) + + useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS") + beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") + if useCISnapshots || beatsLocalPath != "" { + log.Debugf("Configuring Docker image for %s", podName) + + // this method will detect if the GITHUB_CHECK_SHA1 variable is set + artifactName := utils.BuildArtifactName(podName, beatVersion, defaultBeatVersion, "linux", "amd64", "tar.gz", true) + + imagePath, err := utils.FetchBeatsBinary(artifactName, podName, beatVersion, defaultBeatVersion, common.TimeoutFactor, true) + if err != nil { + return err + } + + // load the TAR file into the docker host as a Docker image + err = docker.LoadImage(imagePath) + if err != nil { + return err + } + + beatVersion = beatVersion + "-amd64" + + // tag the image with the proper docker tag, including platform + err = docker.TagImage( + "docker.elastic.co/beats/"+podName+":"+defaultBeatVersion, + "docker.elastic.co/observability-ci/"+podName+":"+beatVersion, + ) + if err != nil { + return err + } + + // load PR image into kind + err = cluster.LoadImage(m.ctx, "docker.elastic.co/observability-ci/"+podName+":"+beatVersion) + if err != nil { + return err + } + + } + + log.Tracef("Caching beat version '%s' for %s", beatVersion, podName) + beatVersions[podName] = beatVersion + + return nil +} + func (m *podsManager) isDeleted(podName string, options []string) error { var buf bytes.Buffer err := m.executeTemplateFor(podName, &buf, options) @@ -388,6 +469,9 @@ func InitializeTestSuite(ctx *godog.TestSuiteContext) { log.DeferExitHandler(cancel) ctx.BeforeSuite(func() { + // init logger + config.Init() + err := cluster.Initialize(suiteContext, "testdata/kind.yml") if err != nil { log.WithError(err).Fatal("Failed to initialize cluster") @@ -409,10 +493,10 @@ func InitializeScenario(ctx *godog.ScenarioContext) { var kubectl kubernetes.Control var pods podsManager - ctx.BeforeScenario(func(*messages.Pickle) { + ctx.BeforeScenario(func(p *messages.Pickle) { kubectl = cluster.Kubectl().WithNamespace(scenarioCtx, "") if kubectl.Namespace != "" { - log.Debugf("Running scenario in namespace: %s", kubectl.Namespace) + log.Debugf("Running scenario %s in namespace: %s", p.Name, kubectl.Namespace) } pods.kubectl = kubectl pods.ctx = scenarioCtx diff --git a/e2e/_suites/kubernetes-autodiscover/testdata/templates/filebeat.yml.tmpl b/e2e/_suites/kubernetes-autodiscover/testdata/templates/filebeat.yml.tmpl index 70cfce3cb6..6bf3627d4d 100644 --- a/e2e/_suites/kubernetes-autodiscover/testdata/templates/filebeat.yml.tmpl +++ b/e2e/_suites/kubernetes-autodiscover/testdata/templates/filebeat.yml.tmpl @@ -45,6 +45,7 @@ spec: containers: - name: filebeat image: docker.elastic.co/{{ beats_namespace }}/filebeat:{{ beats_version }} + imagePullPolicy: IfNotPresent args: [ "-c", "/etc/filebeat.yml", "-e", diff --git a/e2e/_suites/kubernetes-autodiscover/testdata/templates/heartbeat.yml.tmpl b/e2e/_suites/kubernetes-autodiscover/testdata/templates/heartbeat.yml.tmpl index aa02eca6ad..88e6e9dab4 100644 --- a/e2e/_suites/kubernetes-autodiscover/testdata/templates/heartbeat.yml.tmpl +++ b/e2e/_suites/kubernetes-autodiscover/testdata/templates/heartbeat.yml.tmpl @@ -64,6 +64,7 @@ spec: containers: - name: heartbeat image: docker.elastic.co/{{ beats_namespace }}/heartbeat:{{ beats_version }} + imagePullPolicy: IfNotPresent args: [ "-c", "/etc/heartbeat.yml", "-e", diff --git a/e2e/_suites/kubernetes-autodiscover/testdata/templates/metricbeat.yml.tmpl b/e2e/_suites/kubernetes-autodiscover/testdata/templates/metricbeat.yml.tmpl index 042bb56bd7..a21a9ea43e 100644 --- a/e2e/_suites/kubernetes-autodiscover/testdata/templates/metricbeat.yml.tmpl +++ b/e2e/_suites/kubernetes-autodiscover/testdata/templates/metricbeat.yml.tmpl @@ -41,6 +41,7 @@ spec: containers: - name: metricbeat image: docker.elastic.co/{{ beats_namespace }}/metricbeat:{{ beats_version }} + imagePullPolicy: IfNotPresent args: [ "-c", "/etc/metricbeat.yml", "-e", diff --git a/e2e/_suites/metricbeat/metricbeat_test.go b/e2e/_suites/metricbeat/metricbeat_test.go index 3f4a0a00f1..7c168a8071 100644 --- a/e2e/_suites/metricbeat/metricbeat_test.go +++ b/e2e/_suites/metricbeat/metricbeat_test.go @@ -281,7 +281,8 @@ func InitializeMetricbeatTestSuite(ctx *godog.TestSuiteContext) { }).Fatal("The Elasticsearch cluster could not get the healthy status") } - if elasticAPMActive { + elasticAPMEnvironment := shell.GetEnv("ELASTIC_APM_ENVIRONMENT", "ci") + if elasticAPMActive && elasticAPMEnvironment == "local" { steps.AddAPMServicesForInstrumentation(suiteContext, "metricbeat", stackVersion, true, env) } }) diff --git a/internal/kubernetes/kubernetes.go b/internal/kubernetes/kubernetes.go index c8dad1b3a2..a050d6af05 100644 --- a/internal/kubernetes/kubernetes.go +++ b/internal/kubernetes/kubernetes.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package kubernetes import ( @@ -183,3 +187,27 @@ func (c *Cluster) Cleanup(ctx context.Context) { } } } + +// LoadImage loads a Docker image into Kind runtime, using it fully qualified name. +// It does not check cluster availability because a pull error could be present in the pod, +// which will need the load of the requested image, causing a chicken-egg error. +func (c *Cluster) LoadImage(ctx context.Context, image string) error { + shell.CheckInstalledSoftware("kind") + + loadArgs := []string{"load", "docker-image", image} + // default cluster name is equals to 'kind' + if c.kindName != "" { + loadArgs = append(loadArgs, "--name", c.kindName) + } + + result, err := shell.Execute(ctx, ".", "kind", loadArgs...) + if err != nil { + log.WithError(err).Fatal("Failed to load archive into kind") + } + log.WithFields(log.Fields{ + "image": image, + "result": result, + }).Info("Image has been loaded into Kind runtime") + + return nil +}