fix: delay checking stale agent version until it's used (#1016) (#1024)

* fix: delay checking stale agent version until it's used

* fix: use proper kibana version

* chore: handle errors in the caller when the version is not found

- in setup methods, we want to fail (log.Fatal)
- in steps, we want to simply return the error to fail the scenario

(cherry picked from commit 00a568d)

Co-authored-by: Manuel de la Peña <mdelapenya@gmail.com>

e2e/_suites/fleet/fleet.go

@@ -15,6 +15,7 @@ import (
 	"github.com/cucumber/godog"
 	"github.com/elastic/e2e-testing/cli/services"
 	curl "github.com/elastic/e2e-testing/cli/shell"
+	shell "github.com/elastic/e2e-testing/cli/shell"
 	"github.com/elastic/e2e-testing/e2e"
 	"github.com/elastic/e2e-testing/e2e/steps"
 	"github.com/google/uuid"
@@ -166,6 +167,23 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver
 	agentVersionBackup := fts.Version
 	defer func() { fts.Version = agentVersionBackup }()
 
+	agentStaleVersion = shell.GetEnv("ELASTIC_AGENT_STALE_VERSION", agentStaleVersion)
+	// check if stale version is an alias
+	v, err := e2e.GetElasticArtifactVersion(agentStaleVersion)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":   err,
+			"version": agentStaleVersion,
+		}).Error("Failed to get stale version")
+		return err
+	}
+	agentStaleVersion = v
+
+	useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS")
+	if useCISnapshots && !strings.HasSuffix(agentStaleVersion, "-SNAPSHOT") {
+		agentStaleVersion += "-SNAPSHOT"
+	}
+
 	switch version {
 	case "stale":
 		version = agentStaleVersion

e2e/_suites/fleet/ingest_manager_test.go

@@ -33,31 +33,43 @@ func setUpSuite() {
 	}
 
 	// check if base version is an alias
-	agentVersionBase = e2e.GetElasticArtifactVersion(agentVersionBase)
+	v, err := e2e.GetElasticArtifactVersion(agentVersionBase)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":   err,
+			"version": agentVersionBase,
+		}).Fatal("Failed to get agent base version, aborting")
+	}
+	agentVersionBase = v
 
 	timeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", timeoutFactor)
 	agentVersion = shell.GetEnv("BEAT_VERSION", agentVersionBase)
 
-	agentStaleVersion = shell.GetEnv("ELASTIC_AGENT_STALE_VERSION", agentStaleVersion)
-	// check if stale version is an alias
-	agentStaleVersion = e2e.GetElasticArtifactVersion(agentStaleVersion)
-
-	useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS")
-	if useCISnapshots && !strings.HasSuffix(agentStaleVersion, "-SNAPSHOT") {
-		agentStaleVersion += "-SNAPSHOT"
-	}
-
 	// check if version is an alias
-	agentVersion = e2e.GetElasticArtifactVersion(agentVersion)
+	v, err = e2e.GetElasticArtifactVersion(agentVersion)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":   err,
+			"version": agentVersion,
+		}).Fatal("Failed to get agent version, aborting")
+	}
+	agentVersion = v
 
 	stackVersion = shell.GetEnv("STACK_VERSION", stackVersion)
-	stackVersion = e2e.GetElasticArtifactVersion(stackVersion)
+	v, err = e2e.GetElasticArtifactVersion(stackVersion)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":   err,
+			"version": stackVersion,
+		}).Fatal("Failed to get stack version, aborting")
+	}
+	stackVersion = v
 
 	kibanaVersion = shell.GetEnv("KIBANA_VERSION", "")
 	if kibanaVersion == "" {
 		// we want to deploy a released version for Kibana
 		// if not set, let's use stackVersion
-		kibanaVersion = e2e.GetElasticArtifactVersion(stackVersion)
+		kibanaVersion = stackVersion
 	}
 
 	imts = IngestManagerTestSuite{
@@ -110,9 +122,10 @@ func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) {
 			"stackVersion":     stackVersion,
 		}
 
-		profileEnv["kibanaDockerNamespace"] = "observability-ci"
-		if kibanaVersion == "" {
-			profileEnv["kibanaDockerNamespace"] = "kibana"
+		profileEnv["kibanaDockerNamespace"] = "kibana"
+		if strings.HasPrefix(kibanaVersion, "pr") {
+			// because it comes from a PR
+			profileEnv["kibanaDockerNamespace"] = "observability-ci"
 		}
 
 		profile := FleetProfileName

e2e/_suites/helm/helm_charts_test.go

@@ -81,7 +81,14 @@ func setupSuite() {
 	timeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", timeoutFactor)
 
 	stackVersion = shell.GetEnv("STACK_VERSION", stackVersion)
-	stackVersion = e2e.GetElasticArtifactVersion(stackVersion)
+	v, err := e2e.GetElasticArtifactVersion(stackVersion)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":   err,
+			"version": stackVersion,
+		}).Fatal("Failed to get stack version, aborting")
+	}
+	stackVersion = v
 
 	h, err := k8s.HelmFactory(helmVersion)
 	if err != nil {

e2e/_suites/metricbeat/metricbeat_test.go

@@ -69,13 +69,27 @@ func setupSuite() {
 	}
 
 	// check if base version is an alias
-	metricbeatVersionBase = e2e.GetElasticArtifactVersion(metricbeatVersionBase)
+	v, err := e2e.GetElasticArtifactVersion(metricbeatVersionBase)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":   err,
+			"version": metricbeatVersionBase,
+		}).Fatal("Failed to get metricbeat base version, aborting")
+	}
+	metricbeatVersionBase = v
 
 	metricbeatVersion = shell.GetEnv("BEAT_VERSION", metricbeatVersionBase)
 	timeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", timeoutFactor)
 
 	stackVersion = shell.GetEnv("STACK_VERSION", stackVersion)
-	stackVersion = e2e.GetElasticArtifactVersion(stackVersion)
+	v, err = e2e.GetElasticArtifactVersion(stackVersion)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error":   err,
+			"version": stackVersion,
+		}).Fatal("Failed to get stack version, aborting")
+	}
+	stackVersion = v
 
 	serviceManager = services.NewServiceManager()
 

e2e/utils.go

@@ -194,7 +194,7 @@ func GetExponentialBackOff(elapsedTime time.Duration) *backoff.ExponentialBackOf
 // 1. Elastic's artifact repository, building the JSON path query based
 // If the version is a PR, then it will return the version without checking the artifacts API
 // i.e. GetElasticArtifactVersion("$VERSION")
-func GetElasticArtifactVersion(version string) string {
+func GetElasticArtifactVersion(version string) (string, error) {
 	exp := GetExponentialBackOff(time.Minute)
 
 	retryCount := 1
@@ -233,20 +233,16 @@ func GetElasticArtifactVersion(version string) string {
 
 	err := backoff.Retry(apiStatus, exp)
 	if err != nil {
-		log.WithFields(log.Fields{
-			"error":   err,
-			"version": version,
-		}).Fatal("Failed to get version, aborting")
-		return ""
+		return "", err
 	}
 
 	jsonParsed, err := gabs.ParseJSON([]byte(body))
 	if err != nil {
 		log.WithFields(log.Fields{
 			"error":   err,
 			"version": version,
-		}).Fatal("Could not parse the response body to retrieve the version, aborting")
-		return ""
+		}).Error("Could not parse the response body to retrieve the version")
+		return "", err
 	}
 
 	builds := jsonParsed.Path("version.builds")
@@ -259,7 +255,7 @@ func GetElasticArtifactVersion(version string) string {
 		"version": latestVersion,
 	}).Debug("Latest version for current version obtained")
 
-	return latestVersion
+	return latestVersion, nil
 }
 
 // GetElasticArtifactURL returns the URL of a released artifact, which its full name is defined in the first argument,