feat: support using Beats CI snapshots

.ci/Jenkinsfile

@@ -9,6 +9,8 @@ pipeline {
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
     NOTIFY_TO = credentials('notify-to')
     JOB_GCS_BUCKET = credentials('gcs-bucket')
+    DOCKER_ELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    DOCKER_REGISTRY = 'docker.elastic.co'
   }
   options {
     timeout(time: 1, unit: 'HOURS')
@@ -28,15 +30,17 @@ pipeline {
     booleanParam(name: "forceSkipGitChecks", defaultValue: false, description: "If it's needed to check for Git changes to filter by modified sources")
     booleanParam(name: "forceSkipPresubmit", defaultValue: false, description: "If it's needed to execute the pre-submit tests: unit and precommit.")
     string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz')
+    string(name: 'ELASTIC_AGENT_STAND_ALONE_VERSION', defaultValue: '8.0.0-SNAPSHOT', description: 'SemVer version of the stand-alone elastic-agent to be used for Ingest Manager tests. You can use here the tag of your PR to test your changes')
+    booleanParam(name: "ELASTIC_AGENT_USE_CI_SNAPSHOTS", defaultValue: false, description: "If it's needed to use the binary snapshots produced by Beats CI instead of the official releases")
     choice(name: 'LOG_LEVEL', choices: ['INFO', 'DEBUG'], description: 'Log level to be used')
     choice(name: 'RETRY_TIMEOUT', choices: ['3', '5', '7', '11'], description: 'Max number of minutes for timeout backoff strategies')
-    string(name: 'STACK_VERSION_INGEST_MANAGER', defaultValue: '8.0.0-SNAPSHOT', description: 'SemVer version of the stack to be used for Ingest Manager tests.')
-    string(name: 'STACK_VERSION_METRICBEAT', defaultValue: '7.8.0', description: 'SemVer version of the stack to be used for Metricbeat tests.')
+    string(name: 'INGEST_MANAGER_STACK_VERSION', defaultValue: '8.0.0-SNAPSHOT', description: 'SemVer version of the stack to be used for Ingest Manager tests.')
+    string(name: 'METRICBEAT_STACK_VERSION', defaultValue: '7.8.0', description: 'SemVer version of the stack to be used for Metricbeat tests.')
     string(name: 'METRICBEAT_VERSION', defaultValue: '7.8.0', description: 'SemVer version of the metricbeat to be used.')
     string(name: 'HELM_CHART_VERSION', defaultValue: '7.6.1', description: 'SemVer version of Helm chart to be used.')
     string(name: 'HELM_VERSION', defaultValue: '2.16.3', description: 'SemVer version of Helm to be used.')
-    string(name: 'KIND_VERSION', defaultValue: '0.7.0', description: 'SemVer version of Kind to be used.')
-    string(name: 'KUBERNETES_VERSION', defaultValue: '1.15.3', description: 'SemVer version of Kubernetes to be used.')
+    string(name: 'HELM_KIND_VERSION', defaultValue: '0.7.0', description: 'SemVer version of Kind to be used.')
+    string(name: 'HELM_KUBERNETES_VERSION', defaultValue: '1.15.3', description: 'SemVer version of Kubernetes to be used.')
   }
   stages {
     stage('Initializing'){
@@ -47,15 +51,17 @@ pipeline {
         PATH = "${env.PATH}:${env.WORKSPACE}/bin:${env.WORKSPACE}/${env.BASE_DIR}/.ci/scripts"
         GO111MODULE = 'on'
         ELASTIC_AGENT_DOWNLOAD_URL = "${params.ELASTIC_AGENT_DOWNLOAD_URL.trim()}"
+        ELASTIC_AGENT_STAND_ALONE_VERSION = "${params.ELASTIC_AGENT_STAND_ALONE_VERSION.trim()}"
+        ELASTIC_AGENT_USE_CI_SNAPSHOTS = "${params.ELASTIC_AGENT_USE_CI_SNAPSHOTS}"
+        INGEST_MANAGER_STACK_VERSION = "${params.INGEST_MANAGER_STACK_VERSION.trim()}"
         METRICBEAT_VERSION = "${params.METRICBEAT_VERSION.trim()}"
-        STACK_VERSION_INGEST_MANAGER = "${params.STACK_VERSION_INGEST_MANAGER.trim()}"
-        STACK_VERSION_METRICBEAT = "${params.STACK_VERSION_METRICBEAT.trim()}"
+        METRICBEAT_STACK_VERSION = "${params.METRICBEAT_STACK_VERSION.trim()}"
         FORCE_SKIP_GIT_CHECKS = "${params.forceSkipGitChecks}"
         FORCE_SKIP_PRESUBMIT = "${params.forceSkipPresubmit}"
         HELM_CHART_VERSION = "${params.HELM_CHART_VERSION.trim()}"
         HELM_VERSION = "${params.HELM_VERSION.trim()}"
-        KIND_VERSION = "${params.KIND_VERSION.trim()}"
-        KUBERNETES_VERSION = "${params.KUBERNETES_VERSION.trim()}"
+        HELM_KIND_VERSION = "${params.HELM_KIND_VERSION.trim()}"
+        HELM_KUBERNETES_VERSION = "${params.HELM_KUBERNETES_VERSION.trim()}"
         LOG_LEVEL = "${params.LOG_LEVEL.trim()}"
         RETRY_TIMEOUT = "${params.RETRY_TIMEOUT.trim()}"
       }
@@ -204,14 +210,17 @@ def generateStep(Map params = [:]){
 def generateFunctionalTestStep(Map params = [:]){
   def suite = params.get('suite')
   def sneakCaseSuite = suite.toUpperCase().replaceAll("-", "_")
-  def stackVersion = env."STACK_VERSION_${sneakCaseSuite}"
+  def stackVersion = env."${sneakCaseSuite}_STACK_VERSION"
   def feature = params.get('feature')
   return {
     node('ubuntu-18.04 && immutable && docker') {
       try {
         deleteDir()
         unstash 'source'
         withGoEnv(version: "${GO_VERSION}"){
+          if(isInstalled(tool: 'docker', flag: '--version')) {
+            dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+          }
           retry(3){
             dir("${BASE_DIR}"){
               sh script: """.ci/scripts/install-test-dependencies.sh "${suite}" """, label: "Install test dependencies for ${suite}:${feature}"

.ci/scripts/install-helm-test-dependencies.sh

@@ -10,29 +10,29 @@ set -euxo pipefail
 #
 # Parameters:
 #   - HELM_VERSION - that's the Helm version which will be installed and enabled.
-#   - KIND_VERSION - that's the Kind version which will be installed and enabled.
-#   - KUBERNETES_VERSION - that's the Kubernetes version which will be installed and enabled.
+#   - HELM_KIND_VERSION - that's the Kind version which will be installed and enabled.
+#   - HELM_KUBERNETES_VERSION - that's the Kubernetes version which will be installed and enabled.
 #
 
 MSG="parameter missing."
 HOME=${HOME:?$MSG}
 
 HELM_VERSION="${HELM_VERSION:-"2.16.3"}"
 HELM_TAR_GZ_FILE="helm-v${HELM_VERSION}-linux-amd64.tar.gz"
-KIND_VERSION="v${KIND_VERSION:-"0.7.0"}"
-KUBERNETES_VERSION="${KUBERNETES_VERSION:-"1.15.3"}"
+HELM_KIND_VERSION="v${HELM_KIND_VERSION:-"0.7.0"}"
+HELM_KUBERNETES_VERSION="${HELM_KUBERNETES_VERSION:-"1.15.3"}"
 
 HELM_CMD="${HOME}/bin/helm"
 KBC_CMD="${HOME}/bin/kubectl"
 
 # Install kind as a Go binary
-GO111MODULE="on" go get sigs.k8s.io/kind@${KIND_VERSION}
+GO111MODULE="on" go get sigs.k8s.io/kind@${HELM_KIND_VERSION}
 
 mkdir -p "${HOME}/bin" "${HOME}/.kube"
 touch "${HOME}/.kube/config"
 
 # Install kubectl
-curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
+curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/v${HELM_KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
 chmod +x "${KBC_CMD}"
 ${KBC_CMD} version --client
 

cli/config/compose/profiles/ingest-manager/docker-compose.yml

@@ -16,7 +16,7 @@ services:
       - xpack.security.authc.api_key.enabled=true
       - ELASTIC_USERNAME=elastic
       - ELASTIC_PASSWORD=changeme
-    image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-8.0.0-SNAPSHOT}"
+    image: "docker.elastic.co/observability-ci/elasticsearch:${stackVersion:-8.0.0-SNAPSHOT}"
     ports:
       - "9200:9200"
   kibana:
@@ -29,7 +29,7 @@ services:
       test: "curl -f http://localhost:5601/login | grep kbn-injected-metadata 2>&1 >/dev/null"
       retries: 600
       interval: 1s
-    image: "docker.elastic.co/kibana/kibana:${stackVersion:-8.0.0-SNAPSHOT}"
+    image: "docker.elastic.co/observability-ci/kibana:${stackVersion:-8.0.0-SNAPSHOT}"
     ports:
       - "5601:5601"
     volumes:

cli/config/compose/profiles/metricbeat/docker-compose.yml

@@ -9,6 +9,6 @@ services:
       - xpack.monitoring.collection.enabled=true
       - ELASTIC_USERNAME=elastic
       - ELASTIC_PASSWORD=changeme
-    image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion}"
+    image: "docker.elastic.co/observability-ci/elasticsearch:${stackVersion}"
     ports:
       - "9200:9200"

cli/config/compose/services/apm-server/docker-compose.yml

@@ -13,7 +13,7 @@ services:
       - setup.kibana.host=http://kibana:5601
       - setup.template.settings.index.number_of_replicas=0
       - xpack.monitoring.elasticsearch=true
-    image: "docker.elastic.co/apm/apm-server:${apmServerTag}"
+    image: "docker.elastic.co/observability-ci/apm-server:${apmServerTag}"
     ports:
       - "6060:6060"
       - "8200:8200"

cli/config/compose/services/elastic-agent/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '2.3'
 services:
   elastic-agent:
-    image: docker.elastic.co/beats/elastic-agent:${elasticAgentTag:-8.0.0-SNAPSHOT}
+    image: docker.elastic.co/observability-ci/elastic-agent:${elasticAgentTag:-8.0.0-SNAPSHOT}
     depends_on:
       elasticsearch:
         condition: service_healthy

cli/config/compose/services/elasticsearch/docker-compose.yml

@@ -9,7 +9,7 @@ services:
       - xpack.monitoring.collection.enabled=true
       - ELASTIC_USERNAME=elastic
       - ELASTIC_PASSWORD=changeme
-    image: "docker.elastic.co/elasticsearch/elasticsearch:${elasticsearchTag}"
+    image: "docker.elastic.co/observability-ci/elasticsearch:${elasticsearchTag}"
     ports:
       - "9200:9200"
       - "9300:9300"

cli/config/compose/services/kibana/docker-compose.yml

@@ -5,6 +5,6 @@ services:
       - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
       - ELASTIC_USERNAME=elastic
       - ELASTIC_PASSWORD=changeme
-    image: "docker.elastic.co/kibana/kibana:${kibanaTag}"
+    image: "docker.elastic.co/observability-ci/kibana:${kibanaTag}"
     ports:
       - "5601:5601"

cli/config/compose/services/metricbeat/docker-compose.yml

@@ -14,7 +14,7 @@ services:
     ]
     environment:
       - BEAT_STRICT_PERMS=${beatStricPerms:-false}
-    image: "docker.elastic.co/beats/metricbeat:${metricbeatTag}"
+    image: "docker.elastic.co/observability-ci/metricbeat:${metricbeatTag}"
     labels:
       co.elastic.logs/module: "${serviceName}"
     volumes:

cli/config/compose/services/opbeans-go/docker-compose.yml

@@ -9,6 +9,6 @@ services:
       - ELASTIC_APM_LOG_FILE=stderr
       - ELASTIC_APM_LOG_LEVEL=debug
       - OPBEANS_SERVER_PORT=8000
-    image: "opbeans/opbeans-go:${opbeansGoTag}"
+    image: "docker.elastic.co/observability-ci/opbeans-go:${opbeansGoTag}"
     ports:
       - "8000:8000"

cli/config/compose/services/opbeans-java/docker-compose.yml

@@ -7,6 +7,6 @@ services:
       - ELASTIC_APM_SERVER_URL=http://localhost:8200
       - ELASTIC_APM_SERVICE_NAME=opbeans-java
       - OPBEANS_SERVER_PORT=8000
-    image: "opbeans/opbeans-java:${opbeansJavaTag}"
+    image: "docker.elastic.co/observability-ci/opbeans-java:${opbeansJavaTag}"
     ports:
       - "8000:8000"

cli/shell/shell.go

@@ -6,7 +6,10 @@ package shell
 
 import (
 	"bytes"
+	"fmt"
+	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
@@ -54,6 +57,43 @@ func Execute(workspace string, command string, args ...string) (string, error) {
 	return strings.Trim(out.String(), "\n"), nil
 }
 
+// GetEnv returns an environment variable as string
+func GetEnv(envVar string, defaultValue string) string {
+	if value, exists := os.LookupEnv(envVar); exists {
+		return value
+	}
+
+	return defaultValue
+}
+
+// GetEnvBool returns an environment variable as boolean, returning also an error if
+// and only if the variable is not present
+func GetEnvBool(key string) (bool, error) {
+	s := os.Getenv(key)
+	if s == "" {
+		return false, fmt.Errorf("The %s variable is not set", key)
+	}
+
+	v, err := strconv.ParseBool(s)
+	if err != nil {
+		return false, nil
+	}
+
+	return v, nil
+}
+
+// GetEnvInteger returns an environment variable as integer, including a default value
+func GetEnvInteger(envVar string, defaultValue int) int {
+	if value, exists := os.LookupEnv(envVar); exists {
+		v, err := strconv.Atoi(value)
+		if err == nil {
+			return v
+		}
+	}
+
+	return defaultValue
+}
+
 // which checks if software is installed, else it aborts the execution
 func which(binary string) error {
 	path, err := exec.LookPath(binary)

cli/shell/shell_test.go

@@ -0,0 +1,46 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package shell
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetEnvBool(t *testing.T) {
+	type test struct {
+		key   string
+		value string
+	}
+
+	tests := []test{
+		{key: "should_be_true", value: "true"},
+		{key: "should_be_false", value: "false"},
+		{key: "should_be_empty", value: ""},
+		{key: "should_be_not_empty", value: "garbage"},
+	}
+
+	for _, test := range tests {
+		os.Setenv(test.key, test.value)
+
+		val, err := GetEnvBool(test.key)
+
+		if test.key == "should_be_empty" {
+			assert.NotNil(t, err)
+			assert.False(t, val)
+		} else if test.key == "should_be_not_empty" {
+			assert.Nil(t, err)
+			assert.False(t, val)
+		} else if test.key == "should_be_true" {
+			assert.Nil(t, err)
+			assert.True(t, val)
+		} else if test.key == "should_be_false" {
+			assert.Nil(t, err)
+			assert.False(t, val)
+		}
+	}
+}

e2e/Makefile

@@ -37,8 +37,8 @@ functional-test: install-godog
 	OP_LOG_LEVEL=${LOG_LEVEL} \
 	OP_LOG_INCLUDE_TIMESTAMP=${LOG_INCLUDE_TIMESTAMP} \
 	OP_RETRY_TIMEOUT=${RETRY_TIMEOUT} \
-	OP_METRICBEAT_VERSION=${METRICBEAT_VERSION} \
-	OP_STACK_VERSION=${STACK_VERSION} \
+	METRICBEAT_VERSION=${METRICBEAT_VERSION} \
+	STACK_VERSION=${STACK_VERSION} \
 	godog --format=${FORMAT} ${FEATURE_FLAG} ${FEATURE}
 
 .PHONY: notice

e2e/_suites/helm/README.md

@@ -35,7 +35,7 @@ This is an example of the optional configuration:
    # Depending on the versions used, 
    export HELM_VERSION="3.2.2"        # Helm version: for Helm v2.x.x we have to initialise Tiller right after the k8s cluster
    export HELM_CHART_VERSION="7.6.1"  # version of the Elastic's Observability Helm charts
-   export KUBERNETES_VERSION="1.15.3" # version of the cluster to be passedd to kind
+   export HELM_KUBERNETES_VERSION="1.15.3" # version of the cluster to be passedd to kind
    ```
 
 3. Install dependencies.

e2e/_suites/helm/helm_charts_test.go

@@ -489,7 +489,7 @@ func HelmChartFeatureContext(s *godog.Suite) {
 	if value, exists := os.LookupEnv("HELM_CHART_VERSION"); exists {
 		testSuite.Version = value
 	}
-	if value, exists := os.LookupEnv("KUBERNETES_VERSION"); exists {
+	if value, exists := os.LookupEnv("HELM_KUBERNETES_VERSION"); exists {
 		testSuite.KubernetesVersion = value
 	}
 

e2e/_suites/ingest-manager/README.md

@@ -45,10 +45,15 @@ This is an example of the optional configuration:
 
    ```shell
    # There should be a Docker image for the runtime dependencies (elasticsearch, kibana, package registry)
-   export OP_STACK_VERSION=8.0.0-SNAPSHOT
-   # This environment variable will use a fixed version of the Elastic agent binary, obtained from
+   export STACK_VERSION=8.0.0-SNAPSHOT
+   # (Fleet mode) This environment variable will use a fixed version of the Elastic agent binary, obtained from
    # https://artifacts-api.elastic.co/v1/search/8.0.0-SNAPSHOT/elastic-agent
    export ELASTIC_AGENT_DOWNLOAD_URL="https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz"
+   # (Fleet mode) This environment variable will use the snapshots produced by Beats CI. If the above variable
+   # is set, this variable will take no effect
+   export ELASTIC_AGENT_USE_CI_SNAPSHOTS="true"
+   # (Stand-Alone mode) This environment variable will use the its value as the Docker tag produced by Beats CI.
+   export ELASTIC_AGENT_STAND_ALONE_VERSION="78a762c76080aafa34c52386341b590dac24e2df"
    ```
 
 3. Define the proper Docker images to be used in tests (Optional).

e2e/_suites/ingest-manager/fleet.go

@@ -185,7 +185,7 @@ func (fts *FleetTestSuite) systemPackageDashboardsAreListedInFleet() error {
 	log.Debug("Checking system Package dashboards in Fleet")
 
 	dataStreamsCount := 0
-	maxTimeout := 1 * time.Minute
+	maxTimeout := time.Minute
 	retryCount := 1
 
 	exp := e2e.GetExponentialBackOff(maxTimeout)

e2e/_suites/ingest-manager/ingest-manager_test.go

@@ -16,12 +16,13 @@ import (
 	"github.com/elastic/e2e-testing/cli/config"
 	"github.com/elastic/e2e-testing/cli/docker"
 	"github.com/elastic/e2e-testing/cli/services"
+	"github.com/elastic/e2e-testing/cli/shell"
 	"github.com/elastic/e2e-testing/e2e"
 	log "github.com/sirupsen/logrus"
 )
 
 // stackVersion is the version of the stack to use
-// It can be overriden by OP_STACK_VERSION env var
+// It can be overriden by STACK_VERSION env var
 var stackVersion = "8.0.0-SNAPSHOT"
 
 // profileEnv is the environment to be applied to any execution
@@ -38,8 +39,8 @@ const kibanaBaseURL = "http://localhost:5601"
 func init() {
 	config.Init()
 
-	queryRetryTimeout = e2e.GetIntegerFromEnv("OP_RETRY_TIMEOUT", queryRetryTimeout)
-	stackVersion = e2e.GetEnv("OP_STACK_VERSION", stackVersion)
+	queryRetryTimeout = shell.GetEnvInteger("OP_RETRY_TIMEOUT", queryRetryTimeout)
+	stackVersion = shell.GetEnv("STACK_VERSION", stackVersion)
 }
 
 func IngestManagerFeatureContext(s *godog.Suite) {