chore: backports for 7.13.x branch

.ci/.e2e-tests.yaml

@@ -17,9 +17,6 @@ SUITES:
       - name: "Fleet"
         pullRequestFilter: " && ~debian"
         tags: "fleet_mode_agent"
-      - name: "Fleet Server"
-        pullRequestFilter: " && ~debian"
-        tags: "fleet_server"
       - name: "Endpoint Integration"
         pullRequestFilter: " && ~debian"
         tags: "agent_endpoint_integration"

.ci/scripts/clean-docker.sh

@@ -9,7 +9,7 @@ set -euxo pipefail
 # Build and test the app using the install and test make goals.
 #
 
-readonly VERSION="7.13.0-SNAPSHOT"
+readonly VERSION="$(cat $(pwd)/.stack-version)"
 
 main() {
   # refresh docker images

.ci/scripts/fleet-test.sh

@@ -9,10 +9,10 @@ set -euxo pipefail
 # Run the functional tests for fleets using the functional-test wrapper
 #
 # Parameters:
-#   - STACK_VERSION - that's the version of the stack to be tested. Default '7.13.0-SNAPSHOT'.
+#   - STACK_VERSION - that's the version of the stack to be tested. Default is stored in '.stack-version'.
 #
 
-STACK_VERSION=${1:-'7.13.0-SNAPSHOT'}
+STACK_VERSION=${1:-"$(cat $(pwd)/.stack-version)"}
 SUITE='fleet'
 
 # Exclude the nightly tests in the CI.

.ci/scripts/functional-test.sh

@@ -12,14 +12,16 @@ set -euxo pipefail
 # Parameters:
 #   - SUITE - that's the suite to be tested. Default '' which means all of them.
 #   - TAGS - that's the tags to be tested. Default '' which means all of them.
-#   - STACK_VERSION - that's the version of the stack to be tested. Default '7.13.0-SNAPSHOT'.
-#   - BEAT_VERSION - that's the version of the metricbeat to be tested. Default '7.13.0-SNAPSHOT'.
+#   - STACK_VERSION - that's the version of the stack to be tested. Default is stored in '.stack-version'.
+#   - BEAT_VERSION - that's the version of the metricbeat to be tested. Default is stored in '.stack-version'.
 #
 
+BASE_VERSION="$(cat $(pwd)/.stack-version)"
+
 SUITE=${1:-''}
 TAGS=${2:-''}
-STACK_VERSION=${3:-'7.13.0-SNAPSHOT'}
-BEAT_VERSION=${4:-'7.13.0-SNAPSHOT'}
+STACK_VERSION=${3:-"${BASE_VERSION}"}
+BEAT_VERSION=${4:-"${BASE_VERSION}"}
 
 ## Install the required dependencies for the given SUITE
 .ci/scripts/install-test-dependencies.sh "${SUITE}"

.ci/scripts/metricbeat-test.sh

@@ -9,12 +9,14 @@ set -euxo pipefail
 # Run the functional tests for metricbeat using the functional-test wrapper
 #
 # Parameters:
-#   - STACK_VERSION - that's the version of the stack to be tested. Default '7.13.0-SNAPSHOT'.
-#   - BEAT_VERSION - that's the version of the metricbeat to be tested. Default '7.13.0-SNAPSHOT'.
+#   - STACK_VERSION - that's the version of the stack to be tested. Default is stored in '.stack-version'.
+#   - BEAT_VERSION - that's the version of the metricbeat to be tested. Default is stored in '.stack-version'.
 #
 
-STACK_VERSION=${1:-'7.13.0-SNAPSHOT'}
-BEAT_VERSION=${2:-'7.13.0-SNAPSHOT'}
+BASE_VERSION="$(cat $(pwd)/.stack-version)"
+
+STACK_VERSION=${1:-"${BASE_VERSION}"}
+BEAT_VERSION=${2:-"${BASE_VERSION}"}
 SUITE='metricbeat'
 
 .ci/scripts/functional-test.sh "${SUITE}" "" "${STACK_VERSION}" "${BEAT_VERSION}"

.pre-commit-config.yaml

@@ -8,6 +8,8 @@ repos:
         exclude: ^notice/overrides.json
     -   id: check-merge-conflict
     -   id: check-yaml
+        exclude: >
+          (?x)^(cli/config/kubernetes.*)$
     -   id: check-xml
     -   id: end-of-file-fixer
         exclude: >

.stack-version

@@ -0,0 +1 @@
+7.13.0-SNAPSHOT

cli/cmd/deploy.go

@@ -8,7 +8,7 @@ import (
 	"context"
 
 	"github.com/elastic/e2e-testing/cli/config"
-	"github.com/elastic/e2e-testing/internal/compose"
+	"github.com/elastic/e2e-testing/internal/deploy"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
@@ -63,12 +63,16 @@ func buildDeployServiceCommand(srv string) *cobra.Command {
 		Short: `Deploys a ` + srv + ` service`,
 		Long:  `Deploys a ` + srv + ` service, adding it to a running profile, identified by its name`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
 			env := map[string]string{}
 			env = config.PutServiceEnvironment(env, srv, versionToRun)
 
-			err := serviceManager.AddServicesToCompose(context.Background(), deployToProfile, []string{srv}, env)
+			err := serviceManager.AddServicesToCompose(
+				context.Background(),
+				deploy.NewServiceRequest(deployToProfile),
+				[]deploy.ServiceRequest{deploy.NewServiceRequest(srv)},
+				env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"profile":  deployToProfile,
@@ -85,12 +89,16 @@ func buildUndeployServiceCommand(srv string) *cobra.Command {
 		Short: `Undeploys a ` + srv + ` service`,
 		Long:  `Undeploys a ` + srv + ` service, removing it from a running profile, identified by its name`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
 			env := map[string]string{}
 			env = config.PutServiceEnvironment(env, srv, versionToRun)
 
-			err := serviceManager.RemoveServicesFromCompose(context.Background(), deployToProfile, []string{srv}, env)
+			err := serviceManager.RemoveServicesFromCompose(
+				context.Background(),
+				deploy.NewServiceRequest(deployToProfile),
+				[]deploy.ServiceRequest{deploy.NewServiceRequest(srv)},
+				env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"profile":  deployToProfile,

cli/cmd/run.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 
 	"github.com/elastic/e2e-testing/cli/config"
-	"github.com/elastic/e2e-testing/internal/compose"
+	"github.com/elastic/e2e-testing/internal/deploy"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
@@ -64,7 +64,7 @@ func buildRunServiceCommand(srv string) *cobra.Command {
 		Short: `Runs a ` + srv + ` service`,
 		Long:  `Runs a ` + srv + ` service, spinning up a Docker container for it and exposing its internal configuration so that you are able to connect to it in an easy manner`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
 			env := config.PutServiceEnvironment(map[string]string{}, srv, versionToRun)
 
@@ -76,7 +76,8 @@ func buildRunServiceCommand(srv string) *cobra.Command {
 				env[k] = v
 			}
 
-			err := serviceManager.RunCompose(context.Background(), false, []string{srv}, env)
+			err := serviceManager.RunCompose(
+				context.Background(), false, []deploy.ServiceRequest{deploy.NewServiceRequest(srv)}, env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"service": srv,
@@ -96,7 +97,7 @@ Example:
   go run main.go run profile fleet -s elastic-agent:7.13.0-SNAPSHOT
 `,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
 			env := map[string]string{
 				"profileVersion": versionToRun,
@@ -110,14 +111,15 @@ Example:
 				env[k] = v
 			}
 
-			err := serviceManager.RunCompose(context.Background(), true, []string{key}, env)
+			err := serviceManager.RunCompose(
+				context.Background(), true, []deploy.ServiceRequest{deploy.NewServiceRequest(key)}, env)
 			if err != nil {
 				log.WithFields(log.Fields{
 					"profile": key,
 				}).Error("Could not run the profile.")
 			}
 
-			composeNames := []string{}
+			composeNames := []deploy.ServiceRequest{}
 			if len(servicesToRun) > 0 {
 				for _, srv := range servicesToRun {
 					arr := strings.Split(srv, ":")
@@ -137,10 +139,10 @@ Example:
 					}).Trace("Adding service")
 
 					env = config.PutServiceEnvironment(env, image, tag)
-					composeNames = append(composeNames, image)
+					composeNames = append(composeNames, deploy.NewServiceRequest(image))
 				}
 
-				err = serviceManager.AddServicesToCompose(context.Background(), key, composeNames, env)
+				err = serviceManager.AddServicesToCompose(context.Background(), deploy.NewServiceRequest(key), composeNames, env)
 				if err != nil {
 					log.WithFields(log.Fields{
 						"profile":  key,

cli/cmd/stop.go

@@ -8,7 +8,7 @@ import (
 	"context"
 
 	"github.com/elastic/e2e-testing/cli/config"
-	"github.com/elastic/e2e-testing/internal/compose"
+	"github.com/elastic/e2e-testing/internal/deploy"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
@@ -55,9 +55,10 @@ func buildStopServiceCommand(srv string) *cobra.Command {
 		Short: `Stops a ` + srv + ` service`,
 		Long:  `Stops a ` + srv + ` service, stoppping its Docker container`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
-			err := serviceManager.StopCompose(context.Background(), false, []string{srv})
+			err := serviceManager.StopCompose(
+				context.Background(), false, []deploy.ServiceRequest{deploy.NewServiceRequest(srv)})
 			if err != nil {
 				log.WithFields(log.Fields{
 					"service": srv,
@@ -73,9 +74,10 @@ func buildStopProfileCommand(key string, profile config.Profile) *cobra.Command
 		Short: `Stops the ` + profile.Name + ` profile`,
 		Long:  `Stops the ` + profile.Name + ` profile, stopping the Services that compound it`,
 		Run: func(cmd *cobra.Command, args []string) {
-			serviceManager := compose.NewServiceManager()
+			serviceManager := deploy.NewServiceManager()
 
-			err := serviceManager.StopCompose(context.Background(), true, []string{key})
+			err := serviceManager.StopCompose(
+				context.Background(), true, []deploy.ServiceRequest{deploy.NewServiceRequest(key)})
 			if err != nil {
 				log.WithFields(log.Fields{
 					"profile": key,

cli/config/compose/profiles/fleet/configurations/kibana.config.yml

@@ -16,5 +16,5 @@ xpack.fleet.registryUrl: http://package-registry:8080
 xpack.fleet.agents.enabled: true
 xpack.fleet.agents.elasticsearch.host: http://elasticsearch:9200
 xpack.fleet.agents.fleet_server.hosts:
-  - http://kibana:5601
+  - http://fleet-server:8220
 xpack.fleet.agents.tlsCheckDisabled: true

cli/config/compose/profiles/fleet/docker-compose.yml

@@ -29,7 +29,7 @@ services:
       test: "curl -f http://localhost:5601/login | grep kbn-injected-metadata 2>&1 >/dev/null"
       retries: 600
       interval: 1s
-    image: "docker.elastic.co/${kibanaDockerNamespace:-beats}/kibana:${kibanaVersion:-7.13.0-SNAPSHOT}"
+    image: "docker.elastic.co/${kibanaDockerNamespace:-kibana}/kibana:${kibanaVersion:-7.13.0-SNAPSHOT}"
     ports:
       - "5601:5601"
     volumes:
@@ -40,3 +40,22 @@ services:
       test: ["CMD", "curl", "-f", "http://localhost:8080"]
       retries: 300
       interval: 1s
+
+  fleet-server:
+    image: "docker.elastic.co/beats/elastic-agent:${stackVersion:-8.0.0-SNAPSHOT}"
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+      kibana:
+        condition: service_healthy
+    healthcheck:
+      test: "curl -f http://127.0.0.1:8220/api/status | grep HEALTHY 2>&1 >/dev/null"
+      retries: 12
+      interval: 5s
+    environment:
+      - "FLEET_SERVER_ENABLE=1"
+      - "FLEET_SERVER_INSECURE_HTTP=1"
+      - "KIBANA_FLEET_SETUP=1"
+      - "KIBANA_FLEET_HOST=http://kibana:5601"
+      - "FLEET_SERVER_HOST=0.0.0.0"
+      - "FLEET_SERVER_PORT=8220"

cli/config/compose/services/elastic-agent/apm-legacy/config/apm-server.yml

@@ -0,0 +1,21 @@
+monitoring.enabled: true
+http.enabled: true
+http.port: 5067
+http.host: "0.0.0.0"
+apm-server:
+  host: "0.0.0:8200"
+  secret_token: "1234"
+  # Enable APM Server Golang expvar support (https://golang.org/pkg/expvar/).
+  expvar:
+    enabled: true
+    url: "/debug/vars"
+  kibana:
+    # For APM Agent configuration in Kibana, enabled must be true.
+    enabled: true
+    host: "kibana"
+    username: "elastic"
+    password: "changeme"
+output.elasticsearch:
+  hosts: ["http://elasticsearch:9200"]
+  username: "elastic"
+  password: "changeme"

cli/config/compose/services/elastic-agent/apm-legacy/config/capabilities.yml

@@ -0,0 +1,5 @@
+capabilities:
+- rule: allow
+  input: fleet-server
+- rule: deny
+  input: "*"

cli/config/compose/services/elastic-agent/apm-legacy/config/credentials.yml

@@ -0,0 +1,10 @@
+fleet_server:
+  elasticsearch:
+    host: "elasticsearch"
+    username: "elastic"
+    password: "changeme"
+kibana:
+  fleet:
+    host: "kibana"
+    username: "elastic"
+    password: "changeme"

cli/config/compose/services/elastic-agent/apm-legacy/config/fleet-setup.yml

@@ -0,0 +1,9 @@
+fleet:
+  enroll: true
+  force: false
+  insecure: true
+fleet_server:
+  enable: true
+kibana:
+  fleet:
+    setup: true

cli/config/compose/services/centos-systemd/docker-compose.yml → cli/config/compose/services/elastic-agent/centos/docker-compose.yml

@@ -1,6 +1,6 @@
 version: '2.4'
 services:
-  centos-systemd:
+  elastic-agent:
     image: centos/systemd:${centos_systemdTag:-latest} 
     container_name: ${centos_systemdContainerName}
     entrypoint: "/usr/sbin/init"

cli/config/compose/services/elastic-agent/cloud/docker-compose.yml

@@ -0,0 +1,26 @@
+version: '2.4'
+services:
+  elastic-agent:
+    image: docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.0.0-SNAPSHOT}
+    container_name:  ${elasticAgentContainerName}
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+      kibana:
+        condition: service_healthy
+    environment:
+      - "FLEET_SERVER_ENABLE=1"
+      - "FLEET_SERVER_INSECURE_HTTP=1"
+      - "ELASTIC_AGENT_CLOUD=1"
+      - "APM_SERVER_PATH=/apm-legacy/apm-server/"
+      - "STATE_PATH=/apm-legacy/elastic-agent/"
+      - "CONFIG_PATH=/apm-legacy/config/"
+      - "DATA_PATH=/apm-legacy/data/"
+      - "LOGS_PATH=/apm-legacy/logs/"
+      - "HOME_PATH=/apm-legacy/"
+    volumes:
+      - "${apmVolume}:/apm-legacy"
+    ports:
+      - "127.0.0.1:8220:8220"
+      - "127.0.0.1:8200:8200"
+      - "127.0.0.1:5066:5066"

cli/config/compose/services/debian-systemd/docker-compose.yml → cli/config/compose/services/elastic-agent/debian/docker-compose.yml

@@ -1,6 +1,6 @@
 version: '2.4'
 services:
-  debian-systemd:
+  elastic-agent:
     image: alehaa/debian-systemd:${debian_systemdTag:-stretch}
     container_name: ${debian_systemdContainerName} 
     entrypoint: "/sbin/init"

cli/config/compose/services/elastic-agent/docker-compose.yml

@@ -9,12 +9,8 @@ services:
       kibana:
         condition: service_healthy
     environment:
-      - "FLEET_SERVER_ELASTICSEARCH_HOST=http://${elasticsearchHost:-elasticsearch}:${elasticsearchPort:-9200}"
       - "FLEET_SERVER_ENABLE=${fleetServerMode:-0}"
       - "FLEET_SERVER_INSECURE_HTTP=${fleetServerMode:-0}"
-      - "FLEET_SERVER_HOST=0.0.0.0"
-      - "FLEET_SERVER_ELASTICSEARCH_USERNAME=elastic"
-      - "FLEET_SERVER_ELASTICSEARCH_PASSWORD=changeme"
     platform: ${elasticAgentPlatform:-linux/amd64}
     ports:
       - "127.0.0.1:8220:8220"

cli/config/compose/services/fleet-server-centos/docker-compose.yml

@@ -6,4 +6,5 @@ services:
     entrypoint: "/usr/sbin/init"
     privileged: true
     volumes:
+      - ${fleet_server_centosAgentBinarySrcPath:-.}:${fleet_server_centosAgentBinaryTargetPath:-/tmp}
       - /sys/fs/cgroup:/sys/fs/cgroup:ro