Skip to content
This repository has been archived by the owner on Sep 17, 2024. It is now read-only.

Commit

Permalink
Support multiple deployment backends (#1130)
Browse files Browse the repository at this point in the history
* Abstract out deployment

Provides ability to plugin different deployment backends for use in testing.
Current deployment backends supported are "docker" and "kubernetes"

* remove unused import
* remove unsetting of fleet server hostname as it's not needed
* add deployer support to stand-alone
* add elastic-agent to k8s deployment specs
* Update internal/docker/docker.go

Signed-off-by: Adam Stokes <51892+adam-stokes@users.noreply.github.com>
Co-authored-by: Manuel de la Peña <mdelapenya@gmail.com>
  • Loading branch information
adam-stokes and mdelapenya committed May 15, 2021
1 parent 7c64627 commit 55eee96
Show file tree
Hide file tree
Showing 11 changed files with 348 additions and 88 deletions.
25 changes: 25 additions & 0 deletions cli/config/kubernetes/overlays/deployment.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: elastic-agent
labels:
app: elastic-agent
spec:
replicas: 1
selector:
matchLabels:
app: elastic-agent
template:
metadata:
labels:
app: elastic-agent
spec:
containers:
- name: elastic-agent
image: centos/systemd:latest
command: ["/usr/sbin/init"]
securityContext:
allowPrivilegeEscalation: true
runAsUser: 0
capabilities:
add: ["SYS_ADMIN"]
5 changes: 5 additions & 0 deletions cli/config/kubernetes/overlays/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
bases:
- ../../base

resources:
- deployment.yaml
20 changes: 9 additions & 11 deletions e2e/_suites/fleet/fleet.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/cucumber/godog"
"github.com/elastic/e2e-testing/internal/common"
"github.com/elastic/e2e-testing/internal/compose"
"github.com/elastic/e2e-testing/internal/deploy"
"github.com/elastic/e2e-testing/internal/docker"
"github.com/elastic/e2e-testing/internal/elasticsearch"
"github.com/elastic/e2e-testing/internal/installer"
Expand Down Expand Up @@ -46,21 +47,18 @@ type FleetTestSuite struct {
FleetServerPolicy kibana.Policy
Version string // current elastic-agent version
kibanaClient *kibana.Client

deployer deploy.Deployment
// date controls for queries
AgentStoppedDate time.Time
RuntimeDependenciesStartDate time.Time
}

// afterScenario destroys the state created by a scenario
func (fts *FleetTestSuite) afterScenario() {

serviceName := common.ElasticAgentServiceName
serviceManager := compose.NewServiceManager()

if !fts.StandAlone {
agentInstaller := fts.getInstaller()
serviceName = fts.getServiceName(agentInstaller)

if log.IsLevelEnabled(log.DebugLevel) {
err := agentInstaller.PrintLogsFn(fts.Hostname)
Expand Down Expand Up @@ -92,7 +90,7 @@ func (fts *FleetTestSuite) afterScenario() {

developerMode := shell.GetEnvBool("DEVELOPER_MODE")
if !developerMode {
_ = serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName}, common.ProfileEnv)
_ = fts.deployer.Remove([]string{common.FleetProfileName, serviceName}, common.ProfileEnv)
} else {
log.WithField("service", serviceName).Info("Because we are running in development mode, the service won't be stopped")
}
Expand Down Expand Up @@ -335,7 +333,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i
fts.CurrentTokenID = enrollmentKey.ID

var fleetConfig *kibana.FleetConfig
fleetConfig, err = deployAgentToFleet(agentInstaller, containerName, fts.CurrentToken)
fleetConfig, err = deployAgentToFleet(agentInstaller, fts.deployer, containerName, fts.CurrentToken)

if err != nil {
return err
Expand Down Expand Up @@ -987,7 +985,8 @@ func (fts *FleetTestSuite) anAttemptToEnrollANewAgentFails() error {

containerName := fts.getContainerName(agentInstaller, 2) // name of the new container

fleetConfig, err := deployAgentToFleet(agentInstaller, containerName, fts.CurrentToken)
fleetConfig, err := deployAgentToFleet(agentInstaller, fts.deployer, containerName, fts.CurrentToken)

// the installation process for TAR includes the enrollment
if agentInstaller.InstallerType != "tar" {
if err != nil {
Expand Down Expand Up @@ -1125,7 +1124,7 @@ func (fts *FleetTestSuite) checkDataStream() error {
return err
}

func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containerName string, token string) (*kibana.FleetConfig, error) {
func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, deployer deploy.Deployment, containerName string, token string) (*kibana.FleetConfig, error) {
profile := agentInstaller.Profile // name of the runtime dependencies compose file
service := agentInstaller.Service // name of the service
serviceTag := agentInstaller.Tag // docker tag of the service
Expand All @@ -1137,9 +1136,8 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe
// we are setting the container name because Centos service could be reused by any other test suite
common.ProfileEnv[envVarsPrefix+"ContainerName"] = containerName

serviceManager := compose.NewServiceManager()

err := serviceManager.AddServicesToCompose(context.Background(), profile, []string{service}, common.ProfileEnv)
services := []string{profile, service}
err := deployer.Add(services, common.ProfileEnv)
if err != nil {
log.WithFields(log.Fields{
"service": service,
Expand Down
82 changes: 21 additions & 61 deletions e2e/_suites/fleet/ingest_manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,15 @@
package main

import (
"context"
"os"
"strings"
"time"

"github.com/cucumber/godog"
"github.com/cucumber/messages-go/v10"
"github.com/elastic/e2e-testing/cli/config"
"github.com/elastic/e2e-testing/internal/common"
"github.com/elastic/e2e-testing/internal/compose"
"github.com/elastic/e2e-testing/internal/deploy"
"github.com/elastic/e2e-testing/internal/docker"
"github.com/elastic/e2e-testing/internal/elasticsearch"
"github.com/elastic/e2e-testing/internal/installer"
"github.com/elastic/e2e-testing/internal/kibana"
"github.com/elastic/e2e-testing/internal/shell"
Expand All @@ -34,6 +31,8 @@ func setUpSuite() {
log.Error(err)
os.Exit(1)
}

common.Provider = shell.GetEnv("PROVIDER", common.Provider)
developerMode := shell.GetEnvBool("DEVELOPER_MODE")
if developerMode {
log.Info("Running in Developer mode 💻: runtime dependencies between different test runs will be reused to speed up dev cycle")
Expand Down Expand Up @@ -87,6 +86,7 @@ func setUpSuite() {
imts = IngestManagerTestSuite{
Fleet: &FleetTestSuite{
kibanaClient: kibanaClient,
deployer: deploy.New(common.Provider),
Installers: map[string]installer.ElasticAgentInstaller{}, // do not pre-initialise the map
},
}
Expand All @@ -110,17 +110,12 @@ func InitializeIngestManagerTestScenario(ctx *godog.ScenarioContext) {
}

func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) {
serviceManager := compose.NewServiceManager()
developerMode := shell.GetEnvBool("DEVELOPER_MODE")

ctx.BeforeSuite(func() {
setUpSuite()

log.Trace("Installing Fleet runtime dependencies")

common.ProfileEnv = map[string]string{
"kibanaVersion": common.KibanaVersion,
"stackVersion": common.StackVersion,
}
log.Trace("Bootstrapping Fleet Server")

if !shell.GetEnvBool("SKIP_PULL") {
images := []string{
Expand All @@ -138,63 +133,28 @@ func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) {
docker.PullImages(images)
}

common.ProfileEnv["kibanaDockerNamespace"] = "kibana"
if strings.HasPrefix(common.KibanaVersion, "pr") || utils.IsCommit(common.KibanaVersion) {
// because it comes from a PR
common.ProfileEnv["kibanaDockerNamespace"] = "observability-ci"
}

profile := common.FleetProfileName
err := serviceManager.RunCompose(context.Background(), true, []string{profile}, common.ProfileEnv)
if err != nil {
log.WithFields(log.Fields{
"profile": profile,
"error": err.Error(),
}).Fatal("Could not run the runtime dependencies for the profile.")
}

minutesToBeHealthy := time.Duration(common.TimeoutFactor) * time.Minute
healthy, err := elasticsearch.WaitForElasticsearch(context.Background(), minutesToBeHealthy)
if !healthy {
log.WithFields(log.Fields{
"error": err,
"minutes": minutesToBeHealthy,
}).Fatal("The Elasticsearch cluster could not get the healthy status")
}

kibanaClient, err := kibana.NewClient()
if err != nil {
log.WithFields(log.Fields{
"error": err,
}).Fatal("Unable to create kibana client")
}

healthyKibana, err := kibanaClient.WaitForReady(minutesToBeHealthy)
if !healthyKibana {
log.WithFields(log.Fields{
"error": err,
"minutes": minutesToBeHealthy,
}).Fatal("The Kibana instance could not get the healthy status")
}

imts.Fleet.setup()
deployer := deploy.New(common.Provider)
deployer.Bootstrap(func() error {
kibanaClient, err := kibana.NewClient()
if err != nil {
log.WithField("error", err).Fatal("Unable to create kibana client")
}
err = kibanaClient.WaitForFleet()
if err != nil {
log.WithField("error", err).Fatal("Fleet could not be initialized")
}
return nil
})

imts.Fleet.Version = common.AgentVersionBase
imts.Fleet.RuntimeDependenciesStartDate = time.Now().UTC()
})

ctx.AfterSuite(func() {
developerMode := shell.GetEnvBool("DEVELOPER_MODE")
if !developerMode {
log.Debug("Destroying Fleet runtime dependencies")
profile := common.FleetProfileName

err := serviceManager.StopCompose(context.Background(), true, []string{profile})
if err != nil {
log.WithFields(log.Fields{
"error": err,
"profile": profile,
}).Warn("Could not destroy the runtime dependencies for the profile.")
}
deployer := deploy.New(common.Provider)
deployer.Destroy()
}

installers := imts.Fleet.Installers
Expand Down
18 changes: 7 additions & 11 deletions e2e/_suites/fleet/stand-alone.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,17 @@ package main
import (
"context"
"fmt"
"path"
"strings"
"time"

"github.com/cenkalti/backoff/v4"
"github.com/elastic/e2e-testing/cli/config"
"github.com/elastic/e2e-testing/internal/common"
"github.com/elastic/e2e-testing/internal/compose"
"github.com/elastic/e2e-testing/internal/docker"
"github.com/elastic/e2e-testing/internal/installer"
"github.com/elastic/e2e-testing/internal/shell"
"github.com/elastic/e2e-testing/internal/utils"
"path"
"strings"
"time"

"github.com/elastic/e2e-testing/internal/elasticsearch"
log "github.com/sirupsen/logrus"
Expand Down Expand Up @@ -62,9 +62,7 @@ func (fts *FleetTestSuite) thereIsNewDataInTheIndexFromAgent() error {
}

func (fts *FleetTestSuite) theDockerContainerIsStopped(serviceName string) error {
serviceManager := compose.NewServiceManager()

err := serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName}, common.ProfileEnv)
err := fts.deployer.Remove([]string{common.FleetProfileName, serviceName}, common.ProfileEnv)
if err != nil {
return err
}
Expand Down Expand Up @@ -111,8 +109,6 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, composeFilename st
dockerImageTag += "-amd64"
}

serviceManager := compose.NewServiceManager()

common.ProfileEnv["elasticAgentDockerImageSuffix"] = ""
if image != "default" {
common.ProfileEnv["elasticAgentDockerImageSuffix"] = "-" + image
Expand All @@ -130,8 +126,8 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, composeFilename st
common.ProfileEnv[k] = v
}

err := serviceManager.AddServicesToCompose(context.Background(), common.FleetProfileName,
[]string{common.ElasticAgentServiceName}, common.ProfileEnv, composeFilename)
services := []string{common.FleetProfileName, common.ElasticAgentServiceName}
err := fts.deployer.Add(services, common.ProfileEnv)
if err != nil {
log.Error("Could not deploy the elastic-agent")
return err
Expand Down
3 changes: 3 additions & 0 deletions internal/common/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,6 @@ var KibanaVersion = AgentVersionBase
// ProfileEnv is the environment to be applied to any execution
// affecting the runtime dependencies (or profile)
var ProfileEnv map[string]string

// Provider is the deployment provider used, currently docker is supported
var Provider = "docker"
39 changes: 39 additions & 0 deletions internal/deploy/base.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package deploy

import (
"strings"
)

// Deployment interface for operations dealing with deployments of the bits
// required for testing
type Deployment interface {
Add(services []string, env map[string]string) error // adds a service to deployment
Bootstrap(waitCB func() error) error // will bootstrap or reuse existing cluster if kubernetes is selected
Destroy() error // Teardown deployment
ExecIn(service string, cmd []string) (string, error) // Execute arbitrary commands in service
Inspect(service string) (*ServiceManifest, error) // inspects service
Remove(services []string, env map[string]string) error // Removes services from deployment
}

// ServiceManifest information about a service in a deployment
type ServiceManifest struct {
ID string
Name string
Connection string // a string representing how to connect to service
Hostname string
}

// New creates a new deployment
func New(provider string) Deployment {
if strings.EqualFold(provider, "docker") {
return newDockerDeploy()
}
if strings.EqualFold(provider, "kubernetes") {
return newK8sDeploy()
}
return nil
}
Loading

0 comments on commit 55eee96

Please sign in to comment.