Add AWS tests to Jenkinsfile

.ci/scripts/install-terraform.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+MSG="parameter missing."
+TERRAFORM_VERSION=${TERRAFORM_VERSION:?$MSG}
+HOME=${HOME:?$MSG}
+TERRAFORM_CMD="${HOME}/bin/terraform"
+
+OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo - "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${OS}_amd64.zip" > ${TERRAFORM_CMD}.zip
+unzip -o ${TERRAFORM_CMD}.zip -d $(dirname ${TERRAFORM_CMD})
+rm ${TERRAFORM_CMD}.zip
+
+chmod +x "${TERRAFORM_CMD}"

.ci/scripts/terraform-cleanup.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+DIRECTORY=${1:-.}
+
+FAILED=0
+for tfstate in $(find $DIRECTORY -name terraform.tfstate); do
+  cd $(dirname $tfstate)
+  if ! terraform destroy -auto-approve; then
+    FAILED=1
+  fi
+  cd -
+done
+
+exit $FAILED

.gitignore

@@ -39,3 +39,7 @@ x-pack/dockerlogbeat/temproot.tar
 *.test
 *.prof
 *.pyc
+
+# Terraform
+*.terraform
+*.tfstate*

Jenkinsfile

@@ -15,9 +15,11 @@ pipeline {
     BASE_DIR = 'src/github.com/elastic/beats'
     GOX_FLAGS = "-arch amd64"
     DOCKER_COMPOSE_VERSION = "1.21.0"
+    TERRAFORM_VERSION = "0.12.24"
     PIPELINE_LOG_LEVEL = "INFO"
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_REGISTRY = 'docker.elastic.co'
+    AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
     RUNBLD_DISABLE_NOTIFICATIONS = 'true'
   }
   options {
@@ -36,6 +38,11 @@ pipeline {
     booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.')
     booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.')
     booleanParam(name: 'macosTest', defaultValue: true, description: 'Allow macOS stages.')
+
+    booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.')
+    booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.')
+    string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.')
+
     booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
     booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
   }
@@ -352,8 +359,30 @@ pipeline {
               return env.BUILD_METRICBEAT_XPACK != "false"
             }
           }
-          steps {
-            mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test")
+          stages {
+            stage('Prepare cloud integration tests environments'){
+              agent { label 'ubuntu && immutable' }
+              options { skipDefaultCheckout() }
+              steps {
+                startCloudTestEnv('x-pack-metricbeat', [
+                   [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'],
+                ])
+              }
+            }
+            stage('Metricbeat x-pack'){
+              agent { label 'ubuntu && immutable' }
+              options { skipDefaultCheckout() }
+              steps {
+                withCloudTestEnv() {
+                  mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test")
+                }
+              }
+            }
+          }
+          post {
+            cleanup {
+              terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat')
+            }
           }
         }
         stage('Metricbeat crosscompile'){
@@ -683,7 +712,7 @@ def withBeatsEnv(boolean archive, Closure body) {
     "TEST_COVERAGE=true",
     "RACE_DETECTOR=true",
     "PYTHON_ENV=${WORKSPACE}/python-env",
-    "TEST_TAGS=oracle",
+    "TEST_TAGS=${env.TEST_TAGS},oracle",
     "DOCKER_PULL=0",
   ]) {
     deleteDir()
@@ -750,6 +779,7 @@ def installTools() {
   if(isUnix()) {
     retry(i) { sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") }
     retry(i) { sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") }
+    retry(i) { sh(label: "Install Terraform ${TERRAFORM_VERSION}", script: ".ci/scripts/install-terraform.sh") }
     retry(i) { sh(label: "Install Mage", script: "make mage") }
   } else {
     retry(i) { bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") }
@@ -821,6 +851,7 @@ def dumpFilteredEnvironment(){
   echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}"
   echo "STRESS_TESTS: ${env.STRESS_TESTS}"
   echo "STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS}"
+  echo "TEST_TAGS: ${env.TEST_TAGS}"
   echo "GOX_OS: ${env.GOX_OS}"
   echo "GOX_OSARCH: ${env.GOX_OSARCH}"
   echo "GOX_FLAGS: ${env.GOX_FLAGS}"
@@ -907,6 +938,98 @@ def isChangedXPackCode(patterns) {
   return isChanged(allPatterns)
 }
 
+// withCloudTestEnv executes a closure with credentials for cloud test
+// environments.
+def withCloudTestEnv(Closure body) {
+  def maskedVars = []
+  def testTags = "${env.TEST_TAGS}"
+
+  // AWS
+  if (params.allCloudTests || params.awsCloudTests) {
+    testTags = "${testTags},aws"
+    def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data
+    if (!aws.containsKey('access_key')) {
+      error("${AWS_ACCOUNT_SECRET} doesn't contain 'access_key'")
+    }
+    if (!aws.containsKey('secret_key')) {
+      error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'")
+    }
+    maskedVars.addAll([
+      [var: "AWS_REGION", password: params.awsRegion],
+      [var: "AWS_ACCESS_KEY_ID", password: aws.access_key],
+      [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key],
+    ])
+  }
+
+  withEnv([
+    "TEST_TAGS=${testTags}",
+  ]) {
+    withEnvMask(vars: maskedVars) {
+      body()
+    }
+  }
+}
+
+def terraformInit(String directory) {
+  dir(directory) {
+    sh(label: "Terraform Init on ${directory}", script: "terraform init")
+  }
+}
+
+def terraformApply(String directory) {
+  terraformInit(directory)
+  dir(directory) {
+    sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve")
+  }
+}
+
+// Start testing environment on cloud using terraform. Terraform files are
+// stashed so they can be used by other stages. They are also archived in
+// case manual cleanup is needed.
+//
+// Example:
+//   startCloudTestEnv('x-pack-metricbeat', [
+//     [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'],
+//   ])
+//   ...
+//   terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat')
+def startCloudTestEnv(String name, environments = []) {
+  withCloudTestEnv() {
+    withBeatsEnv(false) {
+      def runAll = params.runAllCloudTests
+      try {
+        for (environment in environments) {
+          if (environment.cond || runAll) {
+            retry(2) {
+              terraformApply(environment.dir)
+            }
+          }
+        }
+      } finally {
+        // Archive terraform states in case manual cleanup is needed.
+        archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate')
+      }
+      stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**')
+    }
+  }
+}
+
+
+// Looks for all terraform states in directory and runs terraform destroy for them,
+// it uses terraform states previously stashed by startCloudTestEnv.
+def terraformCleanup(String stashName, String directory) {
+  stage("Remove cloud scenarios in ${directory}"){
+    withCloudTestEnv() {
+      withBeatsEnv(false) {
+        unstash "terraform-${stashName}"
+        retry(2) {
+          sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}")
+        }
+      }
+    }
+  }
+}
+
 def loadConfigEnvVars(){
   def empty = []
   env.GO_VERSION = readFile(".go-version").trim()

dev-tools/mage/common.go

@@ -833,3 +833,28 @@ func ListMatchingEnvVars(prefixes ...string) []string {
 	}
 	return vars
 }
+
+// IntegrationTestEnvVars returns the names of environment variables needed to configure
+// connections to integration test environments.
+func IntegrationTestEnvVars() []string {
+	// Environment variables that can be configured with paths to files
+	// with authentication information.
+	vars := []string{
+		"AWS_SHARED_CREDENTIAL_FILE",
+		"AZURE_AUTH_LOCATION",
+		"GOOGLE_APPLICATION_CREDENTIALS",
+	}
+	// Environment variables with authentication information.
+	prefixes := []string{
+		"AWS_",
+		"AZURE_",
+
+		// Accepted by terraform, but not by many clients, including Beats
+		"GOOGLE_",
+		"GCLOUD_",
+	}
+	for _, prefix := range prefixes {
+		vars = append(vars, ListMatchingEnvVars(prefix)...)
+	}
+	return vars
+}

dev-tools/mage/gotest.go

@@ -156,7 +156,9 @@ func GoTestIntegrationForModule(ctx context.Context) error {
 		foundModule = true
 
 		// Set MODULE because only want that modules tests to run inside the testing environment.
-		runners, err := NewIntegrationRunners(path.Join("./module", fi.Name()), map[string]string{"MODULE": fi.Name()})
+		env := map[string]string{"MODULE": fi.Name()}
+		passThroughEnvs(env, IntegrationTestEnvVars()...)
+		runners, err := NewIntegrationRunners(path.Join("./module", fi.Name()), env)
 		if err != nil {
 			return errors.Wrapf(err, "test setup failed for module %s", fi.Name())
 		}

docs/devguide/terraform.asciidoc

@@ -0,0 +1,101 @@
+[[terraform-beats]]
+== Terraform in Beats
+
+Terraform is used to provision scenarios for integration testing of some cloud
+features. Features implementing integration tests that require the presence of
+cloud resources should have their own Terraform configuration, this configuration
+can be used when developing locally to create (and destroy) resources that allow
+to test these features.
+
+Tests requiring access to cloud providers should be disabled by default with the
+use of build tags.
+
+[[installing-terraform]]
+=== Installing Terraform
+
+Terraform is available in https://www.terraform.io/downloads.html
+
+Download it and place it in some directory in your PATH.
+
+`terraform` is the main command for Terraform and the only one that is usually
+needed to manage configurations. Terraform will also download other plugins that
+implement the specific functionality for each provider. These plugins are
+automatically managed and stored in the working copy, if you want to share the
+plugins between multiple working copies you can manually install them in the
+user the user plugins directory located at `~/.terraform.d/plugins`,
+or `%APPDATA%\terraform.d\plugins on Windows`.
+
+Plugins are available in https://registry.terraform.io/
+
+[[using-terraform]]
+=== Using Terraform
+
+The most important commands when using Terraform are:
+* `terraform init` to do some initial checks and install the required plugins.
+* `terraform apply` to create the resources defined in the configuration.
+* `terraform destroy` to destroy resources previously created.
+
+Cloud providers use to require credentials, they can be provided with the usual
+methods supported by these providers, using environment variables and/or
+credential files.
+
+Terraform stores the last known state of the resources managed by a
+configuration in a `terraform.tfstate` file. It is important to keep this file
+as it is used as input by `terraform destroy`. This file is created in the same
+directory where `terraform apply` is executed.
+
+Please take a look to Terraform documentation for more details: https://www.terraform.io/intro/index.html
+
+[[terraform-configurations]]
+=== Terraform configuration guidelines
+
+The main purpouse of Terraform in Beats is to create and destroy cloud resources
+required by integration tests. For these configurations there are some things to
+take into account:
+* Apply should work without additional inputs or files. Only input will be the
+  required for specific providers, using environment variables or credential
+  files.
+* You must be able to apply the same configuration multiple times in the same
+  account. This will allow to have multiple builds using the same configuration
+  but with different instances of the resources. Some resources are already
+  created with unique identifiers (as EC2 instances), some others have to be
+  explicitly created with unique names (e.g. S3 buckets). For these cases random
+  suffixes can be added to identifiers.
+* Destroy must work without additional input, and should be able to destroy all
+  the resources created by the configuration. There are some resources that need
+  specific flags to be destroyed by `terraform destroy`. For example S3 buckets
+  need a flag to force to empty the bucket before deleting it, or RDS instances
+  need a flag to disable snapshots on deletion.
+
+[[terraform-in-ci]]
+=== Terraform in CI
+
+Integration tests that need the presence of certain resources to work can be
+executed in CI if they provide a Terraform configuration to start these
+resources. These tests are disabled by default in CI.
+
+Terraform states are archived as artifacrs of builds, this allows to manually
+destroy resources created by builds that were not able to do a proper cleanup.
+
+Here is a checklist to add support for a cloud feature in Jenkins:
+* In the feature code:
+  * Tests have a build tag so they are disabled by default. When run from mage,
+    its execution can be selected using the `TEST_TAGS` environment variable, e.g:
+    `TEST_TAGS=aws` for AWS tests.
+  * There is some Terraform configuration that defines a cloud scenario where
+    tests pass. This configuration should be in the directory of the feature.
+* In the Jenkinsfile:
+  * Add a boolean parameter to run the tests on this environment, e.g.
+    `awsCloudTests`. This parameter should be set to false by default.
+  * Add a conditional block in `withCloudTestEnv` that:
+     * Will be executed if the previously added boolean parameter, or `allCloudTests`
+       are set to true.
+     * Adds the tag to `TEST_TAGS` (as comma separated values), so tests are
+       selected.
+     * Defines how to obtain the credentials and provide them to the tests.
+  * In the stage of the specific beat:
+    * Add a stage that calls to `startCloudTestEnv`, if there isn't anyone.
+    * Add a post cleanup step that calls to `terraformCleanup`, if there isn't anyone.
+    * Add a environment to the list of environments started by `startCloudEnv`,
+      with the condition to start the scenario, and the path to the directory
+      with its definition, e.g. `[cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws']`