From 3cd95c462281655ccc528678989a193fba3e1ac5 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 3 Apr 2020 19:16:11 +0200
Subject: [PATCH 01/21] Initial attempt of conditional AWS tests

---
 Jenkinsfile | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 50 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5761f41b0f8..2ba24a18bbc 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -28,6 +28,11 @@ pipeline {
     booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.')
     booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.')
     booleanParam(name: 'macosTest', defaultValue: true, description: 'Allow macOS stages.')
+
+    booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.')
+    // XXX: Set to false by default
+    booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.')
+
     booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
     booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
   }
@@ -55,6 +60,23 @@ pipeline {
         makeTarget("Lint", "check")
       }
     }
+    stage('Prepare integration tests environments'){
+      parallel {
+        stage('AWS cloud scenario'){
+          agent { label 'ubuntu && immutable' }
+          options { skipDefaultCheckout() }
+          when {
+            expression {
+              return params.runAllCloudTests || params.awsCloudTests
+            }
+          }
+          steps {
+            echo "### TODO: provision AWS cloud test scenario"
+            // XXX: Run terraform or ansible to provision resources in AWS
+          }
+        }
+      }
+    }
     stage('Build and Test'){
       failFast false
       parallel {
@@ -345,6 +367,7 @@ pipeline {
             }
           }
           steps {
+            loadCloudTestEnv()
             mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test")
           }
         }
@@ -593,6 +616,11 @@ pipeline {
       }
     }
   }
+  post {
+    cleanup {
+      cleanupAWS()
+    }
+  }
 }
 
 def makeTarget(String context, String target, boolean clean = true) {
@@ -655,7 +683,7 @@ def withBeatsEnv(boolean archive, Closure body) {
     "TEST_COVERAGE=true",
     "RACE_DETECTOR=true",
     "PYTHON_ENV=${WORKSPACE}/python-env",
-    "TEST_TAGS=oracle",
+    "TEST_TAGS=${env.TEST_TAGS},oracle",
     "DOCKER_PULL=0",
   ]) {
     deleteDir()
@@ -872,6 +900,27 @@ def isChangedXPackCode(patterns) {
   return isChanged(allPatterns)
 }
 
+def loadCloudTestEnv() {
+  if (params.allCloudTests || params.awsCloudTests) {
+    env.TEST_TAGS="${env.TEST_TAGS},aws"
+    // XXX: credentials
+  }
+}
+
+def cleanupAWS() {
+  stage('Remove AWS cloud scenario'){
+    when {
+      expression {
+        return params.runAllCloudTests || params.awsCloudTests
+      }
+    }
+    steps {
+      echo "### TODO: cleanup AWS cloud test scenario"
+      // XXX: Run terraform or ansible to cleanup resources in AWS
+    }
+  }
+}
+
 def loadConfigEnvVars(){
   def empty = []
   env.GO_VERSION = readFile(".go-version").trim()

From 41c0a416cb41e241c718a3a28b02328b2adfc3a0 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 6 Apr 2020 13:53:51 +0200
Subject: [PATCH 02/21] Print TEST_TAGS env var

---
 Jenkinsfile | 57 +++++++++++++++++++++++++++++------------------------
 1 file changed, 31 insertions(+), 26 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 2ba24a18bbc..b5189931537 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -33,7 +33,8 @@ pipeline {
     // XXX: Set to false by default
     booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.')
 
-    booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
+    // XXX: Set to false by default
+    booleanParam(name: 'debug', defaultValue: true, description: 'Allow debug logging for Jenkins steps')
     booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
   }
   stages {
@@ -60,23 +61,6 @@ pipeline {
         makeTarget("Lint", "check")
       }
     }
-    stage('Prepare integration tests environments'){
-      parallel {
-        stage('AWS cloud scenario'){
-          agent { label 'ubuntu && immutable' }
-          options { skipDefaultCheckout() }
-          when {
-            expression {
-              return params.runAllCloudTests || params.awsCloudTests
-            }
-          }
-          steps {
-            echo "### TODO: provision AWS cloud test scenario"
-            // XXX: Run terraform or ansible to provision resources in AWS
-          }
-        }
-      }
-    }
     stage('Build and Test'){
       failFast false
       parallel {
@@ -366,9 +350,34 @@ pipeline {
               return env.BUILD_METRICBEAT_XPACK != "false"
             }
           }
-          steps {
-            loadCloudTestEnv()
-            mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test")
+          stages {
+            stage('Prepare cloud integration tests environments'){
+              agent { label 'ubuntu && immutable' }
+              options { skipDefaultCheckout() }
+              when {
+                expression {
+                  return params.runAllCloudTests || params.awsCloudTests
+                }
+              }
+              steps {
+                echo "### TODO: provision AWS cloud test scenario"
+                // XXX: Run terraform or ansible to provision resources in AWS
+                // XXX: Parallelize at the script/mage level as cannot nest parallel blocks here
+              }
+            }
+            stage('Metricbeat x-pack'){
+              agent { label 'ubuntu && immutable' }
+              options { skipDefaultCheckout() }
+              steps {
+                loadCloudTestEnv()
+                mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test")
+              }
+            }
+          }
+          post {
+            cleanup {
+              cleanupAWS()
+            }
           }
         }
         stage('Metricbeat crosscompile'){
@@ -616,11 +625,6 @@ pipeline {
       }
     }
   }
-  post {
-    cleanup {
-      cleanupAWS()
-    }
-  }
 }
 
 def makeTarget(String context, String target, boolean clean = true) {
@@ -812,6 +816,7 @@ def dumpFilteredEnvironment(){
   echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}"
   echo "STRESS_TESTS: ${env.STRESS_TESTS}"
   echo "STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS}"
+  echo "TEST_TAGS: ${env.TEST_TAGS}"
   echo "GOX_OS: ${env.GOX_OS}"
   echo "GOX_OSARCH: ${env.GOX_OSARCH}"
   echo "GOX_FLAGS: ${env.GOX_FLAGS}"

From 5c6154ffcd31ab0a4286c2253729159ac9c16cc1 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 21 Apr 2020 11:13:25 +0200
Subject: [PATCH 03/21] Add credentials

---
 Jenkinsfile                 | 36 ++++++++++++++++++++++++++++++------
 dev-tools/mage/integtest.go |  9 ++++++++-
 2 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b5189931537..19a9dbd9e88 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -11,6 +11,7 @@ pipeline {
     PIPELINE_LOG_LEVEL = "INFO"
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_REGISTRY = 'docker.elastic.co'
+    AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
   }
   options {
     timeout(time: 2, unit: 'HOURS')
@@ -58,7 +59,9 @@ pipeline {
     stage('Lint'){
       options { skipDefaultCheckout() }
       steps {
-        makeTarget("Lint", "check")
+        // XXX: Comment this in
+        // makeTarget("Lint", "check")
+        echo "skipping lint"
       }
     }
     stage('Build and Test'){
@@ -369,8 +372,9 @@ pipeline {
               agent { label 'ubuntu && immutable' }
               options { skipDefaultCheckout() }
               steps {
-                loadCloudTestEnv()
-                mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test")
+                withCloudTestEnv() {
+                  mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test")
+                }
               }
             }
           }
@@ -905,10 +909,30 @@ def isChangedXPackCode(patterns) {
   return isChanged(allPatterns)
 }
 
-def loadCloudTestEnv() {
+def withCloudTestEnv(Closure body) {
+  def maskedVars = []
+  def testTags = "${env.TEST_TAGS}"
+
+  // AWS
   if (params.allCloudTests || params.awsCloudTests) {
-    env.TEST_TAGS="${env.TEST_TAGS},aws"
-    // XXX: credentials
+    testTags = "${testTags},aws"
+    def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data
+    if (!aws.containsKey('access_key')) {
+      error("${AWS_ACCOUNT_SECRET} doesn't contain 'access_key'")
+    }
+    if (!aws.containsKey('secret_key')) {
+      error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'")
+    }
+    maskedVars.addAll([
+      [var: "AWS_ACCESS_KEY_ID", password: aws.access_key],
+      [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key],
+    ])
+  }
+
+  withEnv(["TEST_TAGS=${testTags}"]) {
+    withEnvMask(vars: maskedVars) {
+      body()
+    }
   }
 }
 
diff --git a/dev-tools/mage/integtest.go b/dev-tools/mage/integtest.go
index 6fc26915415..688d941dbdb 100644
--- a/dev-tools/mage/integtest.go
+++ b/dev-tools/mage/integtest.go
@@ -150,6 +150,11 @@ func RunIntegTest(mageTarget string, test func() error, passThroughEnvVars ...st
 		"TEST_TAGS",
 		"PYTHON_EXE",
 		"MODULE",
+
+		// XXX: Move these variables to some other place?
+		"AWS_SESSION_TOKEN",
+		"AWS_ACCESS_KEY_ID",
+		"AWS_SECRET_ACCESS_KEY",
 	}
 	env = append(env, passThroughEnvVars...)
 	return runInIntegTestEnv(mageTarget, test, env...)
@@ -204,7 +209,9 @@ func runInIntegTestEnv(mageTarget string, test func() error, passThroughEnvVars
 		return err
 	}
 	for _, envVar := range passThroughEnvVars {
-		args = append(args, "-e", envVar+"="+os.Getenv(envVar))
+		if value, ok := os.LookupEnv(envVar); ok {
+			args = append(args, "-e", envVar+"="+value)
+		}
 	}
 	if mg.Verbose() {
 		args = append(args, "-e", "MAGEFILE_VERBOSE=1")

From 81d54d821f3c0169748535c814c30eaabdb7f660 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 22 Apr 2020 12:04:16 +0200
Subject: [PATCH 04/21] Create resources with terraform

---
 .ci/scripts/install-terraform.sh          | 18 +++++++++
 .ci/scripts/terraform-cleanup.sh          | 16 ++++++++
 .gitignore                                |  4 ++
 Jenkinsfile                               | 41 ++++++++++++++++-----
 x-pack/metricbeat/module/aws/terraform.tf | 45 +++++++++++++++++++++++
 5 files changed, 115 insertions(+), 9 deletions(-)
 create mode 100755 .ci/scripts/install-terraform.sh
 create mode 100755 .ci/scripts/terraform-cleanup.sh
 create mode 100644 x-pack/metricbeat/module/aws/terraform.tf

diff --git a/.ci/scripts/install-terraform.sh b/.ci/scripts/install-terraform.sh
new file mode 100755
index 00000000000..39aa684d0aa
--- /dev/null
+++ b/.ci/scripts/install-terraform.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+MSG="parameter missing."
+TERRAFORM_VERSION=${TERRAFORM_VERSION:?$MSG}
+HOME=${HOME:?$MSG}
+TERRAFORM_CMD="${HOME}/bin/terraform"
+
+OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo - "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${OS}_amd64.zip" > ${TERRAFORM_CMD}.zip
+unzip -o ${TERRAFORM_CMD}.zip -d $(dirname ${TERRAFORM_CMD})
+rm ${TERRAFORM_CMD}.zip
+
+chmod +x "${TERRAFORM_CMD}"
diff --git a/.ci/scripts/terraform-cleanup.sh b/.ci/scripts/terraform-cleanup.sh
new file mode 100755
index 00000000000..4fd3cdd6e16
--- /dev/null
+++ b/.ci/scripts/terraform-cleanup.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+DIRECTORY=${1:-.}
+
+FAILED=0
+for tfstate in $(find $DIRECTORY -name terraform.tfstate); do
+	cd $(dirname $tfstate)
+        if ! terraform destroy -auto-approve; then
+		FAILED=1
+	fi
+	cd -
+done
+
+exit $FAILED
diff --git a/.gitignore b/.gitignore
index 34266183ac2..fe3a1459213 100644
--- a/.gitignore
+++ b/.gitignore
@@ -39,3 +39,7 @@ x-pack/dockerlogbeat/temproot.tar
 *.test
 *.prof
 *.pyc
+
+# Terraform
+*.terraform
+*.tfstate*
diff --git a/Jenkinsfile b/Jenkinsfile
index 19a9dbd9e88..480b522fdfa 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -8,6 +8,7 @@ pipeline {
     BASE_DIR = 'src/github.com/elastic/beats'
     GOX_FLAGS = "-arch amd64"
     DOCKER_COMPOSE_VERSION = "1.21.0"
+    TERRAFORM_VERSION = "0.12.24"
     PIPELINE_LOG_LEVEL = "INFO"
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_REGISTRY = 'docker.elastic.co'
@@ -363,9 +364,11 @@ pipeline {
                 }
               }
               steps {
-                echo "### TODO: provision AWS cloud test scenario"
-                // XXX: Run terraform or ansible to provision resources in AWS
-                // XXX: Parallelize at the script/mage level as cannot nest parallel blocks here
+                withCloudTestEnv() {
+                  withBeatsEnv(false) {
+                    terraformApply("x-pack/metricbeat/module/aws")
+                  }
+                }
               }
             }
             stage('Metricbeat x-pack'){
@@ -380,7 +383,7 @@ pipeline {
           }
           post {
             cleanup {
-              cleanupAWS()
+              terraformCleanup("x-pack/metricbeat")
             }
           }
         }
@@ -702,6 +705,7 @@ def withBeatsEnv(boolean archive, Closure body) {
     dir("${env.BASE_DIR}") {
       sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh")
       sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh")
+      sh(label: "Install Terraform ${TERRAFORM_VERSION}", script: ".ci/scripts/install-terraform.sh")
       sh(label: "Install Mage", script: "make mage")
       // TODO (2020-04-07): This is a work-around to fix the Beat generator tests.
       // See https://github.com/elastic/beats/issues/17787.
@@ -929,23 +933,42 @@ def withCloudTestEnv(Closure body) {
     ])
   }
 
-  withEnv(["TEST_TAGS=${testTags}"]) {
+  withEnv([
+    "TEST_TAGS=${testTags}",
+  ]) {
     withEnvMask(vars: maskedVars) {
       body()
     }
   }
 }
 
-def cleanupAWS() {
-  stage('Remove AWS cloud scenario'){
+def terraformInit(String directory) {
+  dir(directory) {
+    sh(label: "Terraform Init on ${directory}", script: "terraform init")
+  }
+}
+
+def terraformApply(String directory) {
+  terraformInit(directory)
+  dir(directory) {
+    sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve")
+  }
+}
+
+// Looks for all terraform states in `directory` and runs `terraform destroy` for them
+def terraformCleanup(String directory) {
+  stage("Remove cloud scenarios in ${directory}"){
     when {
       expression {
         return params.runAllCloudTests || params.awsCloudTests
       }
     }
     steps {
-      echo "### TODO: cleanup AWS cloud test scenario"
-      // XXX: Run terraform or ansible to cleanup resources in AWS
+      withCloudTestEnv() {
+        withBeatsEnv(false) {
+          sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh $directory")
+        }
+      }
     }
   }
 }
diff --git a/x-pack/metricbeat/module/aws/terraform.tf b/x-pack/metricbeat/module/aws/terraform.tf
new file mode 100644
index 00000000000..b9c42d6ae67
--- /dev/null
+++ b/x-pack/metricbeat/module/aws/terraform.tf
@@ -0,0 +1,45 @@
+provider "aws" {
+  version = "~> 2.58"
+  region  = "eu-central-1"
+}
+
+provider "random" {
+  version = "~> 2.2"
+}
+
+resource "random_id" "suffix" {
+  byte_length = 4
+}
+
+resource "random_password" "db" {
+  length           = 16
+  special          = true
+  override_special = "/@ "
+}
+
+resource "aws_db_instance" "test" {
+  identifier          = "metricbeat-test-${random_id.suffix.hex}"
+  allocated_storage   = 20 // Gigabytes
+  engine              = "mysql"
+  instance_class      = "db.t2.micro"
+  name                = "metricbeattest"
+  username            = "foo"
+  password            = random_password.db.result
+  skip_final_snapshot = true // Required for cleanup
+}
+
+resource "aws_sqs_queue" "test" {
+  name                      = "metricbeat-test-${random_id.suffix.hex}"
+  receive_wait_time_seconds = 10
+}
+
+resource "aws_s3_bucket" "test" {
+  bucket        = "metricbeat-test-${random_id.suffix.hex}"
+  force_destroy = true // Required for cleanup
+}
+
+resource "aws_s3_bucket_object" "test" {
+  key     = "someobject"
+  bucket  = aws_s3_bucket.test.id
+  content = "something"
+}

From 1003d453fc75e0751167261d6ed4958c3122990e Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 22 Apr 2020 19:21:33 +0200
Subject: [PATCH 05/21] Stash terraform files for cleanup

---
 Jenkinsfile                               | 25 +++++++++++++++--------
 x-pack/metricbeat/module/aws/terraform.tf |  5 ++---
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 480b522fdfa..453b1a92eb7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -37,7 +37,8 @@ pipeline {
 
     // XXX: Set to false by default
     booleanParam(name: 'debug', defaultValue: true, description: 'Allow debug logging for Jenkins steps')
-    booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
+    // XXX: Set to false by default
+    booleanParam(name: 'dry_run', defaultValue: true, description: 'Skip build steps, it is for testing pipeline flow')
   }
   stages {
     /**
@@ -367,6 +368,7 @@ pipeline {
                 withCloudTestEnv() {
                   withBeatsEnv(false) {
                     terraformApply("x-pack/metricbeat/module/aws")
+                    terraformStash("x-pack-metricbeat")
                   }
                 }
               }
@@ -383,7 +385,13 @@ pipeline {
           }
           post {
             cleanup {
-              terraformCleanup("x-pack/metricbeat")
+              // XXX: terraformCleanup(stashName: "x-pack-metricbeat", directory: "x-pack/metricbeat")
+              withCloudTestEnv() {
+                withBeatsEnv(false) {
+                  unstash "terraform-x-pack-metricbeat"
+                  sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh x-pack/metricbeat")
+                }
+              }
             }
           }
         }
@@ -955,17 +963,18 @@ def terraformApply(String directory) {
   }
 }
 
+def terraformStash(String name) {
+  archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate')
+  stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**')
+}
+
 // Looks for all terraform states in `directory` and runs `terraform destroy` for them
-def terraformCleanup(String directory) {
+def terraformCleanup(String stashName, String directory) {
   stage("Remove cloud scenarios in ${directory}"){
-    when {
-      expression {
-        return params.runAllCloudTests || params.awsCloudTests
-      }
-    }
     steps {
       withCloudTestEnv() {
         withBeatsEnv(false) {
+          unstash "terraform-${stashName}"
           sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh $directory")
         }
       }
diff --git a/x-pack/metricbeat/module/aws/terraform.tf b/x-pack/metricbeat/module/aws/terraform.tf
index b9c42d6ae67..e96abc51590 100644
--- a/x-pack/metricbeat/module/aws/terraform.tf
+++ b/x-pack/metricbeat/module/aws/terraform.tf
@@ -12,9 +12,8 @@ resource "random_id" "suffix" {
 }
 
 resource "random_password" "db" {
-  length           = 16
-  special          = true
-  override_special = "/@ "
+  length  = 16
+  special = false
 }
 
 resource "aws_db_instance" "test" {

From 4816889eb767046d0f679f79b2972769383646d6 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Apr 2020 20:20:46 +0200
Subject: [PATCH 06/21] Try to add cleanup as function

---
 Jenkinsfile | 41 +++++++++++++++++++++--------------------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 453b1a92eb7..a9088ccb223 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -35,10 +35,8 @@ pipeline {
     // XXX: Set to false by default
     booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.')
 
-    // XXX: Set to false by default
-    booleanParam(name: 'debug', defaultValue: true, description: 'Allow debug logging for Jenkins steps')
-    // XXX: Set to false by default
-    booleanParam(name: 'dry_run', defaultValue: true, description: 'Skip build steps, it is for testing pipeline flow')
+    booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
+    booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
   }
   stages {
     /**
@@ -365,10 +363,21 @@ pipeline {
                 }
               }
               steps {
+                // XXX: Refactor this as a function like this:
+                // startCloudTestEnv('x-pack-metricbeat', [
+                //   [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'],
+                // ])
                 withCloudTestEnv() {
                   withBeatsEnv(false) {
-                    terraformApply("x-pack/metricbeat/module/aws")
-                    terraformStash("x-pack-metricbeat")
+                    script {
+                      try {
+                        whenTrue(params.runAllCloudTests || params.awsCloudTests) {
+                          terraformApply("x-pack/metricbeat/module/aws")
+                        }
+                      } finally {
+                        terraformStash("x-pack-metricbeat")
+                      }
+                    }
                   }
                 }
               }
@@ -385,13 +394,7 @@ pipeline {
           }
           post {
             cleanup {
-              // XXX: terraformCleanup(stashName: "x-pack-metricbeat", directory: "x-pack/metricbeat")
-              withCloudTestEnv() {
-                withBeatsEnv(false) {
-                  unstash "terraform-x-pack-metricbeat"
-                  sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh x-pack/metricbeat")
-                }
-              }
+              terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat')
             }
           }
         }
@@ -968,15 +971,13 @@ def terraformStash(String name) {
   stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**')
 }
 
-// Looks for all terraform states in `directory` and runs `terraform destroy` for them
+// Looks for all terraform states in directory and runs terraform destroy for them
 def terraformCleanup(String stashName, String directory) {
   stage("Remove cloud scenarios in ${directory}"){
-    steps {
-      withCloudTestEnv() {
-        withBeatsEnv(false) {
-          unstash "terraform-${stashName}"
-          sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh $directory")
-        }
+    withCloudTestEnv() {
+      withBeatsEnv(false) {
+        unstash "terraform-${stashName}"
+        sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}")
       }
     }
   }

From f770986b3e2ed50b17d7e826e6f9641398200e96 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 24 Apr 2020 11:06:34 +0200
Subject: [PATCH 07/21] Refactor env vars handling

---
 Jenkinsfile                               |  2 ++
 dev-tools/mage/common.go                  | 25 +++++++++++++++++++++++
 dev-tools/mage/gotest.go                  |  3 ++-
 dev-tools/mage/integtest.go               |  5 -----
 x-pack/metricbeat/module/aws/terraform.tf |  1 -
 5 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a9088ccb223..2740e5c6ac2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -34,6 +34,7 @@ pipeline {
     booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.')
     // XXX: Set to false by default
     booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.')
+    string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.')
 
     booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
     booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
@@ -939,6 +940,7 @@ def withCloudTestEnv(Closure body) {
       error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'")
     }
     maskedVars.addAll([
+      [var: "AWS_REGION", password: params.awsRegion],
       [var: "AWS_ACCESS_KEY_ID", password: aws.access_key],
       [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key],
     ])
diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go
index 7d31038d06d..d2e75986a29 100644
--- a/dev-tools/mage/common.go
+++ b/dev-tools/mage/common.go
@@ -821,3 +821,28 @@ func ListMatchingEnvVars(prefixes ...string) []string {
 	}
 	return vars
 }
+
+// CloudEnvVars returns the names of environment variables needed to configure
+// connections to cloud environments.
+func CloudEnvVars() []string {
+	// Environment variables that can be configured with paths to files
+	// with authentication information.
+	vars := []string{
+		"AWS_SHARED_CREDENTIAL_FILE",
+		"AZURE_AUTH_LOCATION",
+		"GOOGLE_APPLICATION_CREDENTIALS",
+	}
+	// Environment variables with authentication information.
+	prefixes := []string{
+		"AWS_",
+		"AZURE_",
+
+		// Accepted by terraform, but not by many clients, including Beats
+		"GOOGLE_",
+		"GCLOUD_",
+	}
+	for _, prefix := range prefixes {
+		vars = append(vars, ListMatchingEnvVars(prefix)...)
+	}
+	return vars
+}
diff --git a/dev-tools/mage/gotest.go b/dev-tools/mage/gotest.go
index 9f0050025f9..e5b25579cdb 100644
--- a/dev-tools/mage/gotest.go
+++ b/dev-tools/mage/gotest.go
@@ -137,6 +137,7 @@ func DefaultTestBinaryArgs() TestBinaryArgs {
 // Use RACE_DETECTOR=true to enable the race detector.
 // Use MODULE=module to run only tests for `module`.
 func GoTestIntegrationForModule(ctx context.Context) error {
+	passThruEnvVars := CloudEnvVars()
 	return RunIntegTest("goIntegTest", func() error {
 		module := EnvOr("MODULE", "")
 		if module != "" {
@@ -163,7 +164,7 @@ func GoTestIntegrationForModule(ctx context.Context) error {
 			return errors.New("integration tests failed")
 		}
 		return nil
-	})
+	}, passThruEnvVars...)
 }
 
 // GoTest invokes "go test" and reports the results to stdout. It returns an
diff --git a/dev-tools/mage/integtest.go b/dev-tools/mage/integtest.go
index 688d941dbdb..224006be156 100644
--- a/dev-tools/mage/integtest.go
+++ b/dev-tools/mage/integtest.go
@@ -150,11 +150,6 @@ func RunIntegTest(mageTarget string, test func() error, passThroughEnvVars ...st
 		"TEST_TAGS",
 		"PYTHON_EXE",
 		"MODULE",
-
-		// XXX: Move these variables to some other place?
-		"AWS_SESSION_TOKEN",
-		"AWS_ACCESS_KEY_ID",
-		"AWS_SECRET_ACCESS_KEY",
 	}
 	env = append(env, passThroughEnvVars...)
 	return runInIntegTestEnv(mageTarget, test, env...)
diff --git a/x-pack/metricbeat/module/aws/terraform.tf b/x-pack/metricbeat/module/aws/terraform.tf
index e96abc51590..fc92d2710a4 100644
--- a/x-pack/metricbeat/module/aws/terraform.tf
+++ b/x-pack/metricbeat/module/aws/terraform.tf
@@ -1,6 +1,5 @@
 provider "aws" {
   version = "~> 2.58"
-  region  = "eu-central-1"
 }
 
 provider "random" {

From fb30ac6aa58eb063d7ef7a7b7b68a5d01fd45e2a Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 24 Apr 2020 12:16:53 +0200
Subject: [PATCH 08/21] Add helper to start multiple cloud test environments

---
 .ci/scripts/terraform-cleanup.sh | 10 ++---
 Jenkinsfile                      | 65 +++++++++++++++++++++-----------
 2 files changed, 47 insertions(+), 28 deletions(-)

diff --git a/.ci/scripts/terraform-cleanup.sh b/.ci/scripts/terraform-cleanup.sh
index 4fd3cdd6e16..f1051b9b20d 100755
--- a/.ci/scripts/terraform-cleanup.sh
+++ b/.ci/scripts/terraform-cleanup.sh
@@ -6,11 +6,11 @@ DIRECTORY=${1:-.}
 
 FAILED=0
 for tfstate in $(find $DIRECTORY -name terraform.tfstate); do
-	cd $(dirname $tfstate)
-        if ! terraform destroy -auto-approve; then
-		FAILED=1
-	fi
-	cd -
+  cd $(dirname $tfstate)
+  if ! terraform destroy -auto-approve; then
+    FAILED=1
+  fi
+  cd -
 done
 
 exit $FAILED
diff --git a/Jenkinsfile b/Jenkinsfile
index 2740e5c6ac2..03567fad1ce 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -358,29 +358,10 @@ pipeline {
             stage('Prepare cloud integration tests environments'){
               agent { label 'ubuntu && immutable' }
               options { skipDefaultCheckout() }
-              when {
-                expression {
-                  return params.runAllCloudTests || params.awsCloudTests
-                }
-              }
               steps {
-                // XXX: Refactor this as a function like this:
-                // startCloudTestEnv('x-pack-metricbeat', [
-                //   [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'],
-                // ])
-                withCloudTestEnv() {
-                  withBeatsEnv(false) {
-                    script {
-                      try {
-                        whenTrue(params.runAllCloudTests || params.awsCloudTests) {
-                          terraformApply("x-pack/metricbeat/module/aws")
-                        }
-                      } finally {
-                        terraformStash("x-pack-metricbeat")
-                      }
-                    }
-                  }
-                }
+                startCloudTestEnv('x-pack-metricbeat', [
+                   [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'],
+                ])
               }
             }
             stage('Metricbeat x-pack'){
@@ -925,6 +906,8 @@ def isChangedXPackCode(patterns) {
   return isChanged(allPatterns)
 }
 
+// withCloudTestEnv executes a closure with credentials for cloud test
+// environments.
 def withCloudTestEnv(Closure body) {
   def maskedVars = []
   def testTags = "${env.TEST_TAGS}"
@@ -973,7 +956,43 @@ def terraformStash(String name) {
   stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**')
 }
 
-// Looks for all terraform states in directory and runs terraform destroy for them
+
+// Start testing environment on cloud using terraform and stash its state
+// for manual cleanup if needed.
+//
+// Example:
+//   startCloudTestEnv('x-pack-metricbeat', [
+//     [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'],
+//   ])
+//   ...
+//   terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat')
+def startCloudTestEnv(String name, environments = []) {
+  withCloudTestEnv() {
+    withBeatsEnv(false) {
+      script {
+        def runAll = params.runAllCloudTests
+        def failed = false
+        for (environment in environments) {
+          if (environment.cond || runAll) {
+            try {
+              terraformApply(environment.dir)
+            } catch (Exception e) {
+              failed = true
+            }
+          }
+        }
+        terraformStash(name)
+        if (failed) {
+          error("Some environment failed to start")
+        }
+      }
+    }
+  }
+}
+
+
+// Looks for all terraform states in directory and runs terraform destroy for them,
+// it uses terraform states previously stashed by startCloudTestEnv.
 def terraformCleanup(String stashName, String directory) {
   stage("Remove cloud scenarios in ${directory}"){
     withCloudTestEnv() {

From afb3d691d1f6a787d7adb636c2fb2b097daecdb6 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 24 Apr 2020 12:49:17 +0200
Subject: [PATCH 09/21] Revert some changes for development

---
 Jenkinsfile | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 03567fad1ce..b409cf15a48 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -32,8 +32,7 @@ pipeline {
     booleanParam(name: 'macosTest', defaultValue: true, description: 'Allow macOS stages.')
 
     booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.')
-    // XXX: Set to false by default
-    booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.')
+    booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.')
     string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.')
 
     booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
@@ -60,9 +59,7 @@ pipeline {
     stage('Lint'){
       options { skipDefaultCheckout() }
       steps {
-        // XXX: Comment this in
-        // makeTarget("Lint", "check")
-        echo "skipping lint"
+        makeTarget("Lint", "check")
       }
     }
     stage('Build and Test'){

From b9e62b0927ac0fda45145cf24ab9c412f195824e Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 24 Apr 2020 12:55:40 +0200
Subject: [PATCH 10/21] Skip s3 request integration test

---
 .../module/aws/s3_request/s3_request_integration_test.go        | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go b/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go
index eeae8439f1f..dcb6de3e193 100644
--- a/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go
+++ b/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go
@@ -17,6 +17,8 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("This test doesn't pass with current terraform scenario")
+
 	config := mtest.GetConfigForTest(t, "s3_request", "86400s")
 
 	metricSet := mbtest.NewReportingMetricSetV2Error(t, config)

From 563a4cd1ae8101a429ee94d2a44e65ea82c4266b Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 24 Apr 2020 13:36:01 +0200
Subject: [PATCH 11/21] Remove unneeded script step

---
 Jenkinsfile | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b409cf15a48..b7a3ac37218 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -966,22 +966,20 @@ def terraformStash(String name) {
 def startCloudTestEnv(String name, environments = []) {
   withCloudTestEnv() {
     withBeatsEnv(false) {
-      script {
-        def runAll = params.runAllCloudTests
-        def failed = false
-        for (environment in environments) {
-          if (environment.cond || runAll) {
-            try {
-              terraformApply(environment.dir)
-            } catch (Exception e) {
-              failed = true
-            }
+      def runAll = params.runAllCloudTests
+      def failed = false
+      for (environment in environments) {
+        if (environment.cond || runAll) {
+          try {
+            terraformApply(environment.dir)
+          } catch (Exception e) {
+            failed = true
           }
         }
-        terraformStash(name)
-        if (failed) {
-          error("Some environment failed to start")
-        }
+      }
+      terraformStash(name)
+      if (failed) {
+        error("Some environment failed to start")
       }
     }
   }

From 04c7292a9626298ce956e2035d8244992c6b46d4 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 24 Apr 2020 15:07:10 +0200
Subject: [PATCH 12/21] Fail earlier if some terraform config couldn't be
 applied

---
 Jenkinsfile | 28 ++++++++++------------------
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b7a3ac37218..3fdd4878cad 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -948,14 +948,9 @@ def terraformApply(String directory) {
   }
 }
 
-def terraformStash(String name) {
-  archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate')
-  stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**')
-}
-
-
-// Start testing environment on cloud using terraform and stash its state
-// for manual cleanup if needed.
+// Start testing environment on cloud using terraform. Terraform files are
+// stashed so they can be used by other stages. They are also archived in
+// case manual cleanup is needed.
 //
 // Example:
 //   startCloudTestEnv('x-pack-metricbeat', [
@@ -967,20 +962,17 @@ def startCloudTestEnv(String name, environments = []) {
   withCloudTestEnv() {
     withBeatsEnv(false) {
       def runAll = params.runAllCloudTests
-      def failed = false
-      for (environment in environments) {
-        if (environment.cond || runAll) {
-          try {
+      try {
+        for (environment in environments) {
+          if (environment.cond || runAll) {
             terraformApply(environment.dir)
-          } catch (Exception e) {
-            failed = true
           }
         }
+      } finally {
+        // Archive terraform states in case manual cleanup is needed.
+        archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate')
       }
-      terraformStash(name)
-      if (failed) {
-        error("Some environment failed to start")
-      }
+      stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**')
     }
   }
 }

From 35c4be05436b211c66bc474a7de417e27cf406f0 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 24 Apr 2020 15:23:22 +0200
Subject: [PATCH 13/21] Rename CloudEnvVars to IntegrationTestEnvVars

---
 dev-tools/mage/common.go | 6 +++---
 dev-tools/mage/gotest.go | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go
index d2e75986a29..89b843711e3 100644
--- a/dev-tools/mage/common.go
+++ b/dev-tools/mage/common.go
@@ -822,9 +822,9 @@ func ListMatchingEnvVars(prefixes ...string) []string {
 	return vars
 }
 
-// CloudEnvVars returns the names of environment variables needed to configure
-// connections to cloud environments.
-func CloudEnvVars() []string {
+// IntegrationTestEnvVars returns the names of environment variables needed to configure
+// connections to integration test environments.
+func IntegrationTestEnvVars() []string {
 	// Environment variables that can be configured with paths to files
 	// with authentication information.
 	vars := []string{
diff --git a/dev-tools/mage/gotest.go b/dev-tools/mage/gotest.go
index e5b25579cdb..233773e99c6 100644
--- a/dev-tools/mage/gotest.go
+++ b/dev-tools/mage/gotest.go
@@ -137,7 +137,7 @@ func DefaultTestBinaryArgs() TestBinaryArgs {
 // Use RACE_DETECTOR=true to enable the race detector.
 // Use MODULE=module to run only tests for `module`.
 func GoTestIntegrationForModule(ctx context.Context) error {
-	passThruEnvVars := CloudEnvVars()
+	passThruEnvVars := IntegrationTestEnvVars()
 	return RunIntegTest("goIntegTest", func() error {
 		module := EnvOr("MODULE", "")
 		if module != "" {

From a7e588c40af636ff1923dd97b839497a8a901f4b Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 24 Apr 2020 19:08:18 +0200
Subject: [PATCH 14/21] Add retries to terraform operations

---
 Jenkinsfile | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3fdd4878cad..343b5daaa9c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -965,7 +965,9 @@ def startCloudTestEnv(String name, environments = []) {
       try {
         for (environment in environments) {
           if (environment.cond || runAll) {
-            terraformApply(environment.dir)
+            retry(2) {
+              terraformApply(environment.dir)
+            }
           }
         }
       } finally {
@@ -985,7 +987,9 @@ def terraformCleanup(String stashName, String directory) {
     withCloudTestEnv() {
       withBeatsEnv(false) {
         unstash "terraform-${stashName}"
-        sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}")
+        retry(2) {
+          sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}")
+        }
       }
     }
   }

From 59d9b9259db234c698e58b5230ffe452cee727e1 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 27 Apr 2020 20:04:03 +0200
Subject: [PATCH 15/21] Add docs

---
 docs/devguide/terraform.asciidoc | 77 ++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 docs/devguide/terraform.asciidoc

diff --git a/docs/devguide/terraform.asciidoc b/docs/devguide/terraform.asciidoc
new file mode 100644
index 00000000000..7079e8a30d5
--- /dev/null
+++ b/docs/devguide/terraform.asciidoc
@@ -0,0 +1,77 @@
+[[terraform-beats]]
+== Terraform in Beats
+
+Terraform is used to provision scenarios for integration testing of some cloud
+features. Features implementing integration tests that require the presence of
+cloud resources should have their own Terraform configuration, this configuration
+can be used when developing locally to create (and destroy) resources that allow
+to test these features.
+
+Tests requiring access to cloud providers should be disabled by default with the
+use of build tags.
+
+[[installing-terraform]]
+=== Installing Terraform
+
+Terraform is available in https://www.terraform.io/downloads.html
+
+Download it and place it in some directory in your PATH.
+
+`terraform` is the main command for Terraform and the only one that is usually
+needed to manage configurations. Terraform will also download other plugins that
+implement the specific functionality for each provider. These plugins are
+automatically managed and stored in the working copy, if you want to share the
+plugins between multiple working copies you can manually install them in the
+user the user plugins directory located at `~/.terraform.d/plugins`,
+or `%APPDATA%\terraform.d\plugins on Windows`.
+
+Plugins are available in https://registry.terraform.io/
+
+[[using-terraform]]
+=== Using Terraform
+
+The most important commands when using Terraform are:
+* `terraform init` to do some initial checks and install the required plugins.
+* `terraform apply` to create the resources defined in the configuration.
+* `terraform destroy` to destroy resources previously created.
+
+Cloud providers use to require credentials, they can be provided with the usual
+methods supported by these providers, using environment variables and/or
+credential files.
+
+Terraform stores the last known state of the resources managed by a
+configuration in a `terraform.tfstate` file. It is important to keep this file
+as it is used as input by `terraform destroy`. This file is created in the same
+directory where `terraform apply` is executed.
+
+Please take a look to Terraform documentation for more details: https://www.terraform.io/intro/index.html
+
+[[terraform-in-ci]]
+=== Terraform in CI
+
+Integration tests that need the presence of certain resources to work can be
+executed in CI if they provide a Terraform configuration to start these
+resources. These tests are disabled by default in CI.
+
+Here is a checklist to add support for a cloud feature in Jenkins:
+* In the feature code:
+  * Tests have a build tag so they are disabled by default. When run from mage,
+    its execution can be selected using the `TEST_TAGS` environment variable, e.g:
+    `TEST_TAGS=aws` for AWS tests.
+  * There is some Terraform configuration that defines a cloud scenario where
+    tests pass. This configuration should be in the directory of the feature.
+* In the Jenkinsfile:
+  * Add a boolean parameter to run the tests on this environment, e.g.
+    `awsCloudTests`. This parameter should be set to false by default.
+  * Add a conditional block in `withCloudTestEnv` that:
+     * Will be executed if the previoysly added boolean parameter, or `allCloudTests`
+       are set to true.
+     * Adds the tag to `TEST_TAGS` (as comma separated values), so tests are
+       selected.
+     * Defines how to obtain the credentials and provide them to the tests.
+  * In the stage of the specific beat:
+    * Add a stage that calls to `startCloudTestEnv`, if there isn't anyone.
+    * Add a post cleanup step that calls to `terraformCleanup`, if there isn't anyone.
+    * Add a environment to the list of environments started by `startCloudEnv`,
+      with the condition to start the scenario, and the path to the directory
+      with its definition, e.g. `[cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws']`

From f3fd34a8d14b242737633216a1125c902e6d9311 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 27 Apr 2020 20:07:18 +0200
Subject: [PATCH 16/21] Typo

---
 docs/devguide/terraform.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/devguide/terraform.asciidoc b/docs/devguide/terraform.asciidoc
index 7079e8a30d5..5ac4d33ac16 100644
--- a/docs/devguide/terraform.asciidoc
+++ b/docs/devguide/terraform.asciidoc
@@ -64,7 +64,7 @@ Here is a checklist to add support for a cloud feature in Jenkins:
   * Add a boolean parameter to run the tests on this environment, e.g.
     `awsCloudTests`. This parameter should be set to false by default.
   * Add a conditional block in `withCloudTestEnv` that:
-     * Will be executed if the previoysly added boolean parameter, or `allCloudTests`
+     * Will be executed if the previously added boolean parameter, or `allCloudTests`
        are set to true.
      * Adds the tag to `TEST_TAGS` (as comma separated values), so tests are
        selected.

From f9bb0247bda64ccdc3f25b7c4d3601813bb32b16 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 27 Apr 2020 20:20:09 +0200
Subject: [PATCH 17/21] Add some more info to docs

---
 docs/devguide/terraform.asciidoc | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/docs/devguide/terraform.asciidoc b/docs/devguide/terraform.asciidoc
index 5ac4d33ac16..2c21c8f4314 100644
--- a/docs/devguide/terraform.asciidoc
+++ b/docs/devguide/terraform.asciidoc
@@ -46,6 +46,27 @@ directory where `terraform apply` is executed.
 
 Please take a look to Terraform documentation for more details: https://www.terraform.io/intro/index.html
 
+[[terraform-configurations]]
+=== Terraform configuration guidelines
+
+The main purpouse of Terraform in Beats is to create and destroy cloud resources
+required by integration tests. For these configurations there are some things to
+take into account:
+* Apply should work without additional inputs or files. Only input will be the
+  required for specific providers, using environment variables or credential
+  files.
+* You must be able to apply the same configuration multiple times in the same
+  account. This will allow to have multiple builds using the same configuration
+  but with different instances of the resources. Some resources are already
+  created with unique identifiers (as EC2 instances), some others have to be
+  explicitly created with unique names (e.g. S3 buckets). For these cases random
+  suffixes can be added to identifiers.
+* Destroy must work without additional input, and should be able to destroy all
+  the resources created by the configuration. There are some resources that need
+  specific flags to be destroyed by `terraform destroy`. For example S3 buckets
+  need a flag to force to empty the bucket before deleting it, or RDS instances
+  need a flag to disable snapshots on deletion.
+
 [[terraform-in-ci]]
 === Terraform in CI
 
@@ -53,6 +74,9 @@ Integration tests that need the presence of certain resources to work can be
 executed in CI if they provide a Terraform configuration to start these
 resources. These tests are disabled by default in CI.
 
+Terraform states are archived as artifacrs of builds, this allows to manually
+destroy resources created by builds that were not able to do a proper cleanup.
+
 Here is a checklist to add support for a cloud feature in Jenkins:
 * In the feature code:
   * Tests have a build tag so they are disabled by default. When run from mage,

From 2f74a2e0ae9581e586b877bb3949eecf832b9d91 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 29 Apr 2020 10:57:54 +0200
Subject: [PATCH 18/21] Fix S3 request test

---
 .../module/aws/s3_request/s3_request_integration_test.go     | 2 --
 x-pack/metricbeat/module/aws/terraform.tf                    | 5 +++++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go b/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go
index dcb6de3e193..eeae8439f1f 100644
--- a/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go
+++ b/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go
@@ -17,8 +17,6 @@ import (
 )
 
 func TestFetch(t *testing.T) {
-	t.Skip("This test doesn't pass with current terraform scenario")
-
 	config := mtest.GetConfigForTest(t, "s3_request", "86400s")
 
 	metricSet := mbtest.NewReportingMetricSetV2Error(t, config)
diff --git a/x-pack/metricbeat/module/aws/terraform.tf b/x-pack/metricbeat/module/aws/terraform.tf
index fc92d2710a4..e767a028ab1 100644
--- a/x-pack/metricbeat/module/aws/terraform.tf
+++ b/x-pack/metricbeat/module/aws/terraform.tf
@@ -36,6 +36,11 @@ resource "aws_s3_bucket" "test" {
   force_destroy = true // Required for cleanup
 }
 
+resource "aws_s3_bucket_metric" "test" {
+  bucket = aws_s3_bucket.test.id
+  name   = "EntireBucket"
+}
+
 resource "aws_s3_bucket_object" "test" {
   key     = "someobject"
   bucket  = aws_s3_bucket.test.id

From 85a87e6c7782df414d18d7562429c6f2b2ca80cb Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 30 Apr 2020 10:49:31 +0200
Subject: [PATCH 19/21] Keep support for integration tests of modules without
 environment

Modules that don't define a local environment with docker compose or
kubernetes (kind) are not being executed. Fall back to the old default
of docker in that case.
---
 dev-tools/mage/integtest.go | 69 ++++++++++++++++++++++---------------
 1 file changed, 41 insertions(+), 28 deletions(-)

diff --git a/dev-tools/mage/integtest.go b/dev-tools/mage/integtest.go
index f573d3df80a..d5be206aac6 100644
--- a/dev-tools/mage/integtest.go
+++ b/dev-tools/mage/integtest.go
@@ -185,37 +185,50 @@ func NewIntegrationRunners(path string, passInEnv map[string]string) (Integratio
 		if err != nil {
 			return nil, errors.Wrapf(err, "%s tester failed on Use", t.Name())
 		}
-		if use {
-			// Create the steps for the specific runner.
-			var runnerSteps IntegrationTestSteps
-			requirements := t.StepRequirements()
-			if requirements != nil {
-				runnerSteps = append(runnerSteps, requirements...)
-			}
-			runnerSteps = append(runnerSteps, steps...)
+		if !use {
+			continue
+		}
+		// Create the steps for the specific runner.
+		var runnerSteps IntegrationTestSteps
+		requirements := t.StepRequirements()
+		if requirements != nil {
+			runnerSteps = append(runnerSteps, requirements...)
+		}
+		runnerSteps = append(runnerSteps, steps...)
 
-			// Create the custom env for the runner.
-			env := map[string]string{}
-			for k, v := range passInEnv {
-				env[k] = v
-			}
-			env[insideIntegrationTestEnvVar] = "true"
-			passThroughEnvs(env, defaultPassthroughEnvVars...)
-			if mg.Verbose() {
-				env["MAGEFILE_VERBOSE"] = "1"
-			}
-			if UseVendor {
-				env["GOFLAGS"] = "-mod=vendor"
-			}
+		// Create the custom env for the runner.
+		env := map[string]string{}
+		for k, v := range passInEnv {
+			env[k] = v
+		}
+		env[insideIntegrationTestEnvVar] = "true"
+		passThroughEnvs(env, defaultPassthroughEnvVars...)
+		if mg.Verbose() {
+			env["MAGEFILE_VERBOSE"] = "1"
+		}
+		if UseVendor {
+			env["GOFLAGS"] = "-mod=vendor"
+		}
 
-			runner := &IntegrationRunner{
-				steps:  runnerSteps,
-				tester: t,
-				dir:    dir,
-				env:    env,
-			}
-			runners = append(runners, runner)
+		runner := &IntegrationRunner{
+			steps:  runnerSteps,
+			tester: t,
+			dir:    dir,
+			env:    env,
+		}
+		runners = append(runners, runner)
+	}
+	// Keep support for modules that don't have a local environment defined at the module
+	// level (system, stack and cloud modules by now)
+	if len(runners) == 0 {
+		if mg.Verbose() {
+			fmt.Printf(">> No runner found in %s, using docker\n", path)
+		}
+		runner, err := NewDockerIntegrationRunner("MODULE")
+		if err != nil {
+			return nil, errors.Wrapf(err, "initializing docker runner")
 		}
+		runners = append(runners, runner)
 	}
 	return runners, nil
 }

From eb04578389d425eee02c98af836a6823213f1647 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 30 Apr 2020 13:03:12 +0200
Subject: [PATCH 20/21] Fix fields validation in non utf-8 environments

---
 libbeat/tests/system/beat/beat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py
index fa8e0ad3cb4..9d0bf471635 100644
--- a/libbeat/tests/system/beat/beat.py
+++ b/libbeat/tests/system/beat/beat.py
@@ -586,7 +586,7 @@ def extract_fields(doc_list, name):
 
         # TODO: Make fields_doc path more generic to work with beat-generator. If it can't find file
         # "fields.yml" you should run "make update" on metricbeat folder
-        with open(fields_doc, "r") as f:
+        with open(fields_doc, "r", encoding="utf_8") as f:
             path = os.path.abspath(os.path.dirname(__file__) + "../../../../fields.yml")
             if not os.path.isfile(path):
                 path = os.path.abspath(os.path.dirname(__file__) + "../../../../_meta/fields.common.yml")

From f92d499017eb04cf2bb5ec749f480c4db9843002 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 30 Apr 2020 16:41:08 +0200
Subject: [PATCH 21/21] Pass through all default variables and current pass in
 environment

---
 dev-tools/mage/integtest.go | 47 ++++++++++++++-----------------------
 1 file changed, 17 insertions(+), 30 deletions(-)

diff --git a/dev-tools/mage/integtest.go b/dev-tools/mage/integtest.go
index d5be206aac6..064dc7d8102 100644
--- a/dev-tools/mage/integtest.go
+++ b/dev-tools/mage/integtest.go
@@ -188,33 +188,9 @@ func NewIntegrationRunners(path string, passInEnv map[string]string) (Integratio
 		if !use {
 			continue
 		}
-		// Create the steps for the specific runner.
-		var runnerSteps IntegrationTestSteps
-		requirements := t.StepRequirements()
-		if requirements != nil {
-			runnerSteps = append(runnerSteps, requirements...)
-		}
-		runnerSteps = append(runnerSteps, steps...)
-
-		// Create the custom env for the runner.
-		env := map[string]string{}
-		for k, v := range passInEnv {
-			env[k] = v
-		}
-		env[insideIntegrationTestEnvVar] = "true"
-		passThroughEnvs(env, defaultPassthroughEnvVars...)
-		if mg.Verbose() {
-			env["MAGEFILE_VERBOSE"] = "1"
-		}
-		if UseVendor {
-			env["GOFLAGS"] = "-mod=vendor"
-		}
-
-		runner := &IntegrationRunner{
-			steps:  runnerSteps,
-			tester: t,
-			dir:    dir,
-			env:    env,
+		runner, err := initRunner(t, dir, passInEnv)
+		if err != nil {
+			return nil, errors.Wrapf(err, "initializing %s runner", t.Name())
 		}
 		runners = append(runners, runner)
 	}
@@ -224,7 +200,11 @@ func NewIntegrationRunners(path string, passInEnv map[string]string) (Integratio
 		if mg.Verbose() {
 			fmt.Printf(">> No runner found in %s, using docker\n", path)
 		}
-		runner, err := NewDockerIntegrationRunner("MODULE")
+		tester, ok := globalIntegrationTesters["docker"]
+		if !ok {
+			return nil, fmt.Errorf("docker integration test runner not registered")
+		}
+		runner, err := initRunner(tester, dir, passInEnv)
 		if err != nil {
 			return nil, errors.Wrapf(err, "initializing docker runner")
 		}
@@ -243,6 +223,10 @@ func NewDockerIntegrationRunner(passThroughEnvVars ...string) (*IntegrationRunne
 	if !ok {
 		return nil, fmt.Errorf("docker integration test runner not registered")
 	}
+	return initRunner(tester, cwd, nil, passThroughEnvVars...)
+}
+
+func initRunner(tester IntegrationTester, dir string, passInEnv map[string]string, passThroughEnvVars ...string) (*IntegrationRunner, error) {
 	var runnerSteps IntegrationTestSteps
 	requirements := tester.StepRequirements()
 	if requirements != nil {
@@ -253,8 +237,11 @@ func NewDockerIntegrationRunner(passThroughEnvVars ...string) (*IntegrationRunne
 	env := map[string]string{
 		insideIntegrationTestEnvVar: "true",
 	}
-	passThroughEnvs(env, defaultPassthroughEnvVars...)
+	for name, value := range passInEnv {
+		env[name] = value
+	}
 	passThroughEnvs(env, passThroughEnvVars...)
+	passThroughEnvs(env, defaultPassthroughEnvVars...)
 	if mg.Verbose() {
 		env["MAGEFILE_VERBOSE"] = "1"
 	}
@@ -265,7 +252,7 @@ func NewDockerIntegrationRunner(passThroughEnvVars ...string) (*IntegrationRunne
 	runner := &IntegrationRunner{
 		steps:  runnerSteps,
 		tester: tester,
-		dir:    cwd,
+		dir:    dir,
 		env:    env,
 	}
 	return runner, nil