[Filebeat] Integration tests in CI for AWS-S3 input (#27491)

* [Filebeat] Integration tests in CI for AWS-S3 input

(cherry picked from commit 9d4597a)

Jenkinsfile

@@ -22,7 +22,7 @@ pipeline {
     RUNBLD_DISABLE_NOTIFICATIONS = 'true'
     SLACK_CHANNEL = "#beats-build"
     SNAPSHOT = 'true'
-    TERRAFORM_VERSION = "0.12.30"
+    TERRAFORM_VERSION = "0.13.7"
     XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
   }
   options {
@@ -832,8 +832,10 @@ def withCloudTestEnv(Closure body) {
   def maskedVars = []
   def testTags = "${env.TEST_TAGS}"
 
-  // AWS
-  if (params.allCloudTests || params.awsCloudTests) {
+  // Allow AWS credentials when the build was configured to do so with:
+  //   - the cloudtests build parameters
+  //   - the aws github label
+  if (params.allCloudTests || params.awsCloudTests || matchesPrLabel(label: 'aws')) {
     testTags = "${testTags},aws"
     def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data
     if (!aws.containsKey('access_key')) {
@@ -885,6 +887,7 @@ def startCloudTestEnv(Map args = [:]) {
             // If it failed then cleanup without failing the build
             sh(label: 'Terraform Cleanup', script: ".ci/scripts/terraform-cleanup.sh ${folder}", returnStatus: true)
           }
+          error('startCloudTestEnv: terraform apply failed.')
         } finally {
           // Archive terraform states in case manual cleanup is needed.
           archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate')

x-pack/filebeat/Jenkinsfile.yml

@@ -93,6 +93,19 @@ stages:
         platforms:             ## override default labels in this specific stage.
             - "windows-7"
         stage: extended
+    cloud:
+      cloud: "mage build test"
+      withModule: true       ## run the ITs only if the changeset affects a specific module.
+      dirs:                  ## apply terraform for the given directory.
+        - "x-pack/filebeat/input/awss3/_meta/terraform"
+      when:                  ## Override the top-level when.
+        parameters:
+          - "awsCloudTests"
+        comments:
+          - "/test x-pack/filebeat for aws cloud"
+        labels:
+          - "aws"
+      stage: extended
     #windows-7-32:  See https://github.com/elastic/beats/issues/22315
     #    mage: "mage build unitTest"
     #    platforms:             ## override default labels in this specific stage.

x-pack/metricbeat/module/aws/terraform.tf

@@ -50,7 +50,7 @@ resource "aws_s3_bucket_object" "test" {
 resource "aws_instance" "test" {
   ami           = data.aws_ami.latest-amzn.id
   monitoring    = true
-  instance_type = "t1.micro"
+  instance_type = "t2.micro"
   tags = {
     Name = "metricbeat-test"
   }