diff --git a/.ci/scripts/install-terraform.sh b/.ci/scripts/install-terraform.sh new file mode 100755 index 00000000000..39aa684d0aa --- /dev/null +++ b/.ci/scripts/install-terraform.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -exuo pipefail + +MSG="parameter missing." +TERRAFORM_VERSION=${TERRAFORM_VERSION:?$MSG} +HOME=${HOME:?$MSG} +TERRAFORM_CMD="${HOME}/bin/terraform" + +OS=$(uname -s | tr '[:upper:]' '[:lower:]') + +mkdir -p "${HOME}/bin" + +curl -sSLo - "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${OS}_amd64.zip" > ${TERRAFORM_CMD}.zip +unzip -o ${TERRAFORM_CMD}.zip -d $(dirname ${TERRAFORM_CMD}) +rm ${TERRAFORM_CMD}.zip + +chmod +x "${TERRAFORM_CMD}" diff --git a/.ci/scripts/terraform-cleanup.sh b/.ci/scripts/terraform-cleanup.sh new file mode 100755 index 00000000000..f1051b9b20d --- /dev/null +++ b/.ci/scripts/terraform-cleanup.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -exuo pipefail + +DIRECTORY=${1:-.} + +FAILED=0 +for tfstate in $(find $DIRECTORY -name terraform.tfstate); do + cd $(dirname $tfstate) + if ! terraform destroy -auto-approve; then + FAILED=1 + fi + cd - +done + +exit $FAILED diff --git a/.gitignore b/.gitignore index 34266183ac2..fe3a1459213 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,7 @@ x-pack/dockerlogbeat/temproot.tar *.test *.prof *.pyc + +# Terraform +*.terraform +*.tfstate* diff --git a/Jenkinsfile b/Jenkinsfile index a0382877db6..45eee99bb45 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -15,9 +15,11 @@ pipeline { BASE_DIR = 'src/github.com/elastic/beats' GOX_FLAGS = "-arch amd64" DOCKER_COMPOSE_VERSION = "1.21.0" + TERRAFORM_VERSION = "0.12.24" PIPELINE_LOG_LEVEL = "INFO" DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' DOCKER_REGISTRY = 'docker.elastic.co' + AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' RUNBLD_DISABLE_NOTIFICATIONS = 'true' } options { @@ -36,6 +38,11 @@ pipeline { booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') booleanParam(name: 'macosTest', defaultValue: true, description: 'Allow macOS stages.') + + booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.') + booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.') + string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') + booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps') booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow') } @@ -352,8 +359,30 @@ pipeline { return env.BUILD_METRICBEAT_XPACK != "false" } } - steps { - mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test") + stages { + stage('Prepare cloud integration tests environments'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + steps { + startCloudTestEnv('x-pack-metricbeat', [ + [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'], + ]) + } + } + stage('Metricbeat x-pack'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + steps { + withCloudTestEnv() { + mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test") + } + } + } + } + post { + cleanup { + terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat') + } } } stage('Metricbeat crosscompile'){ @@ -683,7 +712,7 @@ def withBeatsEnv(boolean archive, Closure body) { "TEST_COVERAGE=true", "RACE_DETECTOR=true", "PYTHON_ENV=${WORKSPACE}/python-env", - "TEST_TAGS=oracle", + "TEST_TAGS=${env.TEST_TAGS},oracle", "DOCKER_PULL=0", ]) { deleteDir() @@ -750,6 +779,7 @@ def installTools() { if(isUnix()) { retry(i) { sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") } retry(i) { sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") } + retry(i) { sh(label: "Install Terraform ${TERRAFORM_VERSION}", script: ".ci/scripts/install-terraform.sh") } retry(i) { sh(label: "Install Mage", script: "make mage") } } else { retry(i) { bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } @@ -821,6 +851,7 @@ def dumpFilteredEnvironment(){ echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}" echo "STRESS_TESTS: ${env.STRESS_TESTS}" echo "STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS}" + echo "TEST_TAGS: ${env.TEST_TAGS}" echo "GOX_OS: ${env.GOX_OS}" echo "GOX_OSARCH: ${env.GOX_OSARCH}" echo "GOX_FLAGS: ${env.GOX_FLAGS}" @@ -907,6 +938,98 @@ def isChangedXPackCode(patterns) { return isChanged(allPatterns) } +// withCloudTestEnv executes a closure with credentials for cloud test +// environments. +def withCloudTestEnv(Closure body) { + def maskedVars = [] + def testTags = "${env.TEST_TAGS}" + + // AWS + if (params.allCloudTests || params.awsCloudTests) { + testTags = "${testTags},aws" + def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data + if (!aws.containsKey('access_key')) { + error("${AWS_ACCOUNT_SECRET} doesn't contain 'access_key'") + } + if (!aws.containsKey('secret_key')) { + error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'") + } + maskedVars.addAll([ + [var: "AWS_REGION", password: params.awsRegion], + [var: "AWS_ACCESS_KEY_ID", password: aws.access_key], + [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key], + ]) + } + + withEnv([ + "TEST_TAGS=${testTags}", + ]) { + withEnvMask(vars: maskedVars) { + body() + } + } +} + +def terraformInit(String directory) { + dir(directory) { + sh(label: "Terraform Init on ${directory}", script: "terraform init") + } +} + +def terraformApply(String directory) { + terraformInit(directory) + dir(directory) { + sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve") + } +} + +// Start testing environment on cloud using terraform. Terraform files are +// stashed so they can be used by other stages. They are also archived in +// case manual cleanup is needed. +// +// Example: +// startCloudTestEnv('x-pack-metricbeat', [ +// [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'], +// ]) +// ... +// terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat') +def startCloudTestEnv(String name, environments = []) { + withCloudTestEnv() { + withBeatsEnv(false) { + def runAll = params.runAllCloudTests + try { + for (environment in environments) { + if (environment.cond || runAll) { + retry(2) { + terraformApply(environment.dir) + } + } + } + } finally { + // Archive terraform states in case manual cleanup is needed. + archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') + } + stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**') + } + } +} + + +// Looks for all terraform states in directory and runs terraform destroy for them, +// it uses terraform states previously stashed by startCloudTestEnv. +def terraformCleanup(String stashName, String directory) { + stage("Remove cloud scenarios in ${directory}"){ + withCloudTestEnv() { + withBeatsEnv(false) { + unstash "terraform-${stashName}" + retry(2) { + sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}") + } + } + } + } +} + def loadConfigEnvVars(){ def empty = [] env.GO_VERSION = readFile(".go-version").trim() diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index 21c07ba4398..9fdb189e3fa 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -833,3 +833,28 @@ func ListMatchingEnvVars(prefixes ...string) []string { } return vars } + +// IntegrationTestEnvVars returns the names of environment variables needed to configure +// connections to integration test environments. +func IntegrationTestEnvVars() []string { + // Environment variables that can be configured with paths to files + // with authentication information. + vars := []string{ + "AWS_SHARED_CREDENTIAL_FILE", + "AZURE_AUTH_LOCATION", + "GOOGLE_APPLICATION_CREDENTIALS", + } + // Environment variables with authentication information. + prefixes := []string{ + "AWS_", + "AZURE_", + + // Accepted by terraform, but not by many clients, including Beats + "GOOGLE_", + "GCLOUD_", + } + for _, prefix := range prefixes { + vars = append(vars, ListMatchingEnvVars(prefix)...) + } + return vars +} diff --git a/dev-tools/mage/gotest.go b/dev-tools/mage/gotest.go index 2eb7f9a0b7d..c6e3b6ce430 100644 --- a/dev-tools/mage/gotest.go +++ b/dev-tools/mage/gotest.go @@ -156,7 +156,9 @@ func GoTestIntegrationForModule(ctx context.Context) error { foundModule = true // Set MODULE because only want that modules tests to run inside the testing environment. - runners, err := NewIntegrationRunners(path.Join("./module", fi.Name()), map[string]string{"MODULE": fi.Name()}) + env := map[string]string{"MODULE": fi.Name()} + passThroughEnvs(env, IntegrationTestEnvVars()...) + runners, err := NewIntegrationRunners(path.Join("./module", fi.Name()), env) if err != nil { return errors.Wrapf(err, "test setup failed for module %s", fi.Name()) } diff --git a/docs/devguide/terraform.asciidoc b/docs/devguide/terraform.asciidoc new file mode 100644 index 00000000000..2c21c8f4314 --- /dev/null +++ b/docs/devguide/terraform.asciidoc @@ -0,0 +1,101 @@ +[[terraform-beats]] +== Terraform in Beats + +Terraform is used to provision scenarios for integration testing of some cloud +features. Features implementing integration tests that require the presence of +cloud resources should have their own Terraform configuration, this configuration +can be used when developing locally to create (and destroy) resources that allow +to test these features. + +Tests requiring access to cloud providers should be disabled by default with the +use of build tags. + +[[installing-terraform]] +=== Installing Terraform + +Terraform is available in https://www.terraform.io/downloads.html + +Download it and place it in some directory in your PATH. + +`terraform` is the main command for Terraform and the only one that is usually +needed to manage configurations. Terraform will also download other plugins that +implement the specific functionality for each provider. These plugins are +automatically managed and stored in the working copy, if you want to share the +plugins between multiple working copies you can manually install them in the +user the user plugins directory located at `~/.terraform.d/plugins`, +or `%APPDATA%\terraform.d\plugins on Windows`. + +Plugins are available in https://registry.terraform.io/ + +[[using-terraform]] +=== Using Terraform + +The most important commands when using Terraform are: +* `terraform init` to do some initial checks and install the required plugins. +* `terraform apply` to create the resources defined in the configuration. +* `terraform destroy` to destroy resources previously created. + +Cloud providers use to require credentials, they can be provided with the usual +methods supported by these providers, using environment variables and/or +credential files. + +Terraform stores the last known state of the resources managed by a +configuration in a `terraform.tfstate` file. It is important to keep this file +as it is used as input by `terraform destroy`. This file is created in the same +directory where `terraform apply` is executed. + +Please take a look to Terraform documentation for more details: https://www.terraform.io/intro/index.html + +[[terraform-configurations]] +=== Terraform configuration guidelines + +The main purpouse of Terraform in Beats is to create and destroy cloud resources +required by integration tests. For these configurations there are some things to +take into account: +* Apply should work without additional inputs or files. Only input will be the + required for specific providers, using environment variables or credential + files. +* You must be able to apply the same configuration multiple times in the same + account. This will allow to have multiple builds using the same configuration + but with different instances of the resources. Some resources are already + created with unique identifiers (as EC2 instances), some others have to be + explicitly created with unique names (e.g. S3 buckets). For these cases random + suffixes can be added to identifiers. +* Destroy must work without additional input, and should be able to destroy all + the resources created by the configuration. There are some resources that need + specific flags to be destroyed by `terraform destroy`. For example S3 buckets + need a flag to force to empty the bucket before deleting it, or RDS instances + need a flag to disable snapshots on deletion. + +[[terraform-in-ci]] +=== Terraform in CI + +Integration tests that need the presence of certain resources to work can be +executed in CI if they provide a Terraform configuration to start these +resources. These tests are disabled by default in CI. + +Terraform states are archived as artifacrs of builds, this allows to manually +destroy resources created by builds that were not able to do a proper cleanup. + +Here is a checklist to add support for a cloud feature in Jenkins: +* In the feature code: + * Tests have a build tag so they are disabled by default. When run from mage, + its execution can be selected using the `TEST_TAGS` environment variable, e.g: + `TEST_TAGS=aws` for AWS tests. + * There is some Terraform configuration that defines a cloud scenario where + tests pass. This configuration should be in the directory of the feature. +* In the Jenkinsfile: + * Add a boolean parameter to run the tests on this environment, e.g. + `awsCloudTests`. This parameter should be set to false by default. + * Add a conditional block in `withCloudTestEnv` that: + * Will be executed if the previously added boolean parameter, or `allCloudTests` + are set to true. + * Adds the tag to `TEST_TAGS` (as comma separated values), so tests are + selected. + * Defines how to obtain the credentials and provide them to the tests. + * In the stage of the specific beat: + * Add a stage that calls to `startCloudTestEnv`, if there isn't anyone. + * Add a post cleanup step that calls to `terraformCleanup`, if there isn't anyone. + * Add a environment to the list of environments started by `startCloudEnv`, + with the condition to start the scenario, and the path to the directory + with its definition, e.g. `[cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws']` diff --git a/x-pack/metricbeat/module/aws/terraform.tf b/x-pack/metricbeat/module/aws/terraform.tf new file mode 100644 index 00000000000..e767a028ab1 --- /dev/null +++ b/x-pack/metricbeat/module/aws/terraform.tf @@ -0,0 +1,48 @@ +provider "aws" { + version = "~> 2.58" +} + +provider "random" { + version = "~> 2.2" +} + +resource "random_id" "suffix" { + byte_length = 4 +} + +resource "random_password" "db" { + length = 16 + special = false +} + +resource "aws_db_instance" "test" { + identifier = "metricbeat-test-${random_id.suffix.hex}" + allocated_storage = 20 // Gigabytes + engine = "mysql" + instance_class = "db.t2.micro" + name = "metricbeattest" + username = "foo" + password = random_password.db.result + skip_final_snapshot = true // Required for cleanup +} + +resource "aws_sqs_queue" "test" { + name = "metricbeat-test-${random_id.suffix.hex}" + receive_wait_time_seconds = 10 +} + +resource "aws_s3_bucket" "test" { + bucket = "metricbeat-test-${random_id.suffix.hex}" + force_destroy = true // Required for cleanup +} + +resource "aws_s3_bucket_metric" "test" { + bucket = aws_s3_bucket.test.id + name = "EntireBucket" +} + +resource "aws_s3_bucket_object" "test" { + key = "someobject" + bucket = aws_s3_bucket.test.id + content = "something" +}