Add build tags for cloud tests (elastic#16937)

Add build tags to AWS and Azure integration tests, and
make tests fail if they cannot get a working configuration.

Add support for a `TEST_TAGS` environment variable in
mage targets that add additional build tags to go tests calls.
When `TEST_TAGS` includes a cloud provider, it is expected
that valid credentials to run these tests are available in the
environment, if not, tests will fail. This is intentional, to avoid
skipping tests by mistake due to some misconfiguration in CI.

dev-tools/mage/gotest.go

@@ -67,6 +67,7 @@ func makeGoTestArgs(name string) GoTestArgs {
 		Packages:        []string{"./..."},
 		OutputFile:      fileName + ".out",
 		JUnitReportFile: fileName + ".xml",
+		Tags:            testTagsFromEnv(),
 	}
 	if TestCoverage {
 		params.CoverageProfileFile = fileName + ".cov"
@@ -83,13 +84,20 @@ func makeGoTestArgsForModule(name, module string) GoTestArgs {
 		Packages:        []string{fmt.Sprintf("./module/%s/...", module)},
 		OutputFile:      fileName + ".out",
 		JUnitReportFile: fileName + ".xml",
+		Tags:            testTagsFromEnv(),
 	}
 	if TestCoverage {
 		params.CoverageProfileFile = fileName + ".cov"
 	}
 	return params
 }
 
+// testTagsFromEnv gets a list of comma-separated tags from the TEST_TAGS
+// environment variables, e.g: TEST_TAGS=aws,azure.
+func testTagsFromEnv() []string {
+	return strings.Split(strings.Trim(os.Getenv("TEST_TAGS"), ", "), ",")
+}
+
 // DefaultGoTestUnitArgs returns a default set of arguments for running
 // all unit tests. We tag unit test files with '!integration'.
 func DefaultGoTestUnitArgs() GoTestArgs { return makeGoTestArgs("Unit") }

dev-tools/mage/integtest.go

@@ -147,6 +147,7 @@ func RunIntegTest(mageTarget string, test func() error, passThroughEnvVars ...st
 	env := []string{
 		"TEST_COVERAGE",
 		"RACE_DETECTOR",
+		"TEST_TAGS",
 		"PYTHON_EXE",
 	}
 	env = append(env, passThroughEnvVars...)

metricbeat/magefile.go

@@ -172,6 +172,7 @@ func CollectDocs() error {
 // GoIntegTest executes the Go integration tests.
 // Use TEST_COVERAGE=true to enable code coverage profiling.
 // Use RACE_DETECTOR=true to enable the race detector.
+// Use TEST_TAGS=tag1,tag2 to add additional build tags.
 func GoIntegTest(ctx context.Context) error {
 	return devtools.GoTestIntegrationForModule(ctx)
 }

x-pack/filebeat/input/azureeventhub/azureeventhub_integration_test.go

@@ -3,6 +3,7 @@
 // you may not use this file except in compliance with the Elastic License.
 
 // +build integration
+// +build azure
 
 package azureeventhub
 
@@ -23,26 +24,20 @@ import (
 )
 
 var (
-
-	// setup the input config
 	azureConfig = common.MustNewConfigFrom(common.MapStr{
-		"storage_account_key":       os.Getenv("STORAGE_ACCOUNT_NAME"),
-		"storage_account":           os.Getenv("STORAGE_ACCOUNT_KEY"),
+		"storage_account_key":       lookupEnv("STORAGE_ACCOUNT_NAME"),
+		"storage_account":           lookupEnv("STORAGE_ACCOUNT_KEY"),
 		"storage_account_container": ephContainerName,
-		"connection_string":         os.Getenv("EVENTHUB_CONNECTION_STRING"),
-		"consumer_group":            os.Getenv("EVENTHUB_CONSUMERGROUP"),
-		"eventhub":                  os.Getenv("EVENTHUB_NAME"),
+		"connection_string":         lookupEnv("EVENTHUB_CONNECTION_STRING"),
+		"consumer_group":            lookupEnv("EVENTHUB_CONSUMERGROUP"),
+		"eventhub":                  lookupEnv("EVENTHUB_NAME"),
 	})
 
 	message = "{\"records\":[{\"some_field\":\"this is some message\",\"time\":\"2019-12-17T13:43:44.4946995Z\"}"
 )
 
 func TestInput(t *testing.T) {
-	if os.Getenv("EVENTHUB_NAME") == "" || os.Getenv("EVENTHUB_CONNECTION_STRING") == "" {
-		t.Skip("EVENTHUB_NAME or/and EVENTHUB_CONSUMERGROUP  are not set in environment.")
-	}
-	err := addEventToHub(os.Getenv("EVENTHUB_CONNECTION_STRING"))
-
+	err := addEventToHub(lookupEnv("EVENTHUB_CONNECTION_STRING"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -99,6 +94,14 @@ func TestInput(t *testing.T) {
 	}
 }
 
+func lookupEnv(t *testing.T, varName string) string {
+	value, ok := os.LookupEnv(varName)
+	if !ok {
+		t.Fatalf("Environment variable %s is not set", varName)
+	}
+	return value
+}
+
 func addEventToHub(connStr string) error {
 	hub, err := eventhub.NewHubFromConnectionString(connStr)
 	if err != nil {

x-pack/filebeat/input/s3/s3_integration_test.go

@@ -2,6 +2,9 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
+// +build integration
+// +build aws
+
 package s3
 
 import (
@@ -13,14 +16,14 @@ import (
 	"testing"
 	"time"
 
-	"github.com/aws/aws-sdk-go-v2/service/s3/s3iface"
-	"github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface"
+	"github.com/stretchr/testify/assert"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	awssdk "github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/s3iface"
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/stretchr/testify/assert"
+	"github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface"
 
 	"github.com/elastic/beats/v7/filebeat/channel"
 	"github.com/elastic/beats/v7/filebeat/input"
@@ -38,7 +41,9 @@ const (
 var filePath = filepath.Join("ftest", fileName)
 
 // GetConfigForTest function gets aws credentials for integration tests.
-func getConfigForTest() (config, string) {
+func getConfigForTest(t *testing.T) config {
+	t.Helper()
+
 	awsConfig := awscommon.ConfigAWS{}
 	queueURL := os.Getenv("QUEUE_URL")
 	profileName := os.Getenv("AWS_PROFILE_NAME")
@@ -51,25 +56,25 @@ func getConfigForTest() (config, string) {
 	}
 	switch {
 	case queueURL == "":
-		return config, "Skipping: $QUEUE_URL is not set in environment"
+		t.Fatal("$QUEUE_URL is not set in environment")
 	case profileName == "" && accessKeyID == "":
-		return config, "Skipping: $AWS_ACCESS_KEY_ID or $AWS_PROFILE_NAME not set or set to empty"
+		t.Fatal("$AWS_ACCESS_KEY_ID or $AWS_PROFILE_NAME not set or set to empty")
 	case profileName != "":
 		awsConfig.ProfileName = profileName
 		config.QueueURL = queueURL
 		config.AwsConfig = awsConfig
-		return config, ""
+		return config
 	case secretAccessKey == "":
-		return config, "Skipping: $AWS_SECRET_ACCESS_KEY not set or set to empty"
-	default:
-		awsConfig.AccessKeyID = accessKeyID
-		awsConfig.SecretAccessKey = secretAccessKey
-		if sessionToken != "" {
-			awsConfig.SessionToken = sessionToken
-		}
-		config.AwsConfig = awsConfig
-		return config, ""
+		t.Fatal("$AWS_SECRET_ACCESS_KEY not set or set to empty")
 	}
+
+	awsConfig.AccessKeyID = accessKeyID
+	awsConfig.SecretAccessKey = secretAccessKey
+	if sessionToken != "" {
+		awsConfig.SessionToken = sessionToken
+	}
+	config.AwsConfig = awsConfig
+	return config
 }
 
 func uploadSampleLogFile(t *testing.T, bucketName string, svcS3 s3iface.ClientAPI) {
@@ -160,7 +165,7 @@ func runTest(t *testing.T, cfg *common.Config, run func(t *testing.T, input *s3I
 
 	in, err := NewInput(cfg, connector, inputCtx)
 	if err != nil {
-		t.Skipf("Skipping: %v", err)
+		t.Fatal(err)
 	}
 	s3Input := in.(*s3Input)
 	defer s3Input.Stop()
@@ -224,13 +229,9 @@ func (o *stubOutleter) OnEvent(event beat.Event) bool {
 
 func TestS3Input(t *testing.T) {
 	inputConfig := defaultTestConfig()
+	config := getConfigForTest(t)
 
 	runTest(t, inputConfig, func(t *testing.T, input *s3Input, out *stubOutleter) {
-		config, info := getConfigForTest()
-		if info != "" {
-			t.Skipf("failed to get config for test: %v", info)
-		}
-
 		awsConfig, err := awscommon.GetAWSCredentials(config.AwsConfig)
 		if err != nil {
 

x-pack/metricbeat/magefile.go

@@ -138,6 +138,7 @@ func UnitTest() {
 // GoUnitTest executes the Go unit tests.
 // Use TEST_COVERAGE=true to enable code coverage profiling.
 // Use RACE_DETECTOR=true to enable the race detector.
+// Use TEST_TAGS=tag1,tag2 to add additional build tags.
 func GoUnitTest(ctx context.Context) error {
 	return devtools.GoTest(ctx, devtools.DefaultGoTestUnitArgs())
 }
@@ -158,6 +159,7 @@ func IntegTest() {
 // GoIntegTest executes the Go integration tests.
 // Use TEST_COVERAGE=true to enable code coverage profiling.
 // Use RACE_DETECTOR=true to enable the race detector.
+// Use TEST_TAGS=tag1,tag2 to add additional build tags.
 func GoIntegTest(ctx context.Context) error {
 	return devtools.GoTestIntegrationForModule(ctx)
 }

x-pack/metricbeat/module/aws/billing/billing_integration_test.go

@@ -3,6 +3,7 @@
 // you may not use this file except in compliance with the Elastic License.
 
 // +build integration
+// +build aws
 
 package billing
 
@@ -14,10 +15,7 @@ import (
 )
 
 func TestData(t *testing.T) {
-	config, info := mtest.GetConfigForTest("billing", "300s")
-	if info != "" {
-		t.Skip("Skipping TestData: " + info)
-	}
+	config := mtest.GetConfigForTest(t, "billing", "300s")
 
 	metricSet := mbtest.NewFetcher(t, config)
 	metricSet.WriteEvents(t, "/")

x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go

@@ -3,6 +3,7 @@
 // you may not use this file except in compliance with the Elastic License.
 
 // +build integration
+// +build aws
 
 package cloudwatch
 
@@ -16,10 +17,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
-	config, info := mtest.GetConfigForTest("cloudwatch", "300s")
-	if info != "" {
-		t.Skip("Skipping TestFetch: " + info)
-	}
+	config := mtest.GetConfigForTest(t, "cloudwatch", "300s")
 
 	config = addCloudwatchMetricsToConfig(config)
 	metricSet := mbtest.NewReportingMetricSetV2Error(t, config)
@@ -32,10 +30,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
-	config, info := mtest.GetConfigForTest("cloudwatch", "300s")
-	if info != "" {
-		t.Skip("Skipping TestData: " + info)
-	}
+	config := mtest.GetConfigForTest(t, "cloudwatch", "300s")
 
 	config = addCloudwatchMetricsToConfig(config)
 	metricSet := mbtest.NewFetcher(t, config)

x-pack/metricbeat/module/aws/dynamodb/dynamodb_integration_test.go

@@ -3,6 +3,7 @@
 // you may not use this file except in compliance with the Elastic License.
 
 // +build integration
+// +build aws
 
 package dynamodb
 
@@ -14,10 +15,7 @@ import (
 )
 
 func TestData(t *testing.T) {
-	config, info := mtest.GetConfigForTest("dynamodb", "300s")
-	if info != "" {
-		t.Skip("Skipping TestData: " + info)
-	}
+	config := mtest.GetConfigForTest(t, "dynamodb", "300s")
 
 	metricSet := mbtest.NewFetcher(t, config)
 	metricSet.WriteEvents(t, "/")

x-pack/metricbeat/module/aws/ebs/ebs_integration_test.go

@@ -3,6 +3,7 @@
 // you may not use this file except in compliance with the Elastic License.
 
 // +build integration
+// +build aws
 
 package ebs
 
@@ -14,10 +15,7 @@ import (
 )
 
 func TestData(t *testing.T) {
-	config, info := mtest.GetConfigForTest("ebs", "300s")
-	if info != "" {
-		t.Skip("Skipping TestData: " + info)
-	}
+	config := mtest.GetConfigForTest(t, "ebs", "300s")
 
 	metricSet := mbtest.NewFetcher(t, config)
 	metricSet.WriteEvents(t, "/")

x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go

@@ -3,6 +3,7 @@
 // you may not use this file except in compliance with the Elastic License.
 
 // +build integration
+// +build aws
 
 package ec2
 
@@ -16,10 +17,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
-	config, info := mtest.GetConfigForTest("ec2", "300s")
-	if info != "" {
-		t.Skip("Skipping TestFetch: " + info)
-	}
+	config := mtest.GetConfigForTest(t, "ec2", "300s")
 
 	metricSet := mbtest.NewReportingMetricSetV2Error(t, config)
 	events, errs := mbtest.ReportingFetchV2Error(metricSet)
@@ -66,10 +64,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
-	config, info := mtest.GetConfigForTest("ec2", "300s")
-	if info != "" {
-		t.Skip("Skipping TestData: " + info)
-	}
+	config := mtest.GetConfigForTest(t, "ec2", "300s")
 
 	metricSet := mbtest.NewReportingMetricSetV2Error(t, config)
 	if err := mbtest.WriteEventsReporterV2Error(metricSet, t, "/"); err != nil {

x-pack/metricbeat/module/aws/elb/elb_integration_test.go

@@ -3,6 +3,7 @@
 // you may not use this file except in compliance with the Elastic License.
 
 // +build integration
+// +build aws
 
 package elb
 
@@ -32,10 +33,7 @@ func TestData(t *testing.T) {
 		{"AWS/NetworkELB", "./_meta/data_nlb.json"},
 	}
 
-	config, info := mtest.GetConfigForTest("elb", "300s")
-	if info != "" {
-		t.Skip("Skipping TestData: " + info)
-	}
+	config := mtest.GetConfigForTest(t, "elb", "300s")
 
 	for _, df := range dataFiles {
 		metricSet := mbtest.NewFetcher(t, config)

x-pack/metricbeat/module/aws/lambda/lambda_integration_test.go

@@ -3,6 +3,7 @@
 // you may not use this file except in compliance with the Elastic License.
 
 // +build integration
+// +build aws
 
 package lambda
 
@@ -14,10 +15,7 @@ import (
 )
 
 func TestData(t *testing.T) {
-	config, info := mtest.GetConfigForTest("lambda", "300s")
-	if info != "" {
-		t.Skip("Skipping TestData: " + info)
-	}
+	config := mtest.GetConfigForTest(t, "lambda", "300s")
 
 	metricSet := mbtest.NewFetcher(t, config)
 	metricSet.WriteEvents(t, "/")