From 4c7add3872162350402506ca59218f1fb512e82a Mon Sep 17 00:00:00 2001 From: "Nicholas M. Iodice" Date: Thu, 30 May 2019 08:13:01 -0400 Subject: [PATCH] Implement integration test suite + fix to go mods for dockerized test (#129) Also fixed some formatting in the unit test for azure-simple-hw to normalize casing on variable names. --- .../modules/providers/azure/cosmosdb/main.tf | 6 +- .../providers/azure/cosmosdb/output.tf | 2 +- .../providers/azure/cosmosdb/variables.tf | 22 ++-- .../test/integration/azure_simple_test.go | 105 --------------- .../tests/integration/azure_simple_test.go | 68 ++++++++++ .../{test => tests}/unit/azure_simple_test.go | 14 +- test-harness/Dockerfile | 4 + test-harness/README.md | 79 +++++++++++- test-harness/docker/base-images/Dockerfile | 24 +--- test-harness/infratests/integration.go | 122 ++++++++++++++++++ .../infratests/{infratests.go => unit.go} | 96 +++++++------- test-harness/init.sh | 7 +- test-harness/local-run.sh | 1 + magefile.go => test-harness/magefile.go | 39 +++--- 14 files changed, 375 insertions(+), 214 deletions(-) delete mode 100644 infra/templates/azure-simple-hw/test/integration/azure_simple_test.go create mode 100644 infra/templates/azure-simple-hw/tests/integration/azure_simple_test.go rename infra/templates/azure-simple-hw/{test => tests}/unit/azure_simple_test.go (73%) create mode 100644 test-harness/infratests/integration.go rename test-harness/infratests/{infratests.go => unit.go} (63%) rename magefile.go => test-harness/magefile.go (62%) diff --git a/infra/modules/providers/azure/cosmosdb/main.tf b/infra/modules/providers/azure/cosmosdb/main.tf index 9c260f82..cdc61d1e 100644 --- a/infra/modules/providers/azure/cosmosdb/main.tf +++ b/infra/modules/providers/azure/cosmosdb/main.tf @@ -1,5 +1,5 @@ data "azurerm_resource_group" "cosmosdb" { - name = "${var.service_plan_resource_group_name}" + name = "${var.service_plan_resource_group_name}" } resource "azurerm_cosmosdb_account" "cosmosdb" { @@ -12,11 +12,11 @@ resource "azurerm_cosmosdb_account" "cosmosdb" { enable_automatic_failover = "${var.cosmosdb_automatic_failover}" consistency_policy { - consistency_level = "${var.consistency_level}" + consistency_level = "${var.consistency_level}" } geo_location { location = "${var.primary_replica_location}" failover_priority = 0 } -} \ No newline at end of file +} diff --git a/infra/modules/providers/azure/cosmosdb/output.tf b/infra/modules/providers/azure/cosmosdb/output.tf index 05496660..aa1106ad 100644 --- a/infra/modules/providers/azure/cosmosdb/output.tf +++ b/infra/modules/providers/azure/cosmosdb/output.tf @@ -12,4 +12,4 @@ output "cosmosdb_primary_master_key" { output "cosmosdb_connection_strings" { description = "A list of connection strings available for this CosmosDB account." value = "${azurerm_cosmosdb_account.cosmosdb.connection_strings}" -} \ No newline at end of file +} diff --git a/infra/modules/providers/azure/cosmosdb/variables.tf b/infra/modules/providers/azure/cosmosdb/variables.tf index 75b36c50..13c71188 100644 --- a/infra/modules/providers/azure/cosmosdb/variables.tf +++ b/infra/modules/providers/azure/cosmosdb/variables.tf @@ -9,23 +9,23 @@ variable "cosmosdb_name" { } variable "cosmosdb_kind" { - description = "Determines the kind of CosmosDB to create. Can either be 'GlobalDocumentDB' or 'MongoDB'." - type = "string" - default = "GlobalDocumentDB" + description = "Determines the kind of CosmosDB to create. Can either be 'GlobalDocumentDB' or 'MongoDB'." + type = "string" + default = "GlobalDocumentDB" } variable "cosmosdb_automatic_failover" { - description = "Determines if automatic failover is enabled for the created CosmosDB." - default = false + description = "Determines if automatic failover is enabled for the created CosmosDB." + default = false } variable "consistency_level" { - description = "The Consistency Level to use for this CosmosDB Account. Can be either 'BoundedStaleness', 'Eventual', 'Session', 'Strong' or 'ConsistentPrefix'." - type = "string" - default = "Session" + description = "The Consistency Level to use for this CosmosDB Account. Can be either 'BoundedStaleness', 'Eventual', 'Session', 'Strong' or 'ConsistentPrefix'." + type = "string" + default = "Session" } variable "primary_replica_location" { - description = "The name of the Azure region to host replicated data." - type = "string" -} \ No newline at end of file + description = "The name of the Azure region to host replicated data." + type = "string" +} diff --git a/infra/templates/azure-simple-hw/test/integration/azure_simple_test.go b/infra/templates/azure-simple-hw/test/integration/azure_simple_test.go deleted file mode 100644 index d7006e9d..00000000 --- a/infra/templates/azure-simple-hw/test/integration/azure_simple_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package test - -import ( - "fmt" - "os" - "path" - "strings" - "testing" - "time" - - http_helper "github.com/gruntwork-io/terratest/modules/http-helper" - random "github.com/gruntwork-io/terratest/modules/random" - terraform "github.com/gruntwork-io/terratest/modules/terraform" - terraformCore "github.com/hashicorp/terraform/terraform" -) - -func configureTerraformOptions(t *testing.T, fixtureFolder string) *terraform.Options { - location := os.Getenv("DATACENTER_LOCATION") - //Generate unique azure resources to minimize feature conflicts across an engineering crew. ie cobalt-Geqw4K-appservice, cobalt-Geqw4K-resources - prefix := fmt.Sprintf("cobalt-%s", random.UniqueId()) - tfstate_storage_account := os.Getenv("TF_VAR_remote_state_account") - tf_state_container := os.Getenv("TF_VAR_remote_state_container") - terraformOptions := &terraform.Options{ - // The path to where our Terraform code is located - TerraformDir: fixtureFolder, - Upgrade: true, - Vars: map[string]interface{}{ - "prefix": prefix, - "location": location, - }, - BackendConfig: map[string]interface{}{ - "storage_account_name": tfstate_storage_account, - "container_name": tf_state_container, - }, - } - - return terraformOptions -} - -// RunTestStage executes the given test stage (e.g., setup, teardown, validation) -func RunTestStage(t *testing.T, stageName string, stage func()) { - fmt.Printf("Executing stage '%s'.", stageName) - stage() -} - -func validatePlan(t *testing.T, tfPlanOutput string, tfOptions *terraform.Options) { - terraform.RunTerraformCommand(t, tfOptions, terraform.FormatArgs(tfOptions, "plan", "-out="+tfPlanOutput)...) - - // Read and parse the plan output - f, err := os.Open(path.Join(tfOptions.TerraformDir, tfPlanOutput)) - if err != nil { - fmt.Printf("ERROR: plan parsing error message: %v\n", err) - t.Fatal(err) - } - defer f.Close() - plan, err := terraformCore.ReadPlan(f) - if err != nil { - fmt.Printf("ERROR: plan output message: %v\n", err) - t.Fatal(err) - } - - for _, mod := range plan.Diff.Modules { - if len(mod.Path) == 1 && mod.Path[0] == "root" { - expected := os.Getenv("DATACENTER_LOCATION") - actual := mod.Resources["azurerm_app_service.main"].Attributes["location"].New - if actual != expected { - t.Fatalf("ERROR: Expect %v, but found %v", expected, actual) - } - } - } -} - -func TestITAzureSimple(t *testing.T) { - t.Parallel() - fixtureFolder := "../../" - terraformOptions := configureTerraformOptions(t, fixtureFolder) - defer terraform.Destroy(t, terraformOptions) - - RunTestStage(t, "init", func() { - terraform.Init(t, terraformOptions) - }) - - RunTestStage(t, "validate-plan", func() { - tfPlanOutput := "terraform.tfplan" - validatePlan(t, tfPlanOutput, terraformOptions) - }) - - RunTestStage(t, "apply", func() { - terraform.Apply(t, terraformOptions) - }) - - // Check whether the length of output meets the requirement. In public case, we check whether there occurs a public IP. - RunTestStage(t, "validate-e2e", func() { - hostname := terraform.Output(t, terraformOptions, "app_service_default_hostname") - - // Validate the provisioned webpage container - // It can take several minutes or so for the app to be deployed, so retry a few times - maxRetries := 60 - timeBetweenRetries := 4 * time.Second - - http_helper.HttpGetWithRetryWithCustomValidationE(t, hostname, maxRetries, timeBetweenRetries, func(status int, content string) bool { - return status == 200 && strings.Contains(content, "Hello App Service!") - }) - }) -} diff --git a/infra/templates/azure-simple-hw/tests/integration/azure_simple_test.go b/infra/templates/azure-simple-hw/tests/integration/azure_simple_test.go new file mode 100644 index 00000000..bada3f03 --- /dev/null +++ b/infra/templates/azure-simple-hw/tests/integration/azure_simple_test.go @@ -0,0 +1,68 @@ +package test + +import ( + "fmt" + "os" + "strings" + "testing" + "time" + + httpClient "github.com/gruntwork-io/terratest/modules/http-helper" + "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/microsoft/cobalt/test-harness/infratests" +) + +var prefix = fmt.Sprintf("cobalt-int-tst-%s", random.UniqueId()) +var datacenter = os.Getenv("DATACENTER_LOCATION") + +var tfOptions = &terraform.Options{ + TerraformDir: "../../", + Upgrade: true, + Vars: map[string]interface{}{ + "prefix": prefix, + "location": datacenter, + }, + BackendConfig: map[string]interface{}{ + "storage_account_name": os.Getenv("TF_VAR_remote_state_account"), + "container_name": os.Getenv("TF_VAR_remote_state_container"), + }, +} + +// Validates that the service responds with HTTP 200 status code. A retry strategy +// is used because it may take some time for the application to finish standing up. +func httpGetRespondsWith200(goTest *testing.T, output infratests.TerraformOutput) { + hostname := output["app_service_default_hostname"].(string) + maxRetries := 20 + timeBetweenRetries := 2 * time.Second + expectedResponse := "Hello App Service!" + + err := httpClient.HttpGetWithRetryWithCustomValidationE( + goTest, + hostname, + maxRetries, + timeBetweenRetries, + func(status int, content string) bool { + return status == 200 && strings.Contains(content, expectedResponse) + }, + ) + if err != nil { + goTest.Fatal(err) + } +} + +func TestAzureSimple(t *testing.T) { + testFixture := infratests.IntegrationTestFixture{ + GoTest: t, + TfOptions: tfOptions, + ExpectedTfOutputCount: 2, + ExpectedTfOutput: infratests.TerraformOutput{ + "app_service_name": fmt.Sprintf("%s-appservice", prefix), + "app_service_default_hostname": strings.ToLower(fmt.Sprintf("https://%s-appservice.azurewebsites.net", prefix)), + }, + TfOutputAssertions: []infratests.TerraformOutputValidation{ + httpGetRespondsWith200, + }, + } + infratests.RunIntegrationTests(&testFixture) +} diff --git a/infra/templates/azure-simple-hw/test/unit/azure_simple_test.go b/infra/templates/azure-simple-hw/tests/unit/azure_simple_test.go similarity index 73% rename from infra/templates/azure-simple-hw/test/unit/azure_simple_test.go rename to infra/templates/azure-simple-hw/tests/unit/azure_simple_test.go index 5818a6aa..778364ae 100644 --- a/infra/templates/azure-simple-hw/test/unit/azure_simple_test.go +++ b/infra/templates/azure-simple-hw/tests/unit/azure_simple_test.go @@ -10,7 +10,7 @@ import ( "github.com/microsoft/cobalt/test-harness/infratests" ) -var prefix = fmt.Sprintf("cobalt-%s", random.UniqueId()) +var prefix = fmt.Sprintf("cobalt-unit-tst-%s", random.UniqueId()) var datacenter = os.Getenv("DATACENTER_LOCATION") var tf_options = &terraform.Options{ @@ -27,30 +27,30 @@ var tf_options = &terraform.Options{ } func TestAzureSimple(t *testing.T) { - test_fixture := infratests.UnitTestFixture{ + testFixture := infratests.UnitTestFixture{ GoTest: t, TfOptions: tf_options, ExpectedResourceCount: 3, PlanAssertions: nil, - ExpectedResourceAttributeValues: infratests.ResourceAttributeValueMapping{ - "azurerm_app_service.main": map[string]string{ + ExpectedResourceAttributeValues: infratests.ResourceDescription{ + "azurerm_app_service.main": infratests.AttributeValueMapping{ "resource_group_name": prefix, "location": datacenter, "site_config.0.linux_fx_version": "DOCKER|appsvcsample/static-site:latest", }, - "azurerm_app_service_plan.main": map[string]string{ + "azurerm_app_service_plan.main": infratests.AttributeValueMapping{ "kind": "Linux", "location": datacenter, "reserved": "true", "sku.0.size": "S1", "sku.0.tier": "Standard", }, - "azurerm_resource_group.main": map[string]string{ + "azurerm_resource_group.main": infratests.AttributeValueMapping{ "location": datacenter, "name": prefix, }, }, } - infratests.RunUnitTests(&test_fixture) + infratests.RunUnitTests(&testFixture) } diff --git a/test-harness/Dockerfile b/test-harness/Dockerfile index e6790d05..3e3286ba 100644 --- a/test-harness/Dockerfile +++ b/test-harness/Dockerfile @@ -6,5 +6,9 @@ ARG build_directory RUN echo "INFO: copying $build_directory" # Copy the recently modified terraform templates ADD $build_directory *.go ./ + +# TODO: when this is moved into the orion repo the following line can be deleted +ADD test-harness/ ./test-harness + # Run a fresh clean/format/test run CMD ["go", "run", "magefile.go"] \ No newline at end of file diff --git a/test-harness/README.md b/test-harness/README.md index e26bccf5..fd2083a6 100644 --- a/test-harness/README.md +++ b/test-harness/README.md @@ -14,9 +14,9 @@ This test harness runs automated tests for only the deployment templates that ha This module includes a library that simplifies writing unit and integration [Note: integration test support is *pending*] tests against templates. It aims to extract out the most painful pieces of this process and provide common-sense implementations that can be shared across any template. Care is taken to provide hooks for more in-depth testing if it is needed by the template maintainer. -### Sample usage +### Sample Unit Test Usage -The below test shows how to leverage the library to coordinate and validate the following actions: +The below example shows how easy it is to write a unit test that automatically coordinates the following: - Run `terraform init`, `terraform workspace select`, `terraform plan` and parse the plan output into a [Terraform Plan](https://github.com/hashicorp/terraform/blob/master/terraform/plan.go) - Validate that running the test would only create and not update/delete resources. (Note: This should always be true, otherwise the test is not running in isolation. Not running the test in isolation can be very dangerous and may cause resources to be deleted) @@ -80,6 +80,81 @@ func TestAzureSimple(t *testing.T) { } ``` +### Sample Integration Testing Usage + +The below example shows how easy it is to write an integration test that automatically coordinates the following: + +- Run `terraform init`, `terraform workspace select`, `terraform apply` and parse the template outputs into a Go struct +- Validate that the terraform outputs are correct by asserting that the correct number exist and that any user-supplied key-value mappings are reflected in that output. +- Pass terraform output to user-defined test functions for use-case specific tests. In this case, we simply validate that the application endpoint responds as expected + +```go +package test + +import ( + "fmt" + "os" + "strings" + "testing" + "time" + + httpClient "github.com/gruntwork-io/terratest/modules/http-helper" + "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/microsoft/cobalt/test-harness/infratests" +) + +var prefix = fmt.Sprintf("cobalt-%s", random.UniqueId()) +var datacenter = os.Getenv("DATACENTER_LOCATION") + +var tfOptions = &terraform.Options{ + TerraformDir: "../../", + Upgrade: true, + Vars: map[string]interface{}{ + "prefix": prefix, + "location": datacenter, + }, + BackendConfig: map[string]interface{}{ + "storage_account_name": os.Getenv("TF_VAR_remote_state_account"), + "container_name": os.Getenv("TF_VAR_remote_state_container"), + }, +} + +// Validates that the service responds with HTTP 200 status code. A retry strategy +// is used because it may take some time for the application to finish standing up. +func httpGetRespondsWith200(goTest *testing.T, output infratests.TerraformOutput) { + hostname := output["app_service_default_hostname"].(string) + maxRetries := 20 + timeBetweenRetries := 2 * time.Second + + httpClient.HttpGetWithRetryWithCustomValidationE( + goTest, + hostname, + maxRetries, + timeBetweenRetries, + func(status int, content string) bool { + return status == 200 && strings.Contains(content, "Hello App Service!") + }, + ) +} + +func TestAzureSimple(t *testing.T) { + testFixture := infratests.IntegrationTestFixture{ + GoTest: t, + TfOptions: tfOptions, + ExpectedTfOutputCount: 2, + ExpectedTfOutput: infratests.TerraformOutput{ + "app_service_name": fmt.Sprintf("%s-appservice", prefix), + "app_service_default_hostname": strings.ToLower(fmt.Sprintf("https://%s-appservice.azurewebsites.net", prefix)), + }, + TfOutputAssertions: []infratests.TerraformOutputValidation{ + httpGetRespondsWith200, + }, + } + infratests.RunIntegrationTests(&testFixture) +} +``` + ## Test Setup Locally ### Local Environment Setup diff --git a/test-harness/docker/base-images/Dockerfile b/test-harness/docker/base-images/Dockerfile index 0a733274..b6de7315 100644 --- a/test-harness/docker/base-images/Dockerfile +++ b/test-harness/docker/base-images/Dockerfile @@ -26,7 +26,7 @@ RUN echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ stret tee /etc/apt/sources.list.d/azure-cli.list # Install Azure CLI + core compilers like gcc which are required for dep -RUN apt-get update && apt-get install -y build-essential wget unzip +RUN apt-get update && apt-get install -y build-essential wget unzip azure-cli ENV GOLANG_VERSION=$gover ENV PATH /usr/local/go/bin:/usr/local/go:$PATH @@ -34,33 +34,17 @@ ENV GOPATH $HOME/go ENV GOBIN /usr/local/go # Install Terraform -ARG tfver=0.11.3 +ARG tfver=0.11.13 ENV TF_VERSION=$tfver RUN wget --quiet https://releases.hashicorp.com/terraform/${TF_VERSION}/terraform_${TF_VERSION}_linux_amd64.zip \ && unzip terraform_${TF_VERSION}_linux_amd64.zip \ && mv terraform /usr/bin \ && rm terraform_${TF_VERSION}_linux_amd64.zip -# Go dep! -RUN /bin/bash -c "curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh" - # setup project workspace WORKDIR $HOME/app/ -# Copy over go unit / int tests + mage manifest -ADD *.go ./ -RUN mkdir infra build -ADD infra/ ./infra -ADD test-harness/infratests/ ./test-harness/infratests - -RUN apt-get purge -y wget build-essential wget unzip curl \ - apt-transport-https lsb-release gpg - -# This initializes the name of the module -RUN ["go", "mod", "init", "github.com/microsoft/cobalt"] -# This command will look for local packages and will inflate go.mod and go.sum along the way. -# It will also pull down any missing dependencies. -RUN ["go", "list", "./..."] -RUN rm -r ./infra +ADD go.mod go.sum ./ +RUN ["go", "mod", "download"] CMD bash \ No newline at end of file diff --git a/test-harness/infratests/integration.go b/test-harness/infratests/integration.go new file mode 100644 index 00000000..350da60e --- /dev/null +++ b/test-harness/infratests/integration.go @@ -0,0 +1,122 @@ +/* +This file provides abstractions that simplify the process of integration-testing terraform templates. The goal +is to minimize the boiler plate code required to effectively test terraform templates in order to reduce +the effort required to write robust template integration-tests. +*/ +package infratests + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/gruntwork-io/terratest/modules/terraform" +) + +// TerraformOutput Models terraform output key values +type TerraformOutput map[string]interface{} + +// TerraformOutputValidation A function that can validate terraform output +type TerraformOutputValidation func(goTest *testing.T, output TerraformOutput) + +// IntegrationTestFixture Holds metadata required to execute an integration test against a test against a terraform template +type IntegrationTestFixture struct { + GoTest *testing.T // Go test harness + TfOptions *terraform.Options // Terraform options + ExpectedTfOutputCount int // Expected # of resources that Terraform should create + ExpectedTfOutput TerraformOutput // Expected Terraform Output + TfOutputAssertions []TerraformOutputValidation // user-defined plan assertions + Workspace string +} + +// RunIntegrationTests Executes terraform lifecycle events and verifies the correctness of the resulting resources. +// The following actions are coordinated: +// - Run `terraform init` +// - Create new terraform workspace. This helps prevent accidentally deleting resources +// - Run `terraform apply` +// - Run `terraform output` +// - Validate outputs +// - Run user-supplied validation of outputs +// - Destroy resources +func RunIntegrationTests(fixture *IntegrationTestFixture) { + terraform.Init(fixture.GoTest, fixture.TfOptions) + + workspaceName := fixture.Workspace + if workspaceName == "" { + workspaceName = "default-int-testing" + } + + terraform.WorkspaceSelectOrNew(fixture.GoTest, fixture.TfOptions, workspaceName) + defer terraform.RunTerraformCommand(fixture.GoTest, fixture.TfOptions, "workspace", "delete", workspaceName) + defer terraform.WorkspaceSelectOrNew(fixture.GoTest, fixture.TfOptions, "default") + + terraform.Apply(fixture.GoTest, fixture.TfOptions) + defer terraform.Destroy(fixture.GoTest, fixture.TfOptions) + + output := terraform.OutputAll(fixture.GoTest, fixture.TfOptions) + validateTerraformOutput(fixture, TerraformOutput(output)) +} + +// Coordinates the following validations of a terraform output: +// - The output contains the correct number of items +// - The output values match any user-supplied key-value mappings. This only validates +// that any user-supplied key-value mappings are correct, and will not fail if the +// output has more mappings +// - The output has the correct number of items +// - Run any user-supplied assertions over the output +func validateTerraformOutput(fixture *IntegrationTestFixture, output TerraformOutput) { + validateTerraformOutputCount(fixture, output) + validateTerraformOutputKeyValues(fixture, output) + + // run user-provided assertions over the TF output + for _, outputAssertion := range fixture.TfOutputAssertions { + outputAssertion(fixture.GoTest, output) + } +} + +// Validates that the terraform output contains the expected number of items +func validateTerraformOutputCount(fixture *IntegrationTestFixture, output TerraformOutput) { + if len(output) != fixture.ExpectedTfOutputCount { + fixture.GoTest.Fatal(fmt.Errorf( + "Output unexpectedly had %d entries instead of %d", + len(output), + fixture.ExpectedTfOutputCount, + )) + } +} + +// Validates that any outputs that the user supplies match the actual terraform outputs. +// Note: the comparison is done by converting the expected and actual values into JSON and +// doing a string comparison. This solves a number of complexities, such as: +// - Handles comparison of generic data types automatically +// - Handles differences in key ordering for maps +// - Handles all handling of generics, which is tricky in Go +func validateTerraformOutputKeyValues(fixture *IntegrationTestFixture, output TerraformOutput) { + for expectedKey, expectedValue := range fixture.ExpectedTfOutput { + actualValue, isFound := output[expectedKey] + if !isFound { + fixture.GoTest.Fatal(fmt.Errorf("Output unexpectedly did not contain key %s", expectedKey)) + } + + expectedAsJSON := jsonOrFail(fixture, expectedValue) + actualAsJSON := jsonOrFail(fixture, actualValue) + + if expectedAsJSON != actualAsJSON { + fixture.GoTest.Fatal(fmt.Errorf( + "Output value for '%s' was expected to be '%s' but was '%s'", + expectedKey, + expectedAsJSON, + actualAsJSON, + )) + } + } +} + +// parse data to JSON or fail if an error was encountered +func jsonOrFail(fixture *IntegrationTestFixture, value interface{}) string { + asJSON, err := json.Marshal(value) + if err != nil { + fixture.GoTest.Fatal(err) + } + return string(asJSON) +} diff --git a/test-harness/infratests/infratests.go b/test-harness/infratests/unit.go similarity index 63% rename from test-harness/infratests/infratests.go rename to test-harness/infratests/unit.go index d7fe6a9c..d39f0b11 100644 --- a/test-harness/infratests/infratests.go +++ b/test-harness/infratests/unit.go @@ -1,10 +1,7 @@ /* -Package `infratests` is intended to act as a testing harness that makes testing Terraform templates -easy and efficient. The goal of this package is to minimize the boiler plate code required to effectively -test terraform implementations. - -The current implementation is focused only on unit tests but it will be expanded to harness integration tests -as well. +This file provides abstractions that simplify the process of unit-testing terraform templates. The goal +is to minimize the boiler plate code required to effectively test terraform templates in order to reduce +the effort required to write robust template unit-tests. */ package infratests @@ -20,20 +17,27 @@ import ( terraformCore "github.com/hashicorp/terraform/terraform" ) -type ResourceAttributeValueMapping map[string]map[string]string +// AttributeValueMapping Identifies mapping between attributes and values +type AttributeValueMapping map[string]string + +// ResourceDescription Identifies mappings between resources and attributes +type ResourceDescription map[string]AttributeValueMapping + +// TerraformPlanValidation A function that can run an assertion over a terraform plan type TerraformPlanValidation func(goTest *testing.T, plan *terraformCore.Plan) -// Holds metadata required to execute a unit test against a test against a terraform template +// UnitTestFixture Holds metadata required to execute a unit test against a test against a terraform template type UnitTestFixture struct { GoTest *testing.T // Go test harness TfOptions *terraform.Options // Terraform options - ExpectedResourceCount int // Expected # of resources that Terraform should create + Workspace string + ExpectedResourceCount int // Expected # of resources that Terraform should create // map of maps specifying resource <--> attribute <--> attribute value mappings - ExpectedResourceAttributeValues ResourceAttributeValueMapping + ExpectedResourceAttributeValues ResourceDescription PlanAssertions []TerraformPlanValidation // user-defined plan assertions } -// Executes terraform lifecycle events and verifies the correctness of the resulting terraform. +// RunUnitTests Executes terraform lifecycle events and verifies the correctness of the resulting terraform. // The following actions are coordinated: // - Run `terraform init` // - Create new terraform workspace. This helps prevent accidentally deleting resources @@ -42,19 +46,23 @@ type UnitTestFixture struct { func RunUnitTests(fixture *UnitTestFixture) { terraform.Init(fixture.GoTest, fixture.TfOptions) - workspace_name := random.UniqueId() - terraform.WorkspaceSelectOrNew(fixture.GoTest, fixture.TfOptions, workspace_name) - defer terraform.RunTerraformCommand(fixture.GoTest, fixture.TfOptions, "workspace", "delete", workspace_name) + workspaceName := fixture.Workspace + if workspaceName == "" { + workspaceName = "default-unit-testing" + } + + terraform.WorkspaceSelectOrNew(fixture.GoTest, fixture.TfOptions, workspaceName) + defer terraform.RunTerraformCommand(fixture.GoTest, fixture.TfOptions, "workspace", "delete", workspaceName) defer terraform.WorkspaceSelectOrNew(fixture.GoTest, fixture.TfOptions, "default") - tf_plan_file_path := random.UniqueId() + ".plan" + tfPlanFilePath := random.UniqueId() + ".plan" terraform.RunTerraformCommand( fixture.GoTest, fixture.TfOptions, - terraform.FormatArgs(fixture.TfOptions, "plan", "-input=false", "-out", tf_plan_file_path)...) - defer os.Remove(tf_plan_file_path) + terraform.FormatArgs(fixture.TfOptions, "plan", "-input=false", "-out", tfPlanFilePath)...) + defer os.Remove(tfPlanFilePath) - validateTerraformPlanFile(fixture, tf_plan_file_path) + validateTerraformPlanFile(fixture, tfPlanFilePath) } // Validates a terraform plan file based on the test fixture. The following validations are made: @@ -63,8 +71,8 @@ func RunUnitTests(fixture *UnitTestFixture) { // be brand new infrastructure on each PR cycle. // - The resource <--> attribute <--> attribute value mappings match the parameters from the test fixture // - The plan passes any user-defined assertions -func validateTerraformPlanFile(fixture *UnitTestFixture, tf_plan_file_path string) { - file, err := os.Open(path.Join(fixture.TfOptions.TerraformDir, tf_plan_file_path)) +func validateTerraformPlanFile(fixture *UnitTestFixture, tfPlanFilePath string) { + file, err := os.Open(path.Join(fixture.TfOptions.TerraformDir, tfPlanFilePath)) if err != nil { fixture.GoTest.Fatal(err) } @@ -98,8 +106,8 @@ func validatePlanCreateProperties(fixture *UnitTestFixture, plan *terraformCore. // plans should contain diffs of type `DiffCreate` otherwise the test may accidentally remove resources for _, module := range plan.Diff.Modules { if module.ChangeType() != terraformCore.DiffCreate { - fixture.GoTest.Fatal(errors.New( - fmt.Sprintf("Plan unexpectedly contained an update of type '%s'", module.ChangeType()))) + fixture.GoTest.Fatal( + fmt.Errorf("Plan unexpectedly contained an update of type '%s'", string(module.ChangeType()))) } } @@ -110,8 +118,8 @@ func validatePlanCreateProperties(fixture *UnitTestFixture, plan *terraformCore. // every plan should have the correct number of resources if resourceCount != fixture.ExpectedResourceCount { - fixture.GoTest.Fatal(errors.New(fmt.Sprintf( - "Plan unexpectedly had %d resources instead of %d", resourceCount, fixture.ExpectedResourceCount))) + fixture.GoTest.Fatal(fmt.Errorf( + "Plan unexpectedly had %d resources instead of %d", resourceCount, fixture.ExpectedResourceCount)) } } @@ -119,11 +127,11 @@ func validatePlanCreateProperties(fixture *UnitTestFixture, plan *terraformCore. // from the test fixture func validatePlanResourceKeyValues(fixture *UnitTestFixture, plan *terraformCore.Plan) { // collect actual resource attrubte value mappings by iterating over the TF plan - seen := ResourceAttributeValueMapping{} + seen := ResourceDescription{} for _, module := range plan.Diff.Modules { - for resource, resource_diffs := range module.Resources { - seen[resource] = map[string]string{} - for attribute, vals := range resource_diffs.Attributes { + for resource, resourceDiffs := range module.Resources { + seen[resource] = AttributeValueMapping{} + for attribute, vals := range resourceDiffs.Attributes { seen[resource][attribute] = vals.New } } @@ -131,28 +139,28 @@ func validatePlanResourceKeyValues(fixture *UnitTestFixture, plan *terraformCore // verify that for each of the expected resource attribute value mappings that the expected // values are found in the terraform plan - for resource, expected_attr_val_mappings := range fixture.ExpectedResourceAttributeValues { - _, resource_found := seen[resource] - if !resource_found { - fixture.GoTest.Fatal(errors.New(fmt.Sprintf( - "Plan unexpectedly did not contain a change for resource '%s'", resource))) + for resource, expectedMappings := range fixture.ExpectedResourceAttributeValues { + _, resourceFound := seen[resource] + if !resourceFound { + fixture.GoTest.Fatal(fmt.Errorf( + "Plan unexpectedly did not contain a change for resource '%s'", resource)) } - for expected_attr, expected_val := range expected_attr_val_mappings { - actual_val, attr_found := seen[resource][expected_attr] - if !attr_found { - fixture.GoTest.Fatal(errors.New(fmt.Sprintf( - "Plan unexpectedly did not contain a change for '%s.%s'", resource, expected_attr))) + for expectedAttr, expectedVal := range expectedMappings { + actualVal, attrFound := seen[resource][expectedAttr] + if !attrFound { + fixture.GoTest.Fatal(fmt.Errorf( + "Plan unexpectedly did not contain a change for '%s.%s'", resource, expectedAttr)) } - if expected_val != actual_val { - fixture.GoTest.Fatal(errors.New(fmt.Sprintf( + if expectedVal != actualVal { + fixture.GoTest.Fatal(fmt.Errorf( "Plan unexpectedly had value '%s' instead of '%s' for '%s.%s'", - actual_val, - expected_val, + actualVal, + expectedVal, resource, - expected_attr, - ))) + expectedAttr, + )) } } } diff --git a/test-harness/init.sh b/test-harness/init.sh index 26b37e6f..0298f8a0 100644 --- a/test-harness/init.sh +++ b/test-harness/init.sh @@ -86,7 +86,12 @@ function add_template_if_not_exists() { function load_build_directory() { template_dirs=$( IFS=$' '; echo "${TEST_RUN_MAP[*]}" ) echoInfo "INFO: Running local build for templates: $template_dirs" - mkdir $BUILD_TEMPLATE_DIRS && cp -r $template_dirs *.go $BUILD_TEMPLATE_DIRS + mkdir $BUILD_TEMPLATE_DIRS + mkdir $BUILD_TEMPLATE_DIRS/infra + mkdir $BUILD_TEMPLATE_DIRS/modules + cp -r $template_dirs $BUILD_TEMPLATE_DIRS/infra + cp -r infra/modules $BUILD_TEMPLATE_DIRS/ + cp test-harness/*.go $BUILD_TEMPLATE_DIRS } # Builds the test harness off the template changes from the git log diff --git a/test-harness/local-run.sh b/test-harness/local-run.sh index f0c00e6f..3a6b88b0 100755 --- a/test-harness/local-run.sh +++ b/test-harness/local-run.sh @@ -123,6 +123,7 @@ function run_test_image() { -e TF_VAR_remote_state_account=$TF_VAR_remote_state_account \ -e TF_VAR_remote_state_container=$TF_VAR_remote_state_container \ -e ARM_ACCESS_KEY=$ARM_ACCESS_KEY \ + -e TF_WARN_OUTPUT_ERRORS=$TF_WARN_OUTPUT_ERRORS \ --rm $BUILD_TEST_RUN_IMAGE:$BUILD_BUILDID echoInfo "INFO: Completed test run" diff --git a/magefile.go b/test-harness/magefile.go similarity index 62% rename from magefile.go rename to test-harness/magefile.go index 45d8d75d..07046854 100644 --- a/magefile.go +++ b/test-harness/magefile.go @@ -2,7 +2,6 @@ package main import ( - "errors" "fmt" "os" "path/filepath" @@ -12,14 +11,14 @@ import ( "github.com/magefile/mage/sh" ) -// The default target when the command executes `mage` in Cloud Shell +// Default The default target when the command executes `mage` in Cloud Shell var Default = RunAllTargets func main() { Default() } -// A build step that runs Clean, Format, Unit and Integration in sequence +// RunAllTargets A build step that runs Clean, Format, Unit and Integration in sequence func RunAllTargets() { mg.Deps(CleanAll) mg.Deps(LintCheckGo) @@ -28,20 +27,20 @@ func RunAllTargets() { mg.Deps(RunIntegrationTests) } -// A build step that runs unit tests +// RunUnitTests A build step that runs unit tests func RunUnitTests() error { fmt.Println("INFO: Running unit tests...") return FindAndRunTests("unit") } -// A build step that runs integration tests +// RunIntegrationTests A build step that runs integration tests func RunIntegrationTests() error { fmt.Println("INFO: Running integration tests...") return FindAndRunTests("integration") } -// finds all tests with a given path suffix and runs them using `go test` -func FindAndRunTests(path_suffix string) error { +// FindAndRunTests finds all tests with a given path suffix and runs them using `go test` +func FindAndRunTests(pathSuffix string) error { goModules, err := sh.Output("go", "list", "./...") if err != nil { return err @@ -49,35 +48,35 @@ func FindAndRunTests(path_suffix string) error { testTargetModules := make([]string, 0) for _, module := range strings.Fields(goModules) { - if strings.HasSuffix(module, path_suffix) { + if strings.HasSuffix(module, pathSuffix) { testTargetModules = append(testTargetModules, module) } } if len(testTargetModules) == 0 { - return errors.New(fmt.Sprintf("No modules found for testing prefix '%s'", path_suffix)) + return fmt.Errorf("No modules found for testing prefix '%s'", pathSuffix) } - cmd_args := []string{"test"} - cmd_args = append(cmd_args, testTargetModules...) - cmd_args = append(cmd_args, "-v", "-timeout", "7200s") - return sh.RunV("go", cmd_args...) + cmdArgs := []string{"test"} + cmdArgs = append(cmdArgs, testTargetModules...) + cmdArgs = append(cmdArgs, "-v", "-timeout", "7200s") + return sh.RunV("go", cmdArgs...) } -// A build step that fails if go code is not formatted properly +// LintCheckGo A build step that fails if go code is not formatted properly func LintCheckGo() error { fmt.Println("INFO: Checking format for Go files...") - return VerifyRunsQuietly("go", "fmt", "./...") + return verifyRunsQuietly("go", "fmt", "./...") } -// a build step that fails if terraform files are not formatted properly +// LintCheckTerraform a build step that fails if terraform files are not formatted properly func LintCheckTerraform() error { fmt.Println("INFO: Checking format for Terraform files...") - return VerifyRunsQuietly("terraform", "fmt") + return verifyRunsQuietly("terraform", "fmt") } // runs a command and ensures that the exit code indicates success and that there is no output to stdout -func VerifyRunsQuietly(cmd string, args ...string) error { +func verifyRunsQuietly(cmd string, args ...string) error { output, err := sh.Output(cmd, args...) if err != nil { @@ -88,10 +87,10 @@ func VerifyRunsQuietly(cmd string, args ...string) error { return nil } - return errors.New(fmt.Sprintf("ERROR: command '%s' with arguments %s failed. Output was: '%s'", cmd, args, output)) + return fmt.Errorf("ERROR: command '%s' with arguments %s failed. Output was: '%s'", cmd, args, output) } -// A build step that removes temporary build and test files +// CleanAll A build step that removes temporary build and test files func CleanAll() error { fmt.Println("INFO: Cleaning...") return filepath.Walk(".", func(path string, info os.FileInfo, err error) error {