Implement integration test suite + fix to go mods for dockerized test (…

…#129)

Also fixed some formatting in the unit test for azure-simple-hw to
normalize casing on variable names.

infra/modules/providers/azure/cosmosdb/main.tf

@@ -1,5 +1,5 @@
 data "azurerm_resource_group" "cosmosdb" {
-  name     = "${var.service_plan_resource_group_name}"
+  name = "${var.service_plan_resource_group_name}"
 }
 
 resource "azurerm_cosmosdb_account" "cosmosdb" {
@@ -12,11 +12,11 @@ resource "azurerm_cosmosdb_account" "cosmosdb" {
   enable_automatic_failover = "${var.cosmosdb_automatic_failover}"
 
   consistency_policy {
-    consistency_level       = "${var.consistency_level}"
+    consistency_level = "${var.consistency_level}"
   }
 
   geo_location {
     location          = "${var.primary_replica_location}"
     failover_priority = 0
   }
-}
+}

infra/modules/providers/azure/cosmosdb/output.tf

@@ -12,4 +12,4 @@ output "cosmosdb_primary_master_key" {
 output "cosmosdb_connection_strings" {
   description = "A list of connection strings available for this CosmosDB account."
   value       = "${azurerm_cosmosdb_account.cosmosdb.connection_strings}"
-}
+}

infra/modules/providers/azure/cosmosdb/variables.tf

@@ -9,23 +9,23 @@ variable "cosmosdb_name" {
 }
 
 variable "cosmosdb_kind" {
-    description = "Determines the kind of CosmosDB to create. Can either be 'GlobalDocumentDB' or 'MongoDB'."
-    type        = "string"
-    default     = "GlobalDocumentDB"
+  description = "Determines the kind of CosmosDB to create. Can either be 'GlobalDocumentDB' or 'MongoDB'."
+  type        = "string"
+  default     = "GlobalDocumentDB"
 }
 
 variable "cosmosdb_automatic_failover" {
-    description = "Determines if automatic failover is enabled for the created CosmosDB."
-    default     = false
+  description = "Determines if automatic failover is enabled for the created CosmosDB."
+  default     = false
 }
 
 variable "consistency_level" {
-    description = "The Consistency Level to use for this CosmosDB Account. Can be either 'BoundedStaleness', 'Eventual', 'Session', 'Strong' or 'ConsistentPrefix'."
-    type        = "string"
-    default     = "Session"
+  description = "The Consistency Level to use for this CosmosDB Account. Can be either 'BoundedStaleness', 'Eventual', 'Session', 'Strong' or 'ConsistentPrefix'."
+  type        = "string"
+  default     = "Session"
 }
 
 variable "primary_replica_location" {
-    description = "The name of the Azure region to host replicated data."
-    type        = "string"
-}
+  description = "The name of the Azure region to host replicated data."
+  type        = "string"
+}

infra/templates/azure-simple-hw/tests/integration/azure_simple_test.go

@@ -0,0 +1,68 @@
+package test
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	httpClient "github.com/gruntwork-io/terratest/modules/http-helper"
+	"github.com/gruntwork-io/terratest/modules/random"
+	"github.com/gruntwork-io/terratest/modules/terraform"
+	"github.com/microsoft/cobalt/test-harness/infratests"
+)
+
+var prefix = fmt.Sprintf("cobalt-int-tst-%s", random.UniqueId())
+var datacenter = os.Getenv("DATACENTER_LOCATION")
+
+var tfOptions = &terraform.Options{
+	TerraformDir: "../../",
+	Upgrade:      true,
+	Vars: map[string]interface{}{
+		"prefix":   prefix,
+		"location": datacenter,
+	},
+	BackendConfig: map[string]interface{}{
+		"storage_account_name": os.Getenv("TF_VAR_remote_state_account"),
+		"container_name":       os.Getenv("TF_VAR_remote_state_container"),
+	},
+}
+
+// Validates that the service responds with HTTP 200 status code. A retry strategy
+// is used because it may take some time for the application to finish standing up.
+func httpGetRespondsWith200(goTest *testing.T, output infratests.TerraformOutput) {
+	hostname := output["app_service_default_hostname"].(string)
+	maxRetries := 20
+	timeBetweenRetries := 2 * time.Second
+	expectedResponse := "Hello App Service!"
+
+	err := httpClient.HttpGetWithRetryWithCustomValidationE(
+		goTest,
+		hostname,
+		maxRetries,
+		timeBetweenRetries,
+		func(status int, content string) bool {
+			return status == 200 && strings.Contains(content, expectedResponse)
+		},
+	)
+	if err != nil {
+		goTest.Fatal(err)
+	}
+}
+
+func TestAzureSimple(t *testing.T) {
+	testFixture := infratests.IntegrationTestFixture{
+		GoTest:                t,
+		TfOptions:             tfOptions,
+		ExpectedTfOutputCount: 2,
+		ExpectedTfOutput: infratests.TerraformOutput{
+			"app_service_name":             fmt.Sprintf("%s-appservice", prefix),
+			"app_service_default_hostname": strings.ToLower(fmt.Sprintf("https://%s-appservice.azurewebsites.net", prefix)),
+		},
+		TfOutputAssertions: []infratests.TerraformOutputValidation{
+			httpGetRespondsWith200,
+		},
+	}
+	infratests.RunIntegrationTests(&testFixture)
+}

infra/templates/azure-simple-hw/test/unit/azure_simple_test.go → infra/templates/azure-simple-hw/tests/unit/azure_simple_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/microsoft/cobalt/test-harness/infratests"
 )
 
-var prefix = fmt.Sprintf("cobalt-%s", random.UniqueId())
+var prefix = fmt.Sprintf("cobalt-unit-tst-%s", random.UniqueId())
 var datacenter = os.Getenv("DATACENTER_LOCATION")
 
 var tf_options = &terraform.Options{
@@ -27,30 +27,30 @@ var tf_options = &terraform.Options{
 }
 
 func TestAzureSimple(t *testing.T) {
-	test_fixture := infratests.UnitTestFixture{
+	testFixture := infratests.UnitTestFixture{
 		GoTest:                t,
 		TfOptions:             tf_options,
 		ExpectedResourceCount: 3,
 		PlanAssertions:        nil,
-		ExpectedResourceAttributeValues: infratests.ResourceAttributeValueMapping{
-			"azurerm_app_service.main": map[string]string{
+		ExpectedResourceAttributeValues: infratests.ResourceDescription{
+			"azurerm_app_service.main": infratests.AttributeValueMapping{
 				"resource_group_name":            prefix,
 				"location":                       datacenter,
 				"site_config.0.linux_fx_version": "DOCKER|appsvcsample/static-site:latest",
 			},
-			"azurerm_app_service_plan.main": map[string]string{
+			"azurerm_app_service_plan.main": infratests.AttributeValueMapping{
 				"kind":       "Linux",
 				"location":   datacenter,
 				"reserved":   "true",
 				"sku.0.size": "S1",
 				"sku.0.tier": "Standard",
 			},
-			"azurerm_resource_group.main": map[string]string{
+			"azurerm_resource_group.main": infratests.AttributeValueMapping{
 				"location": datacenter,
 				"name":     prefix,
 			},
 		},
 	}
 
-	infratests.RunUnitTests(&test_fixture)
+	infratests.RunUnitTests(&testFixture)
 }

test-harness/Dockerfile

@@ -6,5 +6,9 @@ ARG build_directory
 RUN echo "INFO: copying $build_directory"
 # Copy the recently modified terraform templates
 ADD $build_directory *.go ./
+
+# TODO: when this is moved into the orion repo the following line can be deleted
+ADD test-harness/ ./test-harness
+
 # Run a fresh clean/format/test run
 CMD ["go", "run", "magefile.go"]

test-harness/README.md

@@ -14,9 +14,9 @@ This test harness runs automated tests for only the deployment templates that ha
 
 This module includes a library that simplifies writing unit and integration [Note: integration test support is *pending*] tests against templates. It aims to extract out the most painful pieces of this process and provide common-sense implementations that can be shared across any template. Care is taken to provide hooks for more in-depth testing if it is needed by the template maintainer.
 
-### Sample usage
+### Sample Unit Test Usage
 
-The below test shows how to leverage the library to coordinate and validate the following actions:
+The below example shows how easy it is to write a unit test that automatically coordinates the following:
 
 - Run `terraform init`, `terraform workspace select`, `terraform plan` and parse the plan output into a [Terraform Plan](https://github.com/hashicorp/terraform/blob/master/terraform/plan.go)
 - Validate that running the test would only create and not update/delete resources. (Note: This should always be true, otherwise the test is not running in isolation. Not running the test in isolation can be very dangerous and may cause resources to be deleted)
@@ -80,6 +80,81 @@ func TestAzureSimple(t *testing.T) {
 }
 ```
 
+### Sample Integration Testing Usage
+
+The below example shows how easy it is to write an integration test that automatically coordinates the following:
+
+- Run `terraform init`, `terraform workspace select`, `terraform apply` and parse the template outputs into a Go struct
+- Validate that the terraform outputs are correct by asserting that the correct number exist and that any user-supplied key-value mappings are reflected in that output.
+- Pass terraform output to user-defined test functions for use-case specific tests. In this case, we simply validate that the application endpoint responds as expected
+
+```go
+package test
+
+import (
+    "fmt"
+    "os"
+    "strings"
+    "testing"
+    "time"
+
+    httpClient "github.com/gruntwork-io/terratest/modules/http-helper"
+    "github.com/gruntwork-io/terratest/modules/random"
+    "github.com/gruntwork-io/terratest/modules/terraform"
+    "github.com/microsoft/cobalt/test-harness/infratests"
+)
+
+var prefix = fmt.Sprintf("cobalt-%s", random.UniqueId())
+var datacenter = os.Getenv("DATACENTER_LOCATION")
+
+var tfOptions = &terraform.Options{
+    TerraformDir: "../../",
+    Upgrade:      true,
+    Vars: map[string]interface{}{
+        "prefix":   prefix,
+        "location": datacenter,
+    },
+    BackendConfig: map[string]interface{}{
+        "storage_account_name": os.Getenv("TF_VAR_remote_state_account"),
+        "container_name":       os.Getenv("TF_VAR_remote_state_container"),
+    },
+}
+
+// Validates that the service responds with HTTP 200 status code. A retry strategy
+// is used because it may take some time for the application to finish standing up.
+func httpGetRespondsWith200(goTest *testing.T, output infratests.TerraformOutput) {
+    hostname := output["app_service_default_hostname"].(string)
+    maxRetries := 20
+    timeBetweenRetries := 2 * time.Second
+
+    httpClient.HttpGetWithRetryWithCustomValidationE(
+        goTest,
+        hostname,
+        maxRetries,
+        timeBetweenRetries,
+        func(status int, content string) bool {
+            return status == 200 && strings.Contains(content, "Hello App Service!")
+        },
+    )
+}
+
+func TestAzureSimple(t *testing.T) {
+    testFixture := infratests.IntegrationTestFixture{
+        GoTest:                t,
+        TfOptions:             tfOptions,
+        ExpectedTfOutputCount: 2,
+        ExpectedTfOutput: infratests.TerraformOutput{
+            "app_service_name":             fmt.Sprintf("%s-appservice", prefix),
+            "app_service_default_hostname": strings.ToLower(fmt.Sprintf("https://%s-appservice.azurewebsites.net", prefix)),
+        },
+        TfOutputAssertions: []infratests.TerraformOutputValidation{
+            httpGetRespondsWith200,
+        },
+    }
+    infratests.RunIntegrationTests(&testFixture)
+}
+```
+
 ## Test Setup Locally
 
 ### Local Environment Setup