From 155eba5eea0a2ab7acebe0c0182d614b5008cdfd Mon Sep 17 00:00:00 2001 From: Iryna Shustava Date: Wed, 26 Aug 2020 13:14:31 -0700 Subject: [PATCH] Add acceptance tests instructions to Contributing (#584) * Update CONTRIBUTING.md with new acceptance tests info * Fix race condition in terraform templates --- .circleci/config.yml | 7 +- CONTRIBUTING.md | 206 +++++++++++++++++- test/acceptance/framework/environment.go | 1 + test/acceptance/tests/example/example_test.go | 58 +++++ test/acceptance/tests/example/main_test.go | 33 +++ test/terraform/gke/main.tf | 14 +- test/terraform/gke/outputs.tf | 4 +- test/terraform/gke/variables.tf | 4 +- 8 files changed, 305 insertions(+), 22 deletions(-) create mode 100644 test/acceptance/tests/example/example_test.go create mode 100644 test/acceptance/tests/example/main_test.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 73536d0278f3..b8d09c35e229 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -126,10 +126,11 @@ jobs: echo "Skipping acceptance tests for forked PRs; marking step successful." circleci step halt fi - primary_context=$(terraform output -state ../../terraform/gke/terraform.tfstate -json | jq -r .context_names.value[0]) - secondary_context=$(terraform output -state ../../terraform/gke/terraform.tfstate -json | jq -r .context_names.value[1]) + eval "$(echo export primary_kubeconfig=$(terraform output -state ../../terraform/gke/terraform.tfstate -json | jq -r .kubeconfigs.value[0]))" + eval "$(echo export secondary_kubeconfig=$(terraform output -state ../../terraform/gke/terraform.tfstate -json | jq -r .kubeconfigs.value[1]))" - gotestsum --junitfile /tmp/gotestsum-report.xml -- ./... -p 1 -timeout 20m -enable-multi-cluster -kubecontext=$primary_context -secondary-kubecontext=$secondary_context + gotestsum --junitfile "$TEST_RESULTS/gotestsum-report.xml" -- ./... -p 1 -timeout 20m -enable-multi-cluster \ + -kubeconfig="$primary_kubeconfig" -secondary-kubeconfig="$secondary_kubeconfig" - store_test_results: path: /tmp/test-results diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 79f7effb9acd..a01623ad52ef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,6 +33,10 @@ The acceptance tests require a Kubernetes cluster with a configured `kubectl`. ```bash brew install kubernetes-helm ``` +* [go](https://golang.org/) (v1.14+) + ```bash + brew install golang + ``` ### Helm 2/3 These tests will work with both Helm 2 and 3 if run through `bats`, e.g. `bats ./test/unit`. If copying the @@ -48,21 +52,64 @@ It's expected that the version of `helm` in your path is Helm 3. In our CI/CD the tests are run against both Helm 2 and Helm 3. +**Note:** Acceptance tests require Helm 3. + ### Running The Tests + +#### Unit Tests To run the unit tests: bats ./test/unit +#### Acceptance Tests + To run the acceptance tests: - bats ./test/acceptance + cd test/acceptance/tests + go test ./... -p 1 + +The above command will run all tests that can run against a single Kubernetes cluster, +using the current context set in your kubeconfig locally. + +**Note:** You must run all tests in serial by passing the `-p 1` flag +because the test suite currently does not support parallel execution. -If the acceptance tests fail, deployed resources in the Kubernetes cluster -may not be properly cleaned up. We recommend recycling the Kubernetes cluster to -start from a clean slate. +You can run other tests by enabling them by passing appropriate flags to `go test`. +For example, to run mesh gateway tests, which require two Kubernetes clusters, +you may use the following command: + + go test ./... -p 1 -timeout 20m \ + -enable-multi-cluster \ + -kubecontext= \ + -secondary-kubecontext= + +Below is the list of available flags: + +``` +-consul-image string + The Consul image to use for all tests. +-consul-k8s-image string + The consul-k8s image to use for all tests. +-enable-multi-cluster + If true, the tests that require multiple Kubernetes clusters will be run. At least one of -secondary-kubeconfig or -secondary-kubecontext is required when this flag is used. +-kubeconfig string + The path to a kubeconfig file. If this is blank, the default kubeconfig path (~/.kube/config) will be used. +-kubecontext string + The name of the Kubernetes context to use. If this is blank, the context set as the current context will be used by default. +-namespace string + The Kubernetes namespace to use for tests. (default "default") +-no-cleanup-on-failure + If true, the tests will not cleanup resources they create when they finish running.Note this flag must be run with -failfast flag, otherwise subsequent tests will fail. +-secondary-kubeconfig string + The path to a kubeconfig file of the secondary k8s cluster. If this is blank, the default kubeconfig path (~/.kube/config) will be used. +-secondary-kubecontext string + The name of the Kubernetes context for the secondary cluster to use. If this is blank, the context set as the current context will be used by default. +-secondary-namespace string + The Kubernetes namespace to use in the secondary k8s cluster. (default "default") +``` **Note:** There is a Terraform configuration in the -[`test/terraform/`](https://github.com/hashicorp/consul-helm/tree/master/test/terraform) directory +[`test/terraform/gke`](./test/terraform/gke) directory that can be used to quickly bring up a GKE cluster and configure `kubectl` and `helm` locally. This can be used to quickly spin up a test cluster for acceptance tests. Unit tests _do not_ require a running Kubernetes @@ -175,3 +222,152 @@ Here are some examples of common test patterns: } ``` Here we are using the `assert_empty` helper command that works with both Helm 2 and 3. + +### Writing Acceptance Tests + +If you are adding a feature that fits thematically with one of the existing test suites, +then you need to add your test cases to the existing test files. +Otherwise, you will need to create a new test suite. + +We recommend to start by either copying the [example test](test/acceptance/tests/example/example_test.go) +or the whole [example test suite](test/acceptance/tests/example), +depending on the test you need to add. + +#### Adding Test Suites + +To add a test suite, copy the [example test suite](test/acceptance/tests/example) +and uncomment the code you need in the [`main_test.go`](test/acceptance/tests/example/main_test.go) file. + +At a minimum, this file needs to contain the following: + +```go +package example + +import ( + "os" + "testing" + + "github.com/hashicorp/consul-helm/test/acceptance/framework" +) + +var suite framework.Suite + +func TestMain(m *testing.M) { + suite = framework.NewSuite(m) + os.Exit(suite.Run()) +} +``` + +If the test suite needs to run only when certain test flags are passed, +you need to handle that in the `TestMain` function. + +```go +func TestMain(m *testing.M) { + // First, create a new suite so that all flags are parsed. + suite = framework.NewSuite(m) + + // Run the suite only if our example feature test flag is set. + if suite.Config().EnableExampleFeature { + os.Exit(suite.Run()) + } else { + fmt.Println("Skipping example feature tests because -enable-example-feature is not set") + os.Exit(0) + } +} +``` + +#### Example Test + +We recommend using the [example test](test/acceptance/tests/example/example_test.go) +as a starting point for adding your tests. + +To write a test, you need access to the environment and context to run it against. +Each test belongs to a test **suite** that contains a test **environment** and test **configuration** created from flags passed to `go test`. +A test **environment** contains references to one or more test **contexts**, +which represents one Kubernetes cluster. + +```go +func TestExample(t *testing.T) { + // Get test configuration. + cfg := suite.Config() + + // Get the default context. + ctx := suite.Environment().DefaultContext(t) + + // Create Helm values for the Helm install. + helmValues := map[string]string{ + "exampleFeature.enabled": "true", + } + + // Generate a random name for this test. + releaseName := helpers.RandomName() + + // Create a new Consul cluster object. + consulCluster := framework.NewHelmCluster(t, helmValues, ctx, cfg, releaseName) + + // Create the Consul cluster with Helm. + consulCluster.Create(t) + + // Make test assertions. +} +``` + +Please see [mesh gateway tests](test/acceptance/tests/mesh-gateway/mesh_gateway_test.go) +for an example of how to use write a test that uses multiple contexts. + +#### Writing Assertions + +Depending on the test you're writing, you may need to write assertions +either by running `kubectl` commands, calling the Kubernetes API, or +the Consul API. + +To run `kubectl` commands, you need to get `KubectlOptions` from the test context. +There are a number of `kubectl` commands available in the `helpers/kubectl.go` file. +For example, to call `kubectl apply` from the test write the following: + +```go +helpers.KubectlApply(t, ctx.KubectlOptions(), filepath) +``` + +Similarly, you can obtain Kubernetes client from your test context. +You can use it to, for example, read all services in a namespace: + +```go +k8sClient := ctx.KubernetesClient(t) +services, err := k8sClient.CoreV1().Services(ctx.KubectlOptions().Namespace).List(metav1.ListOptions{}) +``` + +To make Consul API calls, you can get the Consul client from the `consulCluster` object, +indicating whether the client needs to be secure or not (i.e. whether TLS and ACLs are enabled on the Consul cluster): + +```go +consulClient := consulCluster.SetupConsulClient(t, true) +consulServices, _, err := consulClient.Catalog().Services(nil) +``` + +#### Cleaning Up Resources + +Because you may be creating resources that will not be destroyed automatically +when a test finishes, you need to make sure to clean them up. Most methods and objects +provided by the framework already do that, so you don't need to worry cleaning them up. +However, if your tests create Kubernetes objects, you need to clean them up yourself by +calling `helpers.Cleanup` function. + +**Note:** If you want to keep resources after a test run for debugging purposes, +you can run tests with `-no-cleanup-on-failure` flag. +You need to make sure to clean them up manually before running tests again. + +#### When to Add Acceptance Tests + +Sometimes adding an acceptance test for the feature you're writing may not be the right thing. +Here are some things to consider before adding a test: + +* Is this a test for a happy case scenario? + Generally, we expect acceptance tests to test happy case scenarios. If your test does not, + then perhaps it could be tested by either a unit test in this repository or a test in the + [consul-k8s](https://github.com/hashicorp/consul-k8s) repository. +* Is the test you're going to write for a feature that is scoped to one of the underlying componenets of this Helm chart, + either Consul itself or consul-k8s? In that case, it should be tested there rather than in the Helm chart. + For example, we don't expect acceptance tests to include all the permutations of the consul-k8s commands + and their respective flags. Something like that should be tested in the consul-k8s repository. + \ No newline at end of file diff --git a/test/acceptance/framework/environment.go b/test/acceptance/framework/environment.go index 7e34b32f635c..d27aed5c6cfd 100644 --- a/test/acceptance/framework/environment.go +++ b/test/acceptance/framework/environment.go @@ -87,6 +87,7 @@ func (k kubernetesContext) KubernetesClient(t *testing.T) kubernetes.Interface { configPath, err := k.KubectlOptions().GetConfigPath(t) require.NoError(t, err) + t.Logf("Creating client from config path at %s for context %s", configPath, k.contextName) config, err := k8s.LoadApiClientConfigE(configPath, k.contextName) require.NoError(t, err) diff --git a/test/acceptance/tests/example/example_test.go b/test/acceptance/tests/example/example_test.go new file mode 100644 index 000000000000..fa732ff0a359 --- /dev/null +++ b/test/acceptance/tests/example/example_test.go @@ -0,0 +1,58 @@ +package example + +import ( + "testing" + + "github.com/hashicorp/consul-helm/test/acceptance/framework" + "github.com/hashicorp/consul-helm/test/acceptance/helpers" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestExample(t *testing.T) { + // Get test configuration. + cfg := suite.Config() + + // Get the default context. + ctx := suite.Environment().DefaultContext(t) + + // Create Helm values for the Helm install. + helmValues := map[string]string{ + "exampleFeature.enabled": "true", + } + + // Generate a random name for this test. + releaseName := helpers.RandomName() + + // Create a new Consul cluster object. + consulCluster := framework.NewHelmCluster(t, helmValues, ctx, cfg, releaseName) + + // Create the Consul cluster with Helm. + consulCluster.Create(t) + + // Make test assertions. + + // To run kubectl commands, you need to get KubectlOptions from the test context. + // There are a number of kubectl commands available in the helpers/kubectl.go file. + // For example, to call 'kubectl apply' from the test write the following: + helpers.KubectlApply(t, ctx.KubectlOptions(), "path/to/config") + + // Clean up any Kubernetes resources you have created + helpers.Cleanup(t, cfg.NoCleanupOnFailure, func() { + helpers.KubectlDelete(t, ctx.KubectlOptions(), "path/to/config") + }) + + // Similarly, you can obtain Kubernetes client from your test context. + // You can use it to, for example, read all services in a namespace: + k8sClient := ctx.KubernetesClient(t) + services, err := k8sClient.CoreV1().Services(ctx.KubectlOptions().Namespace).List(metav1.ListOptions{}) + require.NoError(t, err) + require.NotNil(t, services.Items) + + // To make Consul API calls, you can get the Consul client from the consulCluster object, + // indicating whether the client needs to be secure or not (i.e. whether TLS and ACLs are enabled on the Consul cluster): + consulClient := consulCluster.SetupConsulClient(t, true) + consulServices, _, err := consulClient.Catalog().Services(nil) + require.NoError(t, err) + require.NotNil(t, consulServices) +} diff --git a/test/acceptance/tests/example/main_test.go b/test/acceptance/tests/example/main_test.go new file mode 100644 index 000000000000..b5814ece937f --- /dev/null +++ b/test/acceptance/tests/example/main_test.go @@ -0,0 +1,33 @@ +package example + +import ( + "testing" + + "github.com/hashicorp/consul-helm/test/acceptance/framework" +) + +var suite framework.Suite + +func TestMain(m *testing.M) { + // First, uncomment the line below to create a new suite so that all flags are parsed. + /* + suite = framework.NewSuite(m) + */ + + // If the test suite needs to run only when certain test flags are passed, + // you need to handle that in the TestMain function. + // Uncomment and modify example code below if that is the case. + /* + if suite.Config().EnableExampleFeature { + os.Exit(suite.Run()) + } else { + fmt.Println("Skipping example feature tests because -enable-example-feature is not set") + os.Exit(0) + } + */ + + // If the test suite should run in every case, uncomment the line below. + /* + os.Exit(suite.Run()) + */ +} diff --git a/test/terraform/gke/main.tf b/test/terraform/gke/main.tf index 2593785d3918..b40a24a918b7 100644 --- a/test/terraform/gke/main.tf +++ b/test/terraform/gke/main.tf @@ -3,12 +3,12 @@ provider "google" { } resource "random_id" "suffix" { - count = var.cluster_count + count = var.cluster_count byte_length = 4 } data "google_container_engine_versions" "main" { - location = var.zone + location = var.zone version_prefix = "1.15." } @@ -33,7 +33,7 @@ resource "null_resource" "kubectl" { # On creation, we want to setup the kubectl credentials. The easiest way # to do this is to shell out to gcloud. provisioner "local-exec" { - command = "gcloud container clusters get-credentials --zone=${var.zone} ${google_container_cluster.cluster[count.index].name}" + command = "KUBECONFIG=$HOME/.kube/${google_container_cluster.cluster[count.index].name} gcloud container clusters get-credentials --zone=${var.zone} ${google_container_cluster.cluster[count.index].name}" } # On destroy we want to try to clean up the kubectl credentials. This @@ -43,12 +43,6 @@ resource "null_resource" "kubectl" { provisioner "local-exec" { when = destroy on_failure = continue - command = "kubectl config get-clusters | grep ${google_container_cluster.cluster[count.index].name} | xargs -n1 kubectl config delete-cluster" - } - - provisioner "local-exec" { - when = destroy - on_failure = continue - command = "kubectl config get-contexts | grep ${google_container_cluster.cluster[count.index].name} | xargs -n1 kubectl config delete-context" + command = "rm $HOME/.kube/${google_container_cluster.cluster[count.index].name}" } } diff --git a/test/terraform/gke/outputs.tf b/test/terraform/gke/outputs.tf index 2a5d626384b4..95b68d0296b7 100644 --- a/test/terraform/gke/outputs.tf +++ b/test/terraform/gke/outputs.tf @@ -6,6 +6,6 @@ output "cluster_names" { value = google_container_cluster.cluster.*.name } -output "context_names" { - value = [for cl in google_container_cluster.cluster : format("gke_%s_%s_%s", var.project, var.zone, cl.name) ] +output "kubeconfigs" { + value = [for cl in google_container_cluster.cluster : format("$HOME/.kube/%s", cl.name)] } diff --git a/test/terraform/gke/variables.tf b/test/terraform/gke/variables.tf index 393daf7574e1..e0025920ceec 100644 --- a/test/terraform/gke/variables.tf +++ b/test/terraform/gke/variables.tf @@ -17,6 +17,6 @@ variable "init_cli" { } variable "cluster_count" { - default = 1 + default = 1 description = "The number of Kubernetes clusters to create." -} \ No newline at end of file +}