Acceptance Tests for Cluster Peering (#1287)

* Add acceptance tests that run connect tests for peering.
- Does not support TLS, ACLs and T-Proxy.
* Add OSS acceptance test

.circleci/config.yml

@@ -646,7 +646,7 @@ jobs:
       - run: mkdir -p $TEST_RESULTS
 
       - run-acceptance-tests:
-          additional-flags: -kubeconfig="$primary_kubeconfig" -secondary-kubeconfig="$secondary_kubeconfig" -enable-pod-security-policies -enable-transparent-proxy
+          additional-flags: -kubeconfig="$primary_kubeconfig" -secondary-kubeconfig="$secondary_kubeconfig" -enable-pod-security-policies
 
       - store_test_results:
           path: /tmp/test-results
@@ -701,7 +701,7 @@ jobs:
       - run: mkdir -p $TEST_RESULTS
 
       - run-acceptance-tests:
-          additional-flags: -kubeconfig="$primary_kubeconfig" -secondary-kubeconfig="$secondary_kubeconfig" -enable-transparent-proxy
+          additional-flags: -kubeconfig="$primary_kubeconfig" -secondary-kubeconfig="$secondary_kubeconfig"
 
       - store_test_results:
           path: /tmp/test-results
@@ -762,7 +762,7 @@ jobs:
       - run: mkdir -p $TEST_RESULTS
 
       - run-acceptance-tests:
-          additional-flags: -kubeconfig="$primary_kubeconfig" -secondary-kubeconfig="$secondary_kubeconfig" -enable-transparent-proxy
+          additional-flags: -kubeconfig="$primary_kubeconfig" -secondary-kubeconfig="$secondary_kubeconfig"
 
       - store_test_results:
           path: /tmp/test-results
@@ -857,7 +857,7 @@ jobs:
             - ~/.go_workspace/pkg/mod
       - run: mkdir -p $TEST_RESULTS
       - run-acceptance-tests:
-          additional-flags: -use-kind -kubecontext="kind-dc1" -secondary-kubecontext="kind-dc2" -enable-transparent-proxy
+          additional-flags: -use-kind -kubecontext="kind-dc1" -secondary-kubecontext="kind-dc2"
       - store_test_results:
           path: /tmp/test-results
       - store_artifacts:

acceptance/framework/helpers/helpers.go

@@ -12,10 +12,9 @@ import (
 	"time"
 
 	"github.com/gruntwork-io/terratest/modules/helm"
-	"github.com/hashicorp/consul/api"
-
 	"github.com/gruntwork-io/terratest/modules/random"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
+	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

acceptance/framework/k8s/helpers.go

@@ -125,3 +125,17 @@ func ServiceHost(t *testing.T, cfg *config.TestConfig, ctx environment.TestConte
 		return host
 	}
 }
+
+// CopySecret copies a Kubernetes secret from one cluster to another.
+func CopySecret(t *testing.T, sourceContext, destContext environment.TestContext, secretName string) {
+	t.Helper()
+	var secret *corev1.Secret
+	var err error
+	retry.Run(t, func(r *retry.R) {
+		secret, err = sourceContext.KubernetesClient(t).CoreV1().Secrets(sourceContext.KubectlOptions(t).Namespace).Get(context.Background(), secretName, metav1.GetOptions{})
+		secret.ResourceVersion = ""
+		require.NoError(r, err)
+	})
+	_, err = destContext.KubernetesClient(t).CoreV1().Secrets(destContext.KubectlOptions(t).Namespace).Create(context.Background(), secret, metav1.CreateOptions{})
+	require.NoError(t, err)
+}

acceptance/tests/fixtures/bases/peering/peering-acceptor.yaml

@@ -0,0 +1,10 @@
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: PeeringAcceptor
+metadata:
+  name: server
+spec:
+  peer:
+    secret:
+      name: "api-token"
+      key: "data"
+      backend: "kubernetes"

acceptance/tests/fixtures/bases/peering/peering-dialer.yaml

@@ -0,0 +1,10 @@
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: PeeringDialer
+metadata:
+  name: client
+spec:
+  peer:
+    secret:
+      name: "api-token"
+      key: "data"
+      backend: "kubernetes"

acceptance/tests/fixtures/cases/crd-peers/default-namespace/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+  - ../../../bases/exportedservices-default
+
+patchesStrategicMerge:
+- patch.yaml

acceptance/tests/fixtures/cases/crd-peers/default-namespace/patch.yaml

@@ -0,0 +1,10 @@
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: ExportedServices
+metadata:
+  name: default
+spec:
+  services:
+  - name: static-server
+    namespace: default
+    consumers:
+    - peer: client

acceptance/tests/fixtures/cases/crd-peers/default/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+  - ../../../bases/exportedservices-default
+
+patchesStrategicMerge:
+- patch.yaml

acceptance/tests/fixtures/cases/crd-peers/default/patch.yaml

@@ -0,0 +1,9 @@
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: ExportedServices
+metadata:
+  name: default
+spec:
+  services:
+  - name: static-server
+    consumers:
+    - peer: client

acceptance/tests/fixtures/cases/crd-peers/non-default-namespace/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+  - ../../../bases/exportedservices-default
+
+patchesStrategicMerge:
+- patch.yaml

acceptance/tests/fixtures/cases/crd-peers/non-default-namespace/patch.yaml

@@ -0,0 +1,10 @@
+apiVersion: consul.hashicorp.com/v1alpha1
+kind: ExportedServices
+metadata:
+  name: default
+spec:
+  services:
+  - name: static-server
+    namespace: ns1
+    consumers:
+    - peer: client

acceptance/tests/fixtures/cases/static-client-peers/default-namespace/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+  - ../../../bases/static-client
+
+patchesStrategicMerge:
+  - patch.yaml

acceptance/tests/fixtures/cases/static-client-peers/default-namespace/patch.yaml

@@ -0,0 +1,10 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: static-client
+spec:
+  template:
+    metadata:
+      annotations:
+        "consul.hashicorp.com/connect-inject": "true"
+        "consul.hashicorp.com/connect-service-upstreams": "static-server.svc.default.ns.server.peer:1234"

acceptance/tests/fixtures/cases/static-client-peers/default/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+  - ../../../bases/static-client
+
+patchesStrategicMerge:
+  - patch.yaml

acceptance/tests/fixtures/cases/static-client-peers/default/patch.yaml

@@ -0,0 +1,10 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: static-client
+spec:
+  template:
+    metadata:
+      annotations:
+        "consul.hashicorp.com/connect-inject": "true"
+        "consul.hashicorp.com/connect-service-upstreams": "static-server.svc.server.peer:1234"

acceptance/tests/fixtures/cases/static-client-peers/non-default-namespace/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+  - ../../../bases/static-client
+
+patchesStrategicMerge:
+  - patch.yaml

acceptance/tests/fixtures/cases/static-client-peers/non-default-namespace/patch.yaml

@@ -0,0 +1,10 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: static-client
+spec:
+  template:
+    metadata:
+      annotations:
+        "consul.hashicorp.com/connect-inject": "true"
+        "consul.hashicorp.com/connect-service-upstreams": "static-server.svc.ns1.ns.server.peer:1234"

acceptance/tests/partitions/partitions_connect_test.go

@@ -138,19 +138,19 @@ func TestPartitions_Connect(t *testing.T) {
 			caKeySecretName := fmt.Sprintf("%s-consul-ca-key", releaseName)
 
 			logger.Logf(t, "retrieving ca cert secret %s from the server cluster and applying to the client cluster", caCertSecretName)
-			copySecret(t, serverClusterContext, clientClusterContext, caCertSecretName)
+			k8s.CopySecret(t, serverClusterContext, clientClusterContext, caCertSecretName)
 
 			if !c.ACLsAndAutoEncryptEnabled {
 				// When auto-encrypt is disabled, we need both
 				// the CA cert and CA key to be available in the clients cluster to generate client certificates and keys.
 				logger.Logf(t, "retrieving ca key secret %s from the server cluster and applying to the client cluster", caKeySecretName)
-				copySecret(t, serverClusterContext, clientClusterContext, caKeySecretName)
+				k8s.CopySecret(t, serverClusterContext, clientClusterContext, caKeySecretName)
 			}
 
 			partitionToken := fmt.Sprintf("%s-consul-partitions-acl-token", releaseName)
 			if c.ACLsAndAutoEncryptEnabled {
 				logger.Logf(t, "retrieving partition token secret %s from the server cluster and applying to the client cluster", partitionToken)
-				copySecret(t, serverClusterContext, clientClusterContext, partitionToken)
+				k8s.CopySecret(t, serverClusterContext, clientClusterContext, partitionToken)
 			}
 
 			partitionServiceName := fmt.Sprintf("%s-consul-partition", releaseName)
@@ -629,13 +629,3 @@ func TestPartitions_Connect(t *testing.T) {
 		})
 	}
 }
-
-func copySecret(t *testing.T, sourceContext, destContext environment.TestContext, secretName string) {
-	t.Helper()
-
-	secret, err := sourceContext.KubernetesClient(t).CoreV1().Secrets(sourceContext.KubectlOptions(t).Namespace).Get(context.Background(), secretName, metav1.GetOptions{})
-	secret.ResourceVersion = ""
-	require.NoError(t, err)
-	_, err = destContext.KubernetesClient(t).CoreV1().Secrets(destContext.KubectlOptions(t).Namespace).Create(context.Background(), secret, metav1.CreateOptions{})
-	require.NoError(t, err)
-}

acceptance/tests/partitions/partitions_sync_test.go

@@ -128,19 +128,19 @@ func TestPartitions_Sync(t *testing.T) {
 			caKeySecretName := fmt.Sprintf("%s-consul-ca-key", releaseName)
 
 			logger.Logf(t, "retrieving ca cert secret %s from the server cluster and applying to the client cluster", caCertSecretName)
-			copySecret(t, primaryClusterContext, secondaryClusterContext, caCertSecretName)
+			k8s.CopySecret(t, primaryClusterContext, secondaryClusterContext, caCertSecretName)
 
 			if !c.ACLsAndAutoEncryptEnabled {
 				// When auto-encrypt is disabled, we need both
 				// the CA cert and CA key to be available in the clients cluster to generate client certificates and keys.
 				logger.Logf(t, "retrieving ca key secret %s from the server cluster and applying to the client cluster", caKeySecretName)
-				copySecret(t, primaryClusterContext, secondaryClusterContext, caKeySecretName)
+				k8s.CopySecret(t, primaryClusterContext, secondaryClusterContext, caKeySecretName)
 			}
 
 			partitionToken := fmt.Sprintf("%s-consul-partitions-acl-token", releaseName)
 			if c.ACLsAndAutoEncryptEnabled {
 				logger.Logf(t, "retrieving partition token secret %s from the server cluster and applying to the client cluster", partitionToken)
-				copySecret(t, primaryClusterContext, secondaryClusterContext, partitionToken)
+				k8s.CopySecret(t, primaryClusterContext, secondaryClusterContext, partitionToken)
 			}
 
 			partitionServiceName := fmt.Sprintf("%s-consul-partition", releaseName)

acceptance/tests/peering/main_test.go

@@ -0,0 +1,22 @@
+package peering
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	testsuite "github.com/hashicorp/consul-k8s/acceptance/framework/suite"
+)
+
+var suite testsuite.Suite
+
+func TestMain(m *testing.M) {
+	suite = testsuite.NewSuite(m)
+
+	if suite.Config().EnableMultiCluster {
+		os.Exit(suite.Run())
+	} else {
+		fmt.Println("Skipping peering tests because -enable-multi-cluster is not set")
+		os.Exit(0)
+	}
+}