From 78a3569f61b99cb7337779269c1e820da380dc6f Mon Sep 17 00:00:00 2001 From: Oliver Walsh Date: Thu, 22 Feb 2024 00:22:53 +0000 Subject: [PATCH] functional tests --- tests/functional/base_test.go | 28 +++ .../ovndbcluster_controller_test.go | 218 ++++++++++++++++++ tests/functional/ovnnorthd_controller_test.go | 199 ++++++++++++++++ tests/functional/suite_test.go | 5 + 4 files changed, 450 insertions(+) diff --git a/tests/functional/base_test.go b/tests/functional/base_test.go index 04336727..5529763b 100644 --- a/tests/functional/base_test.go +++ b/tests/functional/base_test.go @@ -25,11 +25,13 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" infranetworkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/ovn-operator/api/v1beta1" ovnv1 "github.com/openstack-k8s-operators/ovn-operator/api/v1beta1" ) @@ -47,6 +49,19 @@ func GetDefaultOVNNorthdSpec() ovnv1.OVNNorthdSpec { } } +func GetTLSOVNNorthdSpec() ovnv1.OVNNorthdSpec { + spec := GetDefaultOVNNorthdSpec() + spec.TLS = tls.SimpleService{ + Ca: tls.Ca{ + CaBundleSecretName: CABundleSecretName, + }, + GenericService: tls.GenericService{ + SecretName: ptr.To(OvnDbCertSecretName), + }, + } + return spec +} + func CreateOVNNorthd(namespace string, OVNNorthdName string, spec ovnv1.OVNNorthdSpec) client.Object { name := ovn.CreateOVNNorthd(namespace, spec) return ovn.GetOVNNorthd(name) @@ -72,6 +87,19 @@ func GetDefaultOVNDBClusterSpec() ovnv1.OVNDBClusterSpec { } } +func GetTLSOVNDBClusterSpec() ovnv1.OVNDBClusterSpec { + spec := GetDefaultOVNDBClusterSpec() + spec.TLS = tls.SimpleService{ + Ca: tls.Ca{ + CaBundleSecretName: CABundleSecretName, + }, + GenericService: tls.GenericService{ + SecretName: ptr.To(OvnDbCertSecretName), + }, + } + return spec +} + func CreateOVNDBCluster(namespace string, spec ovnv1.OVNDBClusterSpec) client.Object { name := ovn.CreateOVNDBCluster(namespace, spec) return ovn.GetOVNDBCluster(name) diff --git a/tests/functional/ovndbcluster_controller_test.go b/tests/functional/ovndbcluster_controller_test.go index e046d0f6..d0aa01c3 100644 --- a/tests/functional/ovndbcluster_controller_test.go +++ b/tests/functional/ovndbcluster_controller_test.go @@ -349,4 +349,222 @@ var _ = Describe("OVNDBCluster controller", func() { ) }) }) + + When("OVNDBCluster is created with TLS", func() { + var OVNDBClusterName types.NamespacedName + BeforeEach(func() { + spec := GetTLSOVNDBClusterSpec() + spec.NetworkAttachment = "internalapi" + spec.DBType = v1beta1.SBDBType + instance := CreateOVNDBCluster(namespace, spec) + OVNDBClusterName = types.NamespacedName{Name: instance.GetName(), Namespace: instance.GetNamespace()} + DeferCleanup(th.DeleteInstance, instance) + internalAPINADName := types.NamespacedName{Namespace: namespace, Name: "internalapi"} + nad := th.CreateNetworkAttachmentDefinition(internalAPINADName) + DeferCleanup(th.DeleteInstance, nad) + }) + + It("reports that the CA secret is missing", func() { + th.ExpectConditionWithDetails( + OVNDBClusterName, + ConditionGetterFunc(OVNDBClusterConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + fmt.Sprintf( + "TLSInput error occured in TLS sources Secret %s/combined-ca-bundle not found", + namespace, + ), + ) + th.ExpectCondition( + OVNDBClusterName, + ConditionGetterFunc(OVNDBClusterConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("reports that the cert secret is missing", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + th.ExpectConditionWithDetails( + OVNDBClusterName, + ConditionGetterFunc(OVNDBClusterConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + fmt.Sprintf( + "TLSInput error occured in TLS sources Secret %s/%s not found", + namespace, OvnDbCertSecretName, + ), + ) + th.ExpectCondition( + OVNDBClusterName, + ConditionGetterFunc(OVNDBClusterConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("creates a Statefulset with TLS certs attached", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + + statefulSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovsdbserver-sb", + } + th.SimulateStatefulSetReplicaReadyWithPods(statefulSetName, + map[string][]string{namespace + "/internalapi": {"10.0.0.1"}}, + ) + + ss := th.GetStatefulSet(statefulSetName) + + // check TLS volumes + th.AssertVolumeExists(CABundleSecretName, ss.Spec.Template.Spec.Volumes) + th.AssertVolumeExists("ovsdbserver-sb-tls-certs", ss.Spec.Template.Spec.Volumes) + + svcC := ss.Spec.Template.Spec.Containers[0] + + // check TLS volume mounts + th.AssertVolumeMountExists(CABundleSecretName, "tls-ca-bundle.pem", svcC.VolumeMounts) + th.AssertVolumeMountExists("ovsdbserver-sb-tls-certs", "tls.key", svcC.VolumeMounts) + th.AssertVolumeMountExists("ovsdbserver-sb-tls-certs", "tls.crt", svcC.VolumeMounts) + th.AssertVolumeMountExists("ovsdbserver-sb-tls-certs", "ca.crt", svcC.VolumeMounts) + + // check DB url schema + Eventually(func(g Gomega) { + OVNDBCluster := GetOVNDBCluster(OVNDBClusterName) + g.Expect(OVNDBCluster.Status.DBAddress).To(HavePrefix("ssl:")) + g.Expect(OVNDBCluster.Status.InternalDBAddress).To(HavePrefix("ssl:")) + g.Expect(OVNDBCluster.Status.RaftAddress).To(HavePrefix("ssl:")) + }, timeout, interval).Should(Succeed()) + + // check scripts configure TLS + scriptsCM := types.NamespacedName{ + Namespace: OVNDBClusterName.Namespace, + Name: fmt.Sprintf("%s-%s", OVNDBClusterName.Name, "scripts"), + } + Eventually(func() corev1.ConfigMap { + return *th.GetConfigMap(scriptsCM) + }, timeout, interval).ShouldNot(BeNil()) + + Expect(th.GetConfigMap(scriptsCM).Data["settings.sh"]).Should( + ContainSubstring("DB_SCHEME=\"pssl\"")) + Expect(th.GetConfigMap(scriptsCM).Data["setup.sh"]).Should(And( + ContainSubstring("-db-ssl-key="), + ContainSubstring("-db-ssl-cert="), + ContainSubstring("-db-ssl-ca-cert="), + ContainSubstring("-cluster-remote-proto=ssl"), + )) + + th.ExpectCondition( + OVNDBClusterName, + ConditionGetterFunc(OVNDBClusterConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + }) + + It("reconfigures the pods when CA bundle changes", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + + statefulSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovsdbserver-sb", + } + th.SimulateStatefulSetReplicaReadyWithPods(statefulSetName, + map[string][]string{namespace + "/internalapi": {"10.0.0.1"}}, + ) + + originalHash := GetEnvVarValue( + th.GetStatefulSet(statefulSetName).Spec.Template.Spec.Containers[0].Env, + "CONFIG_HASH", + "", + ) + Expect(originalHash).NotTo(BeEmpty()) + + // Change the content of the CA secret + th.UpdateSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + }, + "tls-ca-bundle.pem", + []byte("DifferentCAData"), + ) + + // Assert that the pod is updated + Eventually(func(g Gomega) { + newHash := GetEnvVarValue( + th.GetStatefulSet(statefulSetName).Spec.Template.Spec.Containers[0].Env, + "CONFIG_HASH", + "", + ) + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(originalHash)) + }, timeout, interval).Should(Succeed()) + }) + + It("reconfigures the pods when cert changes", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + + statefulSetName := types.NamespacedName{ + Namespace: namespace, + Name: "ovsdbserver-sb", + } + th.SimulateStatefulSetReplicaReadyWithPods(statefulSetName, + map[string][]string{namespace + "/internalapi": {"10.0.0.1"}}, + ) + + originalHash := GetEnvVarValue( + th.GetStatefulSet(statefulSetName).Spec.Template.Spec.Containers[0].Env, + "CONFIG_HASH", + "", + ) + Expect(originalHash).NotTo(BeEmpty()) + + // Change the content of the cert secret + th.UpdateSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + }, + "tls.crt", + []byte("DifferentCrtData"), + ) + + // Assert that the pod is updated + Eventually(func(g Gomega) { + newHash := GetEnvVarValue( + th.GetStatefulSet(statefulSetName).Spec.Template.Spec.Containers[0].Env, + "CONFIG_HASH", + "", + ) + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(originalHash)) + }, timeout, interval).Should(Succeed()) + }) + + }) }) diff --git a/tests/functional/ovnnorthd_controller_test.go b/tests/functional/ovnnorthd_controller_test.go index 55379e2f..65771962 100644 --- a/tests/functional/ovnnorthd_controller_test.go +++ b/tests/functional/ovnnorthd_controller_test.go @@ -18,6 +18,7 @@ package functional_test import ( "encoding/json" + "fmt" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" . "github.com/onsi/ginkgo/v2" @@ -257,4 +258,202 @@ var _ = Describe("OVNNorthd controller", func() { }) }) + When("OVNNorthd is created with TLS", func() { + var ovnNorthdName types.NamespacedName + + BeforeEach(func() { + dbs := CreateOVNDBClusters(namespace, map[string][]string{}, 1) + DeferCleanup(DeleteOVNDBClusters, dbs) + spec := GetTLSOVNNorthdSpec() + spec.NetworkAttachment = "internalapi" + ovnNorthdName = ovn.CreateOVNNorthd(namespace, spec) + DeferCleanup(ovn.DeleteOVNNorthd, ovnNorthdName) + internalAPINADName := types.NamespacedName{Namespace: namespace, Name: "internalapi"} + nad := th.CreateNetworkAttachmentDefinition(internalAPINADName) + DeferCleanup(th.DeleteInstance, nad) + }) + + It("reports that the CA secret is missing", func() { + th.ExpectConditionWithDetails( + ovnNorthdName, + ConditionGetterFunc(OVNNorthdConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + fmt.Sprintf( + "TLSInput error occured in TLS sources Secret %s/combined-ca-bundle not found", + namespace, + ), + ) + th.ExpectCondition( + ovnNorthdName, + ConditionGetterFunc(OVNNorthdConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("reports that the cert secret is missing", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + th.ExpectConditionWithDetails( + ovnNorthdName, + ConditionGetterFunc(OVNNorthdConditionGetter), + condition.TLSInputReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + fmt.Sprintf( + "TLSInput error occured in TLS sources Secret %s/%s not found", + namespace, OvnDbCertSecretName, + ), + ) + th.ExpectCondition( + ovnNorthdName, + ConditionGetterFunc(OVNNorthdConditionGetter), + condition.ReadyCondition, + corev1.ConditionFalse, + ) + }) + + It("creates a Deployment with TLS certs attached", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + + deploymentName := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-northd", + } + th.SimulateDeploymentReadyWithPods( + deploymentName, map[string][]string{namespace + "/internalapi": {"10.0.0.1"}}, + ) + + d := th.GetDeployment(deploymentName) + + // check TLS volumes + th.AssertVolumeExists(CABundleSecretName, d.Spec.Template.Spec.Volumes) + th.AssertVolumeExists("ovn-northd-tls-certs", d.Spec.Template.Spec.Volumes) + + svcC := d.Spec.Template.Spec.Containers[0] + + // check TLS volume mounts + th.AssertVolumeMountExists(CABundleSecretName, "tls-ca-bundle.pem", svcC.VolumeMounts) + th.AssertVolumeMountExists("ovn-northd-tls-certs", "tls.key", svcC.VolumeMounts) + th.AssertVolumeMountExists("ovn-northd-tls-certs", "tls.crt", svcC.VolumeMounts) + th.AssertVolumeMountExists("ovn-northd-tls-certs", "ca.crt", svcC.VolumeMounts) + + // check cli args + Expect(svcC.Args).To(And( + ContainElement(ContainSubstring("--private-key=")), + ContainElement(ContainSubstring("--certificate=")), + ContainElement(ContainSubstring("--ca-cert=")), + )) + + th.ExpectCondition( + ovnNorthdName, + ConditionGetterFunc(OVNNorthdConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + }) + + It("reconfigures the pods when CA bundle changes", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + + deploymentName := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-northd", + } + th.SimulateDeploymentReadyWithPods( + deploymentName, map[string][]string{namespace + "/internalapi": {"10.0.0.1"}}, + ) + + originalHash := GetEnvVarValue( + th.GetDeployment(deploymentName).Spec.Template.Spec.Containers[0].Env, + "tls-ca-bundle.pem", + "", + ) + Expect(originalHash).NotTo(BeEmpty()) + + // Change the content of the CA secret + th.UpdateSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + }, + "tls-ca-bundle.pem", + []byte("DifferentCAData"), + ) + + // Assert that the pod is updated + Eventually(func(g Gomega) { + newHash := GetEnvVarValue( + th.GetDeployment(deploymentName).Spec.Template.Spec.Containers[0].Env, + "tls-ca-bundle.pem", + "", + ) + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(originalHash)) + }, timeout, interval).Should(Succeed()) + }) + + It("reconfigures the pods when cert changes", func() { + DeferCleanup(k8sClient.Delete, ctx, th.CreateCABundleSecret(types.NamespacedName{ + Name: CABundleSecretName, + Namespace: namespace, + })) + DeferCleanup(k8sClient.Delete, ctx, th.CreateCertSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + })) + + deploymentName := types.NamespacedName{ + Namespace: namespace, + Name: "ovn-northd", + } + th.SimulateDeploymentReadyWithPods( + deploymentName, map[string][]string{namespace + "/internalapi": {"10.0.0.1"}}, + ) + + originalHash := GetEnvVarValue( + th.GetDeployment(deploymentName).Spec.Template.Spec.Containers[0].Env, + "certs", + "", + ) + Expect(originalHash).NotTo(BeEmpty()) + + // Change the content of the cert secret + th.UpdateSecret(types.NamespacedName{ + Name: OvnDbCertSecretName, + Namespace: namespace, + }, + "tls.crt", + []byte("DifferentCrtData"), + ) + + // Assert that the pod is updated + Eventually(func(g Gomega) { + newHash := GetEnvVarValue( + th.GetDeployment(deploymentName).Spec.Template.Spec.Containers[0].Env, + "certs", + "", + ) + g.Expect(newHash).NotTo(BeEmpty()) + g.Expect(newHash).NotTo(Equal(originalHash)) + }, timeout, interval).Should(Succeed()) + }) + }) }) diff --git a/tests/functional/suite_test.go b/tests/functional/suite_test.go index 92c3e442..73f78b20 100644 --- a/tests/functional/suite_test.go +++ b/tests/functional/suite_test.go @@ -71,6 +71,11 @@ var ( namespace string ) +const ( + CABundleSecretName = "combined-ca-bundle" + OvnDbCertSecretName = "ovndb-tls-cert" +) + func TestAPIs(t *testing.T) { RegisterFailHandler(Fail)