[tlse] TLS database connection

moves requesting the DB before rendering the service configuration. The my.cnf file gets added to the secret holding the service configs. The content of my.cnf is centrally managed in the mariadb-operator and retrieved calling db.GetDatabaseClientConfig(tlsCfg) Depends-On: openstack-k8s-operators/mariadb-operator#190 Depends-On: openstack-k8s-operators/mariadb-operator#191 Jira: OSPRH-4547
gibizer · Feb 26, 2024 · c3d2988 · c3d2988
1 parent 278c02b
commit c3d2988
Show file tree

Hide file tree

Showing 7 changed files with 124 additions and 52 deletions.
diff --git a/controllers/placementapi_controller.go b/controllers/placementapi_controller.go
@@ -349,7 +349,14 @@ func (r *PlacementAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	// all our input checks out so report InputReady
 	instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage)
 
-	err = r.generateServiceConfigMaps(ctx, h, instance, secret, &configMapVars)
+	db, result, err := r.ensureDB(ctx, h, instance)
+	if err != nil {
+		return ctrl.Result{}, err
+	} else if (result != ctrl.Result{}) {
+		return result, nil
+	}
+
+	err = r.generateServiceConfigMaps(ctx, h, instance, secret, &configMapVars, db)
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.ServiceConfigReadyCondition,
@@ -425,11 +432,6 @@ func (r *PlacementAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return result, err
 	}
 
-	result, err = r.ensureDB(ctx, h, instance)
-	if err != nil {
-		return ctrl.Result{}, err
-	}
-
 	apiEndpoints, result, err := r.ensureServiceExposed(ctx, h, instance)
 
 	if (err != nil || result != ctrl.Result{}) {
@@ -971,7 +973,7 @@ func (r *PlacementAPIReconciler) ensureDB(
 	ctx context.Context,
 	h *helper.Helper,
 	instance *placementv1.PlacementAPI,
-) (ctrl.Result, error) {
+) (*mariadbv1.Database, ctrl.Result, error) {
 	// (ksambor) should we use  NewDatabaseWithNamespace instead?
 	db := mariadbv1.NewDatabaseWithNamespace(
 		placement.DatabaseName,
@@ -996,15 +998,15 @@ func (r *PlacementAPIReconciler) ensureDB(
 			condition.SeverityWarning,
 			condition.DBReadyErrorMessage,
 			err.Error()))
-		return ctrl.Result{}, err
+		return db, ctrl.Result{}, err
 	}
 	if (ctrlResult != ctrl.Result{}) {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.DBReadyCondition,
 			condition.RequestedReason,
 			condition.SeverityInfo,
 			condition.DBReadyRunningMessage))
-		return ctrlResult, nil
+		return db, ctrlResult, nil
 	}
 	// wait for the DB to be setup
 	// (ksambor) should we use WaitForDBCreatedWithTimeout instead?
@@ -1016,21 +1018,21 @@ func (r *PlacementAPIReconciler) ensureDB(
 			condition.SeverityWarning,
 			condition.DBReadyErrorMessage,
 			err.Error()))
-		return ctrlResult, err
+		return db, ctrlResult, err
 	}
 	if (ctrlResult != ctrl.Result{}) {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.DBReadyCondition,
 			condition.RequestedReason,
 			condition.SeverityInfo,
 			condition.DBReadyRunningMessage))
-		return ctrlResult, nil
+		return db, ctrlResult, nil
 	}
 
 	// update Status.DatabaseHostname, used to config the service
 	instance.Status.DatabaseHostname = db.GetDatabaseHostname()
 	instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage)
-	return ctrlResult, nil
+	return db, ctrlResult, nil
 }
 
 func (r *PlacementAPIReconciler) ensureDbSync(
@@ -1174,6 +1176,7 @@ func (r *PlacementAPIReconciler) generateServiceConfigMaps(
 	instance *placementv1.PlacementAPI,
 	ospSecret corev1.Secret,
 	envVars *map[string]env.Setter,
+	db *mariadbv1.Database,
 ) error {
 	//
 	// create Secret required for placement input
@@ -1184,10 +1187,19 @@ func (r *PlacementAPIReconciler) generateServiceConfigMaps(
 
 	cmLabels := labels.GetLabels(instance, labels.GetGroupLabel(placement.ServiceName), map[string]string{})
 
+	var tlsCfg *tls.Service
+	if instance.Spec.TLS.Ca.CaBundleSecretName != "" {
+		tlsCfg = &tls.Service{}
+	}
+
 	// customData hold any customization for the service.
 	// custom.conf is going to /etc/<service>/<service>.conf.d
+	// my.cnf is going to /etc/my.cnf
 	// all other files get placed into /etc/<service> to allow overwrite of e.g. policy.json
-	customData := map[string]string{common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig}
+	customData := map[string]string{
+		common.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig,
+		"my.cnf":                           db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
+	}
 	for key, data := range instance.Spec.DefaultConfigOverwrite {
 		customData[key] = data
 	}
@@ -1209,11 +1221,13 @@ func (r *PlacementAPIReconciler) generateServiceConfigMaps(
 		"KeystoneInternalURL": keystoneInternalURL,
 		"KeystonePublicURL":   keystonePublicURL,
 		"PlacementPassword":   string(ospSecret.Data[instance.Spec.PasswordSelectors.Service]),
-		"DBUser":              instance.Spec.DatabaseUser,
-		"DBPassword":          string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]),
-		"DBAddress":           instance.Status.DatabaseHostname,
-		"DBName":              placement.DatabaseName,
 		"log_file":            "/var/log/placement/placement-api.log",
+		"DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf",
+			instance.Spec.DatabaseUser,
+			string(ospSecret.Data[instance.Spec.PasswordSelectors.Database]),
+			instance.Status.DatabaseHostname,
+			placement.DatabaseName,
+		),
 	}
 
 	// create httpd  vhost template parameters

diff --git a/go.mod b/go.mod
@@ -11,7 +11,7 @@ require (
 	github.com/openstack-k8s-operators/keystone-operator/api v0.3.1-0.20240219094943-9bbb46c9afba
 	github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240216173409-86913e6d5885
 	github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240216173409-86913e6d5885
-	github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240219072536-62f6b4dc7798
+	github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240220132409-f96d4d040f4e
 	github.com/openstack-k8s-operators/placement-operator/api v0.3.1-0.20240216174613-3d349f26e681
 	go.uber.org/zap v1.26.0
 	k8s.io/api v0.28.6

diff --git a/go.sum b/go.sum
@@ -99,8 +99,8 @@ github.com/openstack-k8s-operators/lib-common/modules/openstack v0.3.1-0.2024021
 github.com/openstack-k8s-operators/lib-common/modules/openstack v0.3.1-0.20240216173409-86913e6d5885/go.mod h1:8QsCFttAm+X6A8I8EQThGjNjeMAYt2hK7ivbvnR3434=
 github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240216173409-86913e6d5885 h1:ioJ2MO3vAcBkLM+0UBu5IuKW/DPXcyiNSOLq0Xvn+Nw=
 github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240216173409-86913e6d5885/go.mod h1:82nzS+DbBe1tzaMvNHH8FctmZzQ14ZAJysFGsMJiivo=
-github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240219072536-62f6b4dc7798 h1:zL4DdQ5HPXCLHeRMAWC2zI7ypbkZVYg3UkyEFSnzeow=
-github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240219072536-62f6b4dc7798/go.mod h1:PDqfLbP4ZWqQHAu1OtbjfpOGQUKSzLqRJChvE/9pcyQ=
+github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240220132409-f96d4d040f4e h1:6vqp5HZwcGvPH0MII/23iCd97T3/1HJZlONKW6LyNio=
+github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240220132409-f96d4d040f4e/go.mod h1:PDqfLbP4ZWqQHAu1OtbjfpOGQUKSzLqRJChvE/9pcyQ=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

diff --git a/templates/placementapi/config/placement-api-config.json b/templates/placementapi/config/placement-api-config.json
@@ -47,6 +47,12 @@
             "owner": "placement",
             "perm": "0600",
             "optional": true
+        },
+        {
+            "source": "/var/lib/openstack/config/my.cnf",
+            "dest": "/etc/my.cnf",
+            "owner": "placement",
+            "perm": "0644"
         }
     ],
     "permissions": [

diff --git a/templates/placementapi/config/placement-dbsync-config.json b/templates/placementapi/config/placement-dbsync-config.json
@@ -12,6 +12,12 @@
             "dest": "/etc/placement/placement.conf.d/custom.conf",
             "owner": "placement",
             "perm": "0600"
+        },
+        {
+            "source": "/var/lib/openstack/config/my.cnf",
+            "dest": "/etc/my.cnf",
+            "owner": "placement",
+            "perm": "0644"
         }
     ]
 }
diff --git a/templates/placementapi/config/placement.conf b/templates/placementapi/config/placement.conf
@@ -9,7 +9,7 @@ log_file = {{ .log_file }}
 debug = true
 
 [placement_database]
-connection = mysql+pymysql://{{ .DBUser }}:{{ .DBPassword }}@{{ .DBAddress }}/{{ .DBName }}
+connection = {{ .DatabaseConnection }}
 
 [api]
 auth_strategy = keystone

diff --git a/tests/functional/placementapi_controller_test.go b/tests/functional/placementapi_controller_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package functional_test
 
 import (
+	"fmt"
 	"os"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -200,6 +201,17 @@ var _ = Describe("PlacementAPI controller", func() {
 			)
 			DeferCleanup(
 				k8sClient.Delete, ctx, CreatePlacementAPISecret(namespace, SecretName))
+
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			db := mariadb.GetMariaDBDatabase(names.MariaDBDatabaseName)
+			Expect(db.Spec.Name).To(Equal(names.MariaDBDatabaseName.Name))
+
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
 		})
 
 		It("should have input ready", func() {
@@ -239,15 +251,62 @@ var _ = Describe("PlacementAPI controller", func() {
 			DeferCleanup(keystone.DeleteKeystoneAPI, keystoneAPIName)
 		})
 
+		It("creates MariaDB database", func() {
+			th.ExpectCondition(
+				names.PlacementAPIName,
+				ConditionGetterFunc(PlacementConditionGetter),
+				condition.DBReadyCondition,
+				corev1.ConditionFalse,
+			)
+
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			db := mariadb.GetMariaDBDatabase(names.MariaDBDatabaseName)
+			Expect(db.Spec.Name).To(Equal(names.MariaDBDatabaseName.Name))
+
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
+
+			th.ExpectCondition(
+				names.PlacementAPIName,
+				ConditionGetterFunc(PlacementConditionGetter),
+				condition.DBReadyCondition,
+				corev1.ConditionTrue,
+			)
+		})
+
 		It("should have config ready", func() {
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			db := mariadb.GetMariaDBDatabase(names.MariaDBDatabaseName)
+			Expect(db.Spec.Name).To(Equal(names.MariaDBDatabaseName.Name))
+
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
+
 			th.ExpectCondition(
 				names.PlacementAPIName,
 				ConditionGetterFunc(PlacementConditionGetter),
 				condition.ServiceConfigReadyCondition,
 				corev1.ConditionTrue,
 			)
 		})
+
 		It("should create a configuration Secret", func() {
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
+
 			cm := th.GetSecret(names.ConfigMapName)
 
 			conf := cm.Data["placement.conf"]
@@ -260,18 +319,28 @@ var _ = Describe("PlacementAPI controller", func() {
 			Expect(conf).Should(
 				ContainSubstring("password = 12345678"))
 			Expect(conf).Should(
-				ContainSubstring("connection = mysql+pymysql://placement:12345678@/placement"))
-
+				ContainSubstring(fmt.Sprintf("connection = mysql+pymysql://placement:12345678@hostname-for-openstack.%s.svc/placement?read_default_file=/etc/my.cnf", namespace)))
 			custom := cm.Data["custom.conf"]
 			Expect(custom).Should(ContainSubstring("foo = bar"))
 
 			policy := cm.Data["policy.yaml"]
 			Expect(policy).Should(
 				ContainSubstring("\"placement:resource_providers:list\": \"!\""))
 
+			myCnf := cm.Data["my.cnf"]
+			Expect(myCnf).To(
+				ContainSubstring("[client]\nssl=0"))
 		})
 
 		It("creates service account, role and rolebindig", func() {
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(
+				mariadb.DeleteDBService,
+				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
+			)
+			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
+
 			th.ExpectCondition(
 				names.PlacementAPIName,
 				ConditionGetterFunc(PlacementConditionGetter),
@@ -302,33 +371,6 @@ var _ = Describe("PlacementAPI controller", func() {
 			Expect(binding.Subjects).To(HaveLen(1))
 			Expect(binding.Subjects[0].Name).To(Equal(sa.Name))
 		})
-
-		It("creates MariaDB database", func() {
-			th.ExpectCondition(
-				names.PlacementAPIName,
-				ConditionGetterFunc(PlacementConditionGetter),
-				condition.DBReadyCondition,
-				corev1.ConditionFalse,
-			)
-
-			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
-			DeferCleanup(
-				mariadb.DeleteDBService,
-				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
-			)
-			db := mariadb.GetMariaDBDatabase(names.MariaDBDatabaseName)
-			Expect(db.Spec.Name).To(Equal(names.MariaDBDatabaseName.Name))
-
-			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
-			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
-
-			th.ExpectCondition(
-				names.PlacementAPIName,
-				ConditionGetterFunc(PlacementConditionGetter),
-				condition.DBReadyCondition,
-				corev1.ConditionTrue,
-			)
-		})
 		It("creates keystone service", func() {
 			th.ExpectCondition(
 				names.PlacementAPIName,
@@ -757,7 +799,7 @@ var _ = Describe("PlacementAPI controller", func() {
 				mariadb.DeleteDBService,
 				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
 			)
-			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBTLSDatabaseCompleted(names.MariaDBDatabaseName)
 			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
 			keystone.SimulateKeystoneServiceReady(names.KeystoneServiceName)
 			keystone.SimulateKeystoneEndpointReady(names.KeystoneEndpointName)
@@ -795,6 +837,10 @@ var _ = Describe("PlacementAPI controller", func() {
 			Expect(configData).Should(ContainSubstring("SSLCertificateKeyFile   \"/etc/pki/tls/private/internal.key\""))
 			Expect(configData).Should(ContainSubstring("SSLCertificateFile      \"/etc/pki/tls/certs/public.crt\""))
 			Expect(configData).Should(ContainSubstring("SSLCertificateKeyFile   \"/etc/pki/tls/private/public.key\""))
+
+			configData = string(configDataMap.Data["my.cnf"])
+			Expect(configData).To(
+				ContainSubstring("[client]\nssl-ca=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem\nssl=1"))
 		})
 	})
 })
@@ -823,7 +869,7 @@ var _ = Describe("PlacementAPI reconfiguration", func() {
 				mariadb.DeleteDBService,
 				mariadb.CreateDBService(namespace, "openstack", serviceSpec),
 			)
-			mariadb.SimulateMariaDBDatabaseCompleted(names.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBTLSDatabaseCompleted(names.MariaDBDatabaseName)
 			mariadb.SimulateMariaDBAccountCompleted(names.MariaDBDatabaseName)
 			keystone.SimulateKeystoneServiceReady(names.KeystoneServiceName)
 			keystone.SimulateKeystoneEndpointReady(names.KeystoneEndpointName)