Switch to StatefulSet for ovn-controller pod

TODO: - anti-affinity rule - expose storageClass? - maintain the proper number of replicas
openstack-k8s-operators · Jan 4, 2024 · 89c523f · 89c523f
1 parent 33d49aa
commit 89c523f
Show file tree

Hide file tree

Showing 3 changed files with 85 additions and 37 deletions.
diff --git a/controllers/ovncontroller_controller.go b/controllers/ovncontroller_controller.go
@@ -38,12 +38,12 @@ import (
 	"github.com/openstack-k8s-operators/lib-common/modules/common"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/condition"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/configmap"
-	"github.com/openstack-k8s-operators/lib-common/modules/common/daemonset"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/env"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/labels"
 	nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment"
 	common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/statefulset"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/util"
 	"github.com/openstack-k8s-operators/ovn-operator/api/v1beta1"
 	"github.com/openstack-k8s-operators/ovn-operator/pkg/ovncontroller"
@@ -203,7 +203,7 @@ func (r *OVNControllerReconciler) SetupWithManager(mgr ctrl.Manager, ctx context
 		For(&v1beta1.OVNController{}).
 		Owns(&corev1.ConfigMap{}).
 		Owns(&netattdefv1.NetworkAttachmentDefinition{}).
-		Owns(&appsv1.DaemonSet{}).
+		Owns(&appsv1.StatefulSet{}).
 		Owns(&corev1.ServiceAccount{}).
 		Owns(&rbacv1.Role{}).
 		Owns(&rbacv1.RoleBinding{}).
@@ -412,18 +412,18 @@ func (r *OVNControllerReconciler) reconcileNormal(ctx context.Context, instance
 		return ctrl.Result{}, err
 	}
 
-	// Define a new DaemonSet object
-	ovnDaemonSet, err := ovncontroller.DaemonSet(instance, sbCluster, inputHash, serviceLabels, serviceAnnotations)
+	// Define a new StatefulSet object
+	ovnStatefulSet, err := ovncontroller.StatefulSet(instance, sbCluster, inputHash, serviceLabels, serviceAnnotations)
 	if err != nil {
-		Log.Error(err, "Failed to create OVNController DaemonSet")
 		return ctrl.Result{}, err
 	}
-	dset := daemonset.NewDaemonSet(
-		ovnDaemonSet,
+
+	sset := statefulset.NewStatefulSet(
+		ovnStatefulSet,
 		time.Duration(5)*time.Second,
 	)
 
-	ctrlResult, err = dset.CreateOrPatch(ctx, helper)
+	ctrlResult, err = sset.CreateOrPatch(ctx, helper)
 	if err != nil {
 		instance.Status.Conditions.Set(condition.FalseCondition(
 			condition.DeploymentReadyCondition,
@@ -441,8 +441,8 @@ func (r *OVNControllerReconciler) reconcileNormal(ctx context.Context, instance
 		return ctrlResult, nil
 	}
 
-	instance.Status.DesiredNumberScheduled = dset.GetDaemonSet().Status.DesiredNumberScheduled
-	instance.Status.NumberReady = dset.GetDaemonSet().Status.NumberReady
+	instance.Status.DesiredNumberScheduled = sset.GetStatefulSet().Status.Replicas
+	instance.Status.NumberReady = sset.GetStatefulSet().Status.ReadyReplicas
 
 	// verify if network attachment matches expectations
 	networkReady, networkAttachmentStatus, err := nad.VerifyNetworkStatusFromAnnotation(ctx, helper, networkAttachmentsNoPhysNet, serviceLabels, instance.Status.NumberReady)

diff --git a/pkg/ovncontroller/daemonset.go → pkg/ovncontroller/statefulset.go b/pkg/ovncontroller/daemonset.go → pkg/ovncontroller/statefulset.go
@@ -16,23 +16,33 @@ import (
 	"fmt"
 
 	"github.com/openstack-k8s-operators/lib-common/modules/common"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/affinity"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/env"
 	"github.com/openstack-k8s-operators/ovn-operator/api/v1beta1"
+	ovnv1 "github.com/openstack-k8s-operators/ovn-operator/api/v1beta1"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-// DaemonSet func
-func DaemonSet(
-	instance *v1beta1.OVNController,
+const (
+	// ServiceCommand -
+	ServiceCommand = "/usr/local/bin/container-scripts/setup.sh"
+
+	// PvcSuffixEtcOvn -
+	PvcSuffixEtcOvn = "-etc-ovs"
+)
+
+// StatefulSet func
+func StatefulSet(
+	instance *ovnv1.OVNController,
 	sbCluster *v1beta1.OVNDBCluster,
 	configHash string,
 	labels map[string]string,
 	annotations map[string]string,
-) (*appsv1.DaemonSet, error) {
-
+) (*appsv1.StatefulSet, error) {
 	runAsUser := int64(0)
 	privileged := true
 
@@ -149,15 +159,21 @@ func DaemonSet(
 	envVars["PhysicalNetworks"] = env.SetValue(getPhysicalNetworks(instance))
 	envVars["OvnHostName"] = EnvDownwardAPI("spec.nodeName")
 
-	daemonset := &appsv1.DaemonSet{
+	// TODO
+	replicas := int32(1)
+
+	statefulset := &appsv1.StatefulSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      ServiceName,
 			Namespace: instance.Namespace,
 		},
-		Spec: appsv1.DaemonSetSpec{
+		Spec: appsv1.StatefulSetSpec{
 			Selector: &metav1.LabelSelector{
 				MatchLabels: labels,
 			},
+			ServiceName: ServiceName,
+			// Replicas:    instance.Spec.Replicas,
+			Replicas: &replicas,
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels:      labels,
@@ -188,7 +204,7 @@ func DaemonSet(
 								Privileged: &privileged,
 							},
 							Env:                      env.MergeEnvs([]corev1.EnvVar{}, envVars),
-							VolumeMounts:             GetOvsDbVolumeMounts(),
+							VolumeMounts:             GetOvsDbVolumeMounts(instance.Name + PvcSuffixEtcOvn),
 							Resources:                instance.Spec.Resources,
 							LivenessProbe:            ovsDbLivenessProbe,
 							TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
@@ -254,12 +270,56 @@ func DaemonSet(
 			},
 		},
 	}
-	daemonset.Spec.Template.Spec.Volumes = GetVolumes(instance.Name, instance.Namespace)
 
-	if instance.Spec.NodeSelector != nil && len(instance.Spec.NodeSelector) > 0 {
-		daemonset.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector
+	// https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention
+	statefulset.Spec.PersistentVolumeClaimRetentionPolicy = &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
+		WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
+		WhenScaled:  appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
 	}
 
-	return daemonset, nil
+	blockOwnerDeletion := false
+	ownerRef := metav1.NewControllerRef(instance, instance.GroupVersionKind())
+	ownerRef.BlockOwnerDeletion = &blockOwnerDeletion
+
+	// TODO
+	storageClass := "local-storage"
+
+	statefulset.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:            instance.Name + PvcSuffixEtcOvn,
+				Namespace:       instance.Namespace,
+				Labels:          labels,
+				OwnerReferences: []metav1.OwnerReference{*ownerRef},
+			},
+			Spec: corev1.PersistentVolumeClaimSpec{
+				AccessModes: []corev1.PersistentVolumeAccessMode{
+					corev1.ReadWriteOnce,
+				},
+				StorageClassName: &storageClass,
+				Resources: corev1.ResourceRequirements{
+					Requests: corev1.ResourceList{
+						// TODO
+						corev1.ResourceStorage: resource.MustParse("10G"),
+					},
+				},
+			},
+		},
+	}
+	statefulset.Spec.Template.Spec.Volumes = GetVolumes(instance.Name)
+	// If possible two pods of the same service should not
+	// run on the same worker node. If this is not possible
+	// the get still created on the same worker node.
+	statefulset.Spec.Template.Spec.Affinity = affinity.DistributePods(
+		common.AppSelector,
+		[]string{
+			ServiceName,
+		},
+		corev1.LabelHostname,
+	)
+	if instance.Spec.NodeSelector != nil && len(instance.Spec.NodeSelector) > 0 {
+		statefulset.Spec.Template.Spec.NodeSelector = instance.Spec.NodeSelector
+	}
 
+	return statefulset, nil
 }
diff --git a/pkg/ovncontroller/volumes.go b/pkg/ovncontroller/volumes.go
@@ -1,27 +1,15 @@
 package ovncontroller
 
 import (
-	"fmt"
-
 	corev1 "k8s.io/api/core/v1"
 )
 
 // GetVolumes -
-func GetVolumes(name string, namespace string) []corev1.Volume {
+func GetVolumes(name string) []corev1.Volume {
 
 	var scriptsVolumeDefaultMode int32 = 0755
-	directoryOrCreate := corev1.HostPathDirectoryOrCreate
 
 	return []corev1.Volume{
-		{
-			Name: "etc-ovs",
-			VolumeSource: corev1.VolumeSource{
-				HostPath: &corev1.HostPathVolumeSource{
-					Path: fmt.Sprintf("/var/home/core/%s/etc/ovs", namespace),
-					Type: &directoryOrCreate,
-				},
-			},
-		},
 		{
 			Name: "var-run",
 			VolumeSource: corev1.VolumeSource{
@@ -44,10 +32,10 @@ func GetVolumes(name string, namespace string) []corev1.Volume {
 }
 
 // GetOvsDbVolumeMounts - ovsdb-server VolumeMounts
-func GetOvsDbVolumeMounts() []corev1.VolumeMount {
+func GetOvsDbVolumeMounts(name string) []corev1.VolumeMount {
 	return []corev1.VolumeMount{
 		{
-			Name:      "etc-ovs",
+			Name:      name,
 			MountPath: "/etc/openvswitch",
 			ReadOnly:  false,
 		},