refactor: change datanode from deployment to statefulset

apis/v1alpha1/constants.go

@@ -21,4 +21,10 @@ const (
 	defaultEtcdStorageClassName = "standard" // 'standard' is the default local storage class of kind.
 	defaultEtcdStorageSize      = "10Gi"
 	defaultEtcdStorageMountPath = "/var/run/etcd"
+
+	// The default storage settings for datanode.
+	defaultDataNodeStorageName      = "datanode"
+	defaultDataNodeStorageClassName = "standard" // 'standard' is the default local storage class of kind.
+	defaultDataNodeStorageSize      = "10Gi"
+	defaultDataNodeStorageMountPath = "/greptimedb/data"
 )

apis/v1alpha1/defaulting.go

@@ -49,6 +49,7 @@ func (in *GreptimeDBCluster) SetDefaults() {
 		} else {
 			in.Spec.Datanode.Template.overlay(in.Spec.Base)
 		}
+		in.Spec.Datanode.Storage.setDefaults()
 	}
 
 	if in.Spec.HTTPServicePort == 0 {
@@ -64,6 +65,29 @@ func (in *GreptimeDBCluster) SetDefaults() {
 	}
 }
 
+func (in *StorageSpec) setDefaults() {
+	if in == nil {
+		return
+	}
+
+	if in.Name == "" {
+		in.Name = defaultDataNodeStorageName
+	}
+
+	if in.MountPath == "" {
+		in.MountPath = defaultDataNodeStorageMountPath
+	}
+
+	if in.StorageClassName == nil {
+		storageClassName := defaultDataNodeStorageClassName
+		in.StorageClassName = &storageClassName
+	}
+
+	if in.StorageSize == "" {
+		in.StorageSize = defaultDataNodeStorageSize
+	}
+}
+
 func (in *EtcdSpec) setDefaults() {
 	if in == nil {
 		return

apis/v1alpha1/greptimedbcluster_types.go

@@ -287,6 +287,7 @@ type FrontendSpec struct {
 type DatanodeSpec struct {
 	ComponentSpec `json:",inline"`
 
+	Storage StorageSpec `json:"storage,omitempty"`
 	// More datanode settings can be added here...
 }
 

config/crd/bases/greptime.cloud_greptimedbclusters.yaml

@@ -1975,6 +1975,17 @@ spec:
                   replicas:
                     format: int32
                     type: integer
+                  storage:
+                    properties:
+                      mountPath:
+                        type: string
+                      name:
+                        type: string
+                      storageClassName:
+                        type: string
+                      storageSize:
+                        type: string
+                    type: object
                   template:
                     properties:
                       activeDeadlineSeconds:

controllers/greptimedbcluster/greptimedbcluster_controller.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"k8s.io/klog/v2"
 	"strconv"
 	"strings"
 
@@ -17,7 +18,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/retry"
-	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -261,33 +261,18 @@ func (r *Reconciler) syncDatanode(ctx context.Context, cluster *v1alpha1.Greptim
 		return false, err
 	}
 
-	newDatanodeDeployment, err := r.buildDatanodeDeployment(cluster)
+	newDatanodeStatefulSet, err := r.buildDatanodeStatefulSet(cluster)
 	if err != nil {
 		return false, err
 	}
 
-	datanodeDeployment, err := r.createIfNotExist(ctx, new(appsv1.Deployment), newDatanodeDeployment)
+	datanodeStatefulSet, err := r.createIfNotExist(ctx, new(appsv1.StatefulSet), newDatanodeStatefulSet)
 	if err != nil {
 		return false, err
 	}
 
-	if deployment, ok := datanodeDeployment.(*appsv1.Deployment); ok {
-		needToUpdate, err := r.isNeedToUpdate(deployment, new(appsv1.DeploymentSpec), &newDatanodeDeployment.Spec)
-		if err != nil {
-			return false, err
-		}
-
-		if r.isDeploymentReady(deployment) && !needToUpdate {
-			return true, nil
-		}
-
-		if needToUpdate {
-			if err := r.Update(ctx, newDatanodeDeployment); err != nil {
-				return false, err
-			}
-		}
-
-		return false, nil
+	if statefulSet, ok := datanodeStatefulSet.(*appsv1.StatefulSet); ok {
+		return r.isStatefulSetReady(statefulSet), nil
 	}
 
 	return false, nil
@@ -656,13 +641,13 @@ func (r *Reconciler) buildDatanodeService(cluster *v1alpha1.GreptimeDBCluster) (
 	return service, nil
 }
 
-func (r *Reconciler) buildDatanodeDeployment(cluster *v1alpha1.GreptimeDBCluster) (*appsv1.Deployment, error) {
-	deployment := &appsv1.Deployment{
+func (r *Reconciler) buildDatanodeStatefulSet(cluster *v1alpha1.GreptimeDBCluster) (*appsv1.StatefulSet, error) {
+	statefulset := &appsv1.StatefulSet{
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace: cluster.Namespace,
 			Name:      cluster.Name + "-datanode",
+			Namespace: cluster.Namespace,
 		},
-		Spec: appsv1.DeploymentSpec{
+		Spec: appsv1.StatefulSetSpec{
 			Replicas: &cluster.Spec.Datanode.Replicas,
 			Selector: &metav1.LabelSelector{
 				MatchLabels: map[string]string{
@@ -676,8 +661,6 @@ func (r *Reconciler) buildDatanodeDeployment(cluster *v1alpha1.GreptimeDBCluster
 					},
 					Annotations: cluster.Spec.Datanode.Template.Annotations,
 				},
-
-				// TODO(zyy17): Fill more arguments.
 				Spec: corev1.PodSpec{
 					Containers: []corev1.Container{
 						{
@@ -686,6 +669,12 @@ func (r *Reconciler) buildDatanodeDeployment(cluster *v1alpha1.GreptimeDBCluster
 							Resources: *cluster.Spec.Datanode.Template.MainContainer.Resources,
 							Command:   cluster.Spec.Datanode.Template.MainContainer.Command,
 							Args:      cluster.Spec.Datanode.Template.MainContainer.Args,
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      cluster.Spec.Datanode.Storage.Name,
+									MountPath: cluster.Spec.Datanode.Storage.MountPath,
+								},
+							},
 							Ports: []corev1.ContainerPort{
 								{
 									Name:          "grpc",
@@ -707,22 +696,39 @@ func (r *Reconciler) buildDatanodeDeployment(cluster *v1alpha1.GreptimeDBCluster
 					},
 				},
 			},
+			VolumeClaimTemplates: []corev1.PersistentVolumeClaim{
+				{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: cluster.Spec.Datanode.Storage.Name,
+					},
+					Spec: corev1.PersistentVolumeClaimSpec{
+						StorageClassName: cluster.Spec.Datanode.Storage.StorageClassName,
+						AccessModes: []corev1.PersistentVolumeAccessMode{
+							corev1.ReadWriteOnce,
+						},
+						Resources: corev1.ResourceRequirements{
+							Requests: corev1.ResourceList{
+								corev1.ResourceStorage: resource.MustParse(cluster.Spec.Datanode.Storage.StorageSize),
+							},
+						},
+					},
+				},
+			},
 		},
 	}
-
 	for k, v := range cluster.Spec.Datanode.Template.Labels {
-		deployment.Labels[k] = v
+		statefulset.Labels[k] = v
 	}
 
-	if err := controllerutil.SetControllerReference(cluster, deployment, r.Scheme); err != nil {
+	if err := controllerutil.SetControllerReference(cluster, statefulset, r.Scheme); err != nil {
 		return nil, err
 	}
 
-	if err := r.setLastAppliedResourceSpecAnnotation(deployment, deployment.Spec); err != nil {
+	if err := r.setLastAppliedResourceSpecAnnotation(statefulset, statefulset.Spec); err != nil {
 		return nil, err
 	}
 
-	return deployment, nil
+	return statefulset, nil
 }
 
 func (r *Reconciler) createIfNotExist(ctx context.Context, source, newObject client.Object) (client.Object, error) {

controllers/greptimedbcluster/greptimedbcluster_controller_test.go

@@ -64,11 +64,11 @@ var _ = Describe("Test greptimedbcluster controller", func() {
 		By("Check datanode resource")
 		svc = &corev1.Service{}
 		checkResource(testNamespace, testClusterName+"-datanode", svc, req)
-		deployment = &appsv1.Deployment{}
-		checkResource(testNamespace, testClusterName+"-datanode", deployment, req)
+		statefulSet = &appsv1.StatefulSet{}
+		checkResource(testNamespace, testClusterName+"-datanode", statefulSet, req)
 
 		// Move forward to sync frontend.
-		Expect(makeDeploymentReady(deployment, cluster.Spec.Datanode.Replicas)).NotTo(HaveOccurred(), "failed to update datanode deploylemt status")
+		Expect(makeStatefulSetReady(statefulSet, cluster.Spec.Datanode.Replicas)).NotTo(HaveOccurred(), "failed to update datanode statefulset status")
 
 		By("Check frontend resource")
 		svc = &corev1.Service{}

go.mod

@@ -29,12 +29,14 @@ require (
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
 	github.com/google/uuid v1.1.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
@@ -59,6 +61,7 @@ require (
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
+	golang.org/x/tools v0.1.10 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect

go.sum

@@ -165,6 +165,8 @@ github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -242,6 +244,8 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -767,6 +771,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
+golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
+golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=