feat: Support setting labels and annotations for the CP Service

smartxworks · Apr 17, 2023 · 8cac0f5 · 8cac0f5
1 parent 49f86ce
commit 8cac0f5
Show file tree

Hide file tree

Showing 10 changed files with 99 additions and 28 deletions.
diff --git a/api/v1beta1/virtinkcluster_types.go b/api/v1beta1/virtinkcluster_types.go
@@ -17,15 +17,30 @@ type VirtinkClusterSpec struct {
 	// ControlPlaneEndpoint represents the endpoint used to communicate with the control plane.
 	ControlPlaneEndpoint capiv1beta1.APIEndpoint `json:"controlPlaneEndpoint,omitempty"`
 
-	// ControlPlaneServiceType can be used to modify type of service that fronts the control plane nodes to handle the
+	// ControlPlaneServiceTemplate can be used to modify service that fronts the control plane nodes to handle the
 	// api-server traffic (port 6443). This field is optional, by default control plane nodes will use a service
-	// of type ClusterIP, which will make workload cluster only accessible within the same cluster.
-	ControlPlaneServiceType *corev1.ServiceType `json:"controlPlaneServiceType,omitempty"`
+	// of type ClusterIP, which will make workload cluster only accessible within the same cluster. Note, this does
+	// not aim to expose the entire service spec to users, but only provides capability to modify the service metadata
+	// and the service type.
+	ControlPlaneServiceTemplate ControlPlaneServiceTemplate `json:"controlPlaneServiceTemplate,omitempty"`
 
 	// InfraClusterSecretRef is a reference to a secret with a kubeconfig for external cluster used for infra.
 	InfraClusterSecretRef *corev1.ObjectReference `json:"infraClusterSecretRef,omitempty"`
 }
 
+// ControlPlaneServiceTemplate describes the template for the control plane service.
+type ControlPlaneServiceTemplate struct {
+	// Service metadata allows to set labels and annotations for the service.
+	// This field is optional.
+	// +kubebuilder:pruning:PreserveUnknownFields
+	ObjectMeta metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	// Type can be used to modify type of service that fronts the control plane nodes to handle the
+	// api-server traffic (port 6443). This field is optional, by default control plane nodes will use a service
+	// of type ClusterIP, which will make workload cluster only accessible within the same cluster.
+	Type *corev1.ServiceType `json:"type,omitempty"`
+}
+
 // VirtinkClusterStatus defines the observed state of VirtinkCluster
 type VirtinkClusterStatus struct {
 	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkclusters.yaml
@@ -60,13 +60,28 @@ spec:
                 - host
                 - port
                 type: object
-              controlPlaneServiceType:
-                description: ControlPlaneServiceType can be used to modify type of
-                  service that fronts the control plane nodes to handle the api-server
-                  traffic (port 6443). This field is optional, by default control
-                  plane nodes will use a service of type ClusterIP, which will make
-                  workload cluster only accessible within the same cluster.
-                type: string
+              controlPlaneServiceTemplate:
+                description: ControlPlaneServiceTemplate can be used to modify service
+                  that fronts the control plane nodes to handle the api-server traffic
+                  (port 6443). This field is optional, by default control plane nodes
+                  will use a service of type ClusterIP, which will make workload cluster
+                  only accessible within the same cluster. Note, this does not aim
+                  to expose the entire service spec to users, but only provides capability
+                  to modify the service metadata and the service type.
+                properties:
+                  metadata:
+                    description: Service metadata allows to set labels and annotations
+                      for the service. This field is optional.
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
+                  type:
+                    description: Type can be used to modify type of service that fronts
+                      the control plane nodes to handle the api-server traffic (port
+                      6443). This field is optional, by default control plane nodes
+                      will use a service of type ClusterIP, which will make workload
+                      cluster only accessible within the same cluster.
+                    type: string
+                type: object
               infraClusterSecretRef:
                 description: InfraClusterSecretRef is a reference to a secret with
                   a kubeconfig for external cluster used for infra.

diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_virtinkclustertemplates.yaml
@@ -56,14 +56,30 @@ spec:
                         - host
                         - port
                         type: object
-                      controlPlaneServiceType:
-                        description: ControlPlaneServiceType can be used to modify
-                          type of service that fronts the control plane nodes to handle
-                          the api-server traffic (port 6443). This field is optional,
+                      controlPlaneServiceTemplate:
+                        description: ControlPlaneServiceTemplate can be used to modify
+                          service that fronts the control plane nodes to handle the
+                          api-server traffic (port 6443). This field is optional,
                           by default control plane nodes will use a service of type
                           ClusterIP, which will make workload cluster only accessible
-                          within the same cluster.
-                        type: string
+                          within the same cluster. Note, this does not aim to expose
+                          the entire service spec to users, but only provides capability
+                          to modify the service metadata and the service type.
+                        properties:
+                          metadata:
+                            description: Service metadata allows to set labels and
+                              annotations for the service. This field is optional.
+                            type: object
+                            x-kubernetes-preserve-unknown-fields: true
+                          type:
+                            description: Type can be used to modify type of service
+                              that fronts the control plane nodes to handle the api-server
+                              traffic (port 6443). This field is optional, by default
+                              control plane nodes will use a service of type ClusterIP,
+                              which will make workload cluster only accessible within
+                              the same cluster.
+                            type: string
+                        type: object
                       infraClusterSecretRef:
                         description: InfraClusterSecretRef is a reference to a secret
                           with a kubeconfig for external cluster used for infra.

diff --git a/controllers/virtinkcluster_controller.go b/controllers/virtinkcluster_controller.go
@@ -149,7 +149,7 @@ func (r *VirtinkClusterReconciler) reconcile(ctx context.Context, cluster *infra
 			r.Recorder.Eventf(cluster, corev1.EventTypeNormal, "CreatedControlPlaneService", "Created control plane Service %q", controlPlaneService.Name)
 		}
 
-		if cluster.Spec.ControlPlaneServiceType != nil && *cluster.Spec.ControlPlaneServiceType == corev1.ServiceTypeLoadBalancer {
+		if cluster.Spec.ControlPlaneServiceTemplate.Type != nil && *cluster.Spec.ControlPlaneServiceTemplate.Type == corev1.ServiceTypeLoadBalancer {
 			if len(controlPlaneService.Status.LoadBalancer.Ingress) == 0 {
 				return fmt.Errorf("control plane load balancer is not ready")
 			}
@@ -210,8 +210,10 @@ func (r *VirtinkClusterReconciler) buildControlPlaneService(ctx context.Context,
 			}},
 		},
 	}
-	if cluster.Spec.ControlPlaneServiceType != nil {
-		service.Spec.Type = *cluster.Spec.ControlPlaneServiceType
+	service.Labels = cluster.Spec.ControlPlaneServiceTemplate.ObjectMeta.Labels
+	service.Annotations = cluster.Spec.ControlPlaneServiceTemplate.ObjectMeta.Annotations
+	if cluster.Spec.ControlPlaneServiceTemplate.Type != nil {
+		service.Spec.Type = *cluster.Spec.ControlPlaneServiceTemplate.Type
 	}
 	return service, nil
 }

diff --git a/docs/external-cluster.md b/docs/external-cluster.md
@@ -63,6 +63,8 @@ SA_TOKEN="$(kubectl get secret "${SA_SECRET}" -o jsonpath='{.data.token}' | base
 kubectl config --kubeconfig virtink-infra-cluster.kubeconfig set-credentials kubernetes-admin "--token=${SA_TOKEN}"
 ```
 
+> **Note**: In more recent versions, including K8S v1.24, the long term API token will not be automatically created for the ServiceAccount, you may [manually create a token Secret for the ServiceAccount](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#manually-create-a-long-lived-api-token-for-a-serviceaccount), and set the enviroment variable `SA_SECRET` above.
+
 Create a secret in managment cluster and set environment variables before generating workload cluster configuration.
 
 ```shell

diff --git a/templates/cluster-template-cdi-internal.yaml b/templates/cluster-template-cdi-internal.yaml
@@ -22,7 +22,8 @@ kind: VirtinkCluster
 metadata:
   name: "${CLUSTER_NAME}"
 spec:
-  controlPlaneServiceType: "${VIRTINK_CONTROL_PLANE_SERVICE_TYPE:=NodePort}"
+  controlPlaneServiceTemplate:
+    type: "${VIRTINK_CONTROL_PLANE_SERVICE_TYPE:=NodePort}"
 ---
 apiVersion: controlplane.cluster.x-k8s.io/v1beta1
 kind: KubeadmControlPlane

diff --git a/templates/cluster-template-cdi.yaml b/templates/cluster-template-cdi.yaml
@@ -22,7 +22,8 @@ kind: VirtinkCluster
 metadata:
   name: "${CLUSTER_NAME}"
 spec:
-  controlPlaneServiceType: "${VIRTINK_CONTROL_PLANE_SERVICE_TYPE:=NodePort}"
+  controlPlaneServiceTemplate:
+    type: "${VIRTINK_CONTROL_PLANE_SERVICE_TYPE:=NodePort}"
   infraClusterSecretRef:
     name: "${VIRTINK_INFRA_CLUSTER_SECRET_NAME}"
     namespace: "${VIRTINK_INFRA_CLUSTER_SECRET_NAMESPACE}"

diff --git a/templates/cluster-template-internal.yaml b/templates/cluster-template-internal.yaml
@@ -22,7 +22,8 @@ kind: VirtinkCluster
 metadata:
   name: "${CLUSTER_NAME}"
 spec:
-  controlPlaneServiceType: "${VIRTINK_CONTROL_PLANE_SERVICE_TYPE:=NodePort}"
+  controlPlaneServiceTemplate:
+    type: "${VIRTINK_CONTROL_PLANE_SERVICE_TYPE:=NodePort}"
 ---
 apiVersion: controlplane.cluster.x-k8s.io/v1beta1
 kind: KubeadmControlPlane

diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml
@@ -22,7 +22,8 @@ kind: VirtinkCluster
 metadata:
   name: "${CLUSTER_NAME}"
 spec:
-  controlPlaneServiceType: "${VIRTINK_CONTROL_PLANE_SERVICE_TYPE:=NodePort}"
+  controlPlaneServiceTemplate:
+    type: "${VIRTINK_CONTROL_PLANE_SERVICE_TYPE:=NodePort}"
   infraClusterSecretRef:
     name: "${VIRTINK_INFRA_CLUSTER_SECRET_NAME}"
     namespace: "${VIRTINK_INFRA_CLUSTER_SECRET_NAMESPACE}"