Adding TLS Certificate Authentication to gRPC

Signed-off-by: zhzhuang-zju <m17799853869@163.com>

artifacts/deploy/karmada-descheduler.yaml

@@ -27,6 +27,9 @@ spec:
             - /bin/karmada-descheduler
             - --kubeconfig=/etc/kubeconfig
             - --bind-address=0.0.0.0
+            - --trusted-ca-file=/etc/karmada/pki/ca.crt
+            - --cert-file=/etc/karmada/pki/karmada.crt
+            - --key-file=/etc/karmada/pki/karmada.key
             - --v=4
           livenessProbe:
             httpGet:
@@ -38,10 +41,16 @@ spec:
             periodSeconds: 15
             timeoutSeconds: 5
           volumeMounts:
+            - name: karmada-certs
+              mountPath: /etc/karmada/pki
+              readOnly: true
             - name: kubeconfig
               subPath: kubeconfig
               mountPath: /etc/kubeconfig
       volumes:
+        - name: karmada-certs
+          secret:
+            secretName: karmada-cert-secret
         - name: kubeconfig
           secret:
             secretName: kubeconfig

artifacts/deploy/karmada-scheduler-estimator.yaml

@@ -27,6 +27,10 @@ spec:
             - /bin/karmada-scheduler-estimator
             - --kubeconfig=/etc/{{member_cluster_name}}-kubeconfig
             - --cluster-name={{member_cluster_name}}
+            - --cert-file=/etc/karmada/pki/karmada.crt
+            - --key-file=/etc/karmada/pki/karmada.key
+            - --client-cert-auth=true
+            - --trusted-ca-file=/etc/karmada/pki/ca.crt
           livenessProbe:
             httpGet:
               path: /healthz
@@ -37,10 +41,16 @@ spec:
             periodSeconds: 15
             timeoutSeconds: 5
           volumeMounts:
+            - name: karmada-certs
+              mountPath: /etc/karmada/pki
+              readOnly: true
             - name: member-kubeconfig
               subPath: {{member_cluster_name}}-kubeconfig
               mountPath: /etc/{{member_cluster_name}}-kubeconfig
       volumes:
+        - name: karmada-certs
+          secret:
+            secretName: karmada-cert-secret
         - name: member-kubeconfig
           secret:
             secretName: {{member_cluster_name}}-kubeconfig

artifacts/deploy/karmada-scheduler.yaml

@@ -39,11 +39,20 @@ spec:
             - --secure-port=10351
             - --enable-scheduler-estimator=true
             - --v=4
+            - --trusted-ca-file=/etc/karmada/pki/ca.crt
+            - --cert-file=/etc/karmada/pki/karmada.crt
+            - --key-file=/etc/karmada/pki/karmada.key
           volumeMounts:
+            - name: karmada-certs
+              mountPath: /etc/karmada/pki
+              readOnly: true
             - name: kubeconfig
               subPath: kubeconfig
               mountPath: /etc/kubeconfig
       volumes:
+        - name: karmada-certs
+          secret:
+            secretName: karmada-cert-secret
         - name: kubeconfig
           secret:
             secretName: kubeconfig

charts/karmada/templates/_helpers.tpl

@@ -349,6 +349,13 @@ app: {{- include "karmada.name" .}}-search
   {{- end }}
 {{- end -}}
 
+{{- define "karmada.scheduler.cert.volume" -}}
+{{ $name :=  include "karmada.name" . }}
+- name: karmada-certs
+  secret:
+    secretName: {{ $name }}-cert
+{{- end -}}
+
 {{/*
 Return the proper karmada internal etcd image name
 */}}

charts/karmada/templates/karmada-descheduler.yaml

@@ -52,6 +52,9 @@ spec:
             - --kubeconfig=/etc/kubeconfig
             - --bind-address=0.0.0.0
             - --leader-elect-resource-namespace={{ $systemNamespace }}
+            - --trusted-ca-file=/etc/karmada/pki/ca.crt
+            - --cert-file=/etc/karmada/pki/karmada.crt
+            - --key-file=/etc/karmada/pki/karmada.key
             - --v=4
           livenessProbe:
             httpGet:
@@ -63,12 +66,16 @@ spec:
             periodSeconds: 15
             timeoutSeconds: 5
           volumeMounts:
+            - name: karmada-certs
+              mountPath: /etc/karmada/pki
+              readOnly: true
           {{- include "karmada.kubeconfig.volumeMount" . | nindent 12 }}
           resources:
           {{- toYaml .Values.descheduler.resources | nindent 12 }}
       volumes:
       {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.descheduler.kubeconfig.volume" . | nindent 8 }}
+      {{- include "karmada.scheduler.cert.volume" . | nindent 8 }}
 
 {{ if .Values.descheduler.podDisruptionBudget }}
 ---

charts/karmada/templates/karmada-scheduler-estimator.yaml

@@ -48,6 +48,10 @@ spec:
             - /bin/karmada-scheduler-estimator
             - --kubeconfig=/etc/{{ $clusterName }}-kubeconfig
             - --cluster-name={{ $clusterName }}
+            - --cert-file=/etc/karmada/pki/karmada.crt
+            - --key-file=/etc/karmada/pki/karmada.key
+            - --client-cert-auth=true
+            - --trusted-ca-file=/etc/karmada/pki/ca.crt
             {{- with (include "karmada.schedulerEstimator.featureGates" (dict "featureGatesArg" $.Values.schedulerEstimator.featureGates)) }}
             - {{ . }}
             {{- end}}
@@ -61,12 +65,16 @@ spec:
             periodSeconds: 15
             timeoutSeconds: 5
           volumeMounts:
+            - name: karmada-certs
+              mountPath: /etc/karmada/pki
+              readOnly: true
             - name: member-kubeconfig
               subPath: {{ $clusterName }}-kubeconfig
               mountPath: /etc/{{ $clusterName }}-kubeconfig
           resources:
           {{- toYaml $.Values.schedulerEstimator.resources | nindent 12 }}
       volumes:
+      {{- include "karmada.scheduler.cert.volume" $ | nindent 8 }}
         - name: member-kubeconfig
           secret:
             secretName: {{ $clusterName }}-kubeconfig

charts/karmada/templates/karmada-scheduler.yaml

@@ -53,6 +53,9 @@ spec:
             - --bind-address=0.0.0.0
             - --secure-port=10351
             - --leader-elect-resource-namespace={{ $systemNamespace }}
+            - --trusted-ca-file=/etc/karmada/pki/ca.crt
+            - --cert-file=/etc/karmada/pki/karmada.crt
+            - --key-file=/etc/karmada/pki/karmada.key
           livenessProbe:
             httpGet:
               path: /healthz
@@ -63,12 +66,16 @@ spec:
             periodSeconds: 15
             timeoutSeconds: 5
           volumeMounts:
+            - name: karmada-certs
+              mountPath: /etc/karmada/pki
+              readOnly: true
           {{- include "karmada.kubeconfig.volumeMount" . | nindent 12 }}
           resources:
           {{- toYaml .Values.scheduler.resources | nindent 12 }}
       volumes:
       {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
+      {{- include "karmada.scheduler.cert.volume" . | nindent 8 }}
 
 {{ if .Values.scheduler.podDisruptionBudget }}
 ---

cmd/descheduler/app/options/options.go

@@ -62,6 +62,14 @@ type Options struct {
 	SchedulerEstimatorServicePrefix string
 	// SchedulerEstimatorPort is the port that the accurate scheduler estimator server serves at.
 	SchedulerEstimatorPort int
+	// InsecureSkipVerify controls whether verifies the grpc server's certificate chain and host name.
+	InsecureSkipVerify bool
+	// CertFile the certificate used for SSL/TLS connections.
+	CertFile string
+	// KeyFile the key for the certificate.
+	KeyFile string
+	// TrustedCAFile Trusted certificate authority.
+	TrustedCAFile string
 	// DeschedulingInterval specifies time interval for descheduler to run.
 	DeschedulingInterval metav1.Duration
 	// UnschedulableThreshold specifies the period of pod unschedulable condition.
@@ -99,6 +107,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver.")
 	fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.")
 	fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.")
+	fs.StringVar(&o.CertFile, "cert-file", "", "The certificate used for SSL/TLS connections.")
+	fs.StringVar(&o.KeyFile, "key-file", "", "The key for the certificate.")
+	fs.StringVar(&o.TrustedCAFile, "trusted-ca-file", "", "Trusted certificate authority.")
+	fs.BoolVar(&o.InsecureSkipVerify, "insecure-skip-verify", false, "Controls whether verifies the grpc server's certificate chain and host name.")
 	fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name")
 	fs.DurationVar(&o.DeschedulingInterval.Duration, "descheduling-interval", defaultDeschedulingInterval, "Time interval between two consecutive descheduler executions. Setting this value instructs the descheduler to run in a continuous loop at the interval specified.")
 	fs.DurationVar(&o.UnschedulableThreshold.Duration, "unschedulable-threshold", defaultUnschedulableThreshold, "The period of pod unschedulable condition. This value is considered as a classification standard of unschedulable replicas.")

cmd/scheduler-estimator/app/options/options.go

@@ -40,6 +40,16 @@ type Options struct {
 	SecurePort int
 	// ServerPort is the port that the server gRPC serves at.
 	ServerPort int
+	// ClientCertAuth when this is set, server will check all incoming HTTPS requests for a client certificate signed by the trusted CA,
+	// requests that don’t supply a valid client certificate will fail. If authentication is enabled,
+	// the certificate provides credentials for the user name given by the Common Name field.
+	ClientCertAuth bool
+	// CertFile the certificate used for SSL/TLS connections.
+	CertFile string
+	// KeyFile the key for the certificate.
+	KeyFile string
+	// TrustedCAFile Trusted certificate authority.
+	TrustedCAFile string
 	// ClusterAPIQPS is the QPS to use while talking with cluster kube-apiserver.
 	ClusterAPIQPS float32
 	// ClusterAPIBurst is the burst to allow while talking with cluster kube-apiserver.
@@ -64,6 +74,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&o.ClusterName, "cluster-name", o.ClusterName, "Name of member cluster that the estimator serves for.")
 	fs.StringVar(&o.BindAddress, "bind-address", defaultBindAddress, "The IP address on which to listen for the --secure-port port.")
 	fs.IntVar(&o.ServerPort, "server-port", defaultServerPort, "The secure port on which to serve gRPC.")
+	fs.StringVar(&o.CertFile, "cert-file", "", "The certificate used for SSL/TLS connections.")
+	fs.StringVar(&o.KeyFile, "key-file", "", "The key for the certificate.")
+	fs.BoolVar(&o.ClientCertAuth, "client-cert-auth", false, "When this is set, server will check all incoming HTTPS requests for a client certificate signed by the trusted CA, requests that don’t supply a valid client certificate will fail. If authentication is enabled, the certificate provides credentials for the user name given by the Common Name field.")
+	fs.StringVar(&o.TrustedCAFile, "trusted-ca-file", "", "Trusted certificate authority.")
 	fs.IntVar(&o.SecurePort, "secure-port", defaultHealthzPort, "The secure port on which to serve HTTPS.")
 	fs.Float32Var(&o.ClusterAPIQPS, "kube-api-qps", 20.0, "QPS to use while talking with apiserver.")
 	fs.IntVar(&o.ClusterAPIBurst, "kube-api-burst", 30, "Burst to use while talking with apiserver.")

cmd/scheduler/app/options/options.go

@@ -71,6 +71,14 @@ type Options struct {
 	SchedulerEstimatorServicePrefix string
 	// SchedulerEstimatorPort is the port that the accurate scheduler estimator server serves at.
 	SchedulerEstimatorPort int
+	// InsecureSkipVerify controls whether verifies the grpc server's certificate chain and host name.
+	InsecureSkipVerify bool
+	// CertFile the certificate used for SSL/TLS connections.
+	CertFile string
+	// KeyFile the key for the certificate.
+	KeyFile string
+	// TrustedCAFile Trusted certificate authority.
+	TrustedCAFile string
 
 	// EnableEmptyWorkloadPropagation represents whether workload with 0 replicas could be propagated to member clusters.
 	EnableEmptyWorkloadPropagation bool
@@ -138,6 +146,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 	fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.")
 	fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name")
 	fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.")
+	fs.StringVar(&o.CertFile, "cert-file", "", "The certificate used for SSL/TLS connections.")
+	fs.StringVar(&o.KeyFile, "key-file", "", "The key for the certificate.")
+	fs.StringVar(&o.TrustedCAFile, "trusted-ca-file", "", "Trusted certificate authority.")
+	fs.BoolVar(&o.InsecureSkipVerify, "insecure-skip-verify", false, "Controls whether verifies the grpc server's certificate chain and host name.")
 	fs.BoolVar(&o.EnableEmptyWorkloadPropagation, "enable-empty-workload-propagation", false, "Enable workload with replicas 0 to be propagated to member clusters.")
 	fs.StringSliceVar(&o.Plugins, "plugins", []string{"*"},
 		fmt.Sprintf("A list of plugins to enable. '*' enables all build-in and customized plugins, 'foo' enables the plugin named 'foo', '*,-foo' disables the plugin named 'foo'.\nAll build-in plugins: %s.", strings.Join(frameworkplugins.NewInTreeRegistry().FactoryNames(), ",")))

cmd/scheduler/app/scheduler.go

@@ -170,7 +170,7 @@ func run(opts *options.Options, stopChan <-chan struct{}, registryOptions ...Opt
 		scheduler.WithEnableSchedulerEstimator(opts.EnableSchedulerEstimator),
 		scheduler.WithDisableSchedulerEstimatorInPullMode(opts.DisableSchedulerEstimatorInPullMode),
 		scheduler.WithSchedulerEstimatorServicePrefix(opts.SchedulerEstimatorServicePrefix),
-		scheduler.WithSchedulerEstimatorPort(opts.SchedulerEstimatorPort),
+		scheduler.WithGRPC(opts.SchedulerEstimatorPort, opts.CertFile, opts.KeyFile, opts.TrustedCAFile, opts.InsecureSkipVerify),
 		scheduler.WithSchedulerEstimatorTimeout(opts.SchedulerEstimatorTimeout),
 		scheduler.WithEnableEmptyWorkloadPropagation(opts.EnableEmptyWorkloadPropagation),
 		scheduler.WithEnableSchedulerPlugin(opts.Plugins),

operator/pkg/controlplane/manifests.go

@@ -188,6 +188,9 @@ spec:
         - --secure-port=10351
         - --enable-scheduler-estimator=true
         - --leader-elect-resource-namespace={{ .SystemNamespace }}
+        - --trusted-ca-file=/etc/karmada/pki/ca.crt
+        - --cert-file=/etc/karmada/pki/karmada.crt
+        - --key-file=/etc/karmada/pki/karmada.key
         - --v=4
         livenessProbe:
           httpGet:
@@ -199,10 +202,16 @@ spec:
           periodSeconds: 15
           timeoutSeconds: 5
         volumeMounts:
+        - name: karmada-certs
+          mountPath: /etc/karmada/pki
+          readOnly: true
         - name: kubeconfig
           subPath: kubeconfig
           mountPath: /etc/karmada/kubeconfig
       volumes:
+        - name: karmada-certs
+          secret:
+            secretName: {{ .KarmadaCertsSecret }}
         - name: kubeconfig
           secret:
             secretName: {{ .KubeconfigSecret }}
@@ -241,6 +250,9 @@ spec:
         - --kubeconfig=/etc/karmada/kubeconfig
         - --bind-address=0.0.0.0
         - --leader-elect-resource-namespace={{ .SystemNamespace }}
+        - --trusted-ca-file=/etc/karmada/pki/ca.crt
+        - --cert-file=/etc/karmada/pki/karmada.crt
+        - --key-file=/etc/karmada/pki/karmada.key
         - --v=4
         livenessProbe:
           httpGet:
@@ -252,10 +264,16 @@ spec:
           periodSeconds: 15
           timeoutSeconds: 5
         volumeMounts:
+        - name: karmada-certs
+          mountPath: /etc/karmada/pki
+          readOnly: true
         - name: kubeconfig
           subPath: kubeconfig
           mountPath: /etc/karmada/kubeconfig
       volumes:
+        - name: karmada-certs
+          secret:
+            secretName: {{ .KarmadaCertsSecret }}
         - name: kubeconfig
           secret:
             secretName: {{ .KubeconfigSecret }}

pkg/descheduler/descheduler.go

@@ -45,6 +45,7 @@ import (
 	"github.com/karmada-io/karmada/pkg/util"
 	"github.com/karmada-io/karmada/pkg/util/fedinformer"
 	"github.com/karmada-io/karmada/pkg/util/gclient"
+	grpcutil "github.com/karmada-io/karmada/pkg/util/grpc"
 )
 
 const (
@@ -65,7 +66,7 @@ type Descheduler struct {
 
 	schedulerEstimatorCache         *estimatorclient.SchedulerEstimatorCache
 	schedulerEstimatorServicePrefix string
-	schedulerEstimatorPort          int
+	grpcConfig                      *grpcutil.Config
 	schedulerEstimatorWorker        util.AsyncWorker
 
 	unschedulableThreshold time.Duration
@@ -77,15 +78,21 @@ type Descheduler struct {
 func NewDescheduler(karmadaClient karmadaclientset.Interface, kubeClient kubernetes.Interface, opts *options.Options) *Descheduler {
 	factory := informerfactory.NewSharedInformerFactory(karmadaClient, 0)
 	desched := &Descheduler{
-		KarmadaClient:                   karmadaClient,
-		KubeClient:                      kubeClient,
-		informerFactory:                 factory,
-		bindingInformer:                 factory.Work().V1alpha2().ResourceBindings().Informer(),
-		bindingLister:                   factory.Work().V1alpha2().ResourceBindings().Lister(),
-		clusterInformer:                 factory.Cluster().V1alpha1().Clusters().Informer(),
-		clusterLister:                   factory.Cluster().V1alpha1().Clusters().Lister(),
-		schedulerEstimatorCache:         estimatorclient.NewSchedulerEstimatorCache(),
-		schedulerEstimatorPort:          opts.SchedulerEstimatorPort,
+		KarmadaClient:           karmadaClient,
+		KubeClient:              kubeClient,
+		informerFactory:         factory,
+		bindingInformer:         factory.Work().V1alpha2().ResourceBindings().Informer(),
+		bindingLister:           factory.Work().V1alpha2().ResourceBindings().Lister(),
+		clusterInformer:         factory.Cluster().V1alpha1().Clusters().Informer(),
+		clusterLister:           factory.Cluster().V1alpha1().Clusters().Lister(),
+		schedulerEstimatorCache: estimatorclient.NewSchedulerEstimatorCache(),
+		grpcConfig: &grpcutil.Config{
+			ServerPort:         opts.SchedulerEstimatorPort,
+			TrustedCAFile:      opts.TrustedCAFile,
+			CertFile:           opts.CertFile,
+			KeyFile:            opts.KeyFile,
+			InsecureSkipVerify: opts.InsecureSkipVerify,
+		},
 		schedulerEstimatorServicePrefix: opts.SchedulerEstimatorServicePrefix,
 		unschedulableThreshold:          opts.UnschedulableThreshold.Duration,
 		deschedulingInterval:            opts.DeschedulingInterval.Duration,
@@ -273,7 +280,7 @@ func (d *Descheduler) establishEstimatorConnections() {
 		return
 	}
 	for i := range clusterList.Items {
-		if err = estimatorclient.EstablishConnection(d.KubeClient, clusterList.Items[i].Name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorPort); err != nil {
+		if err = estimatorclient.EstablishConnection(d.KubeClient, clusterList.Items[i].Name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.grpcConfig); err != nil {
 			klog.Error(err)
 		}
 	}
@@ -293,7 +300,7 @@ func (d *Descheduler) reconcileEstimatorConnection(key util.QueueKey) error {
 		}
 		return err
 	}
-	return estimatorclient.EstablishConnection(d.KubeClient, name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorPort)
+	return estimatorclient.EstablishConnection(d.KubeClient, name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.grpcConfig)
 }
 
 func (d *Descheduler) recordDescheduleResultEventForResourceBinding(rb *workv1alpha2.ResourceBinding, message string, err error) {