From 3bce7367c4bb3d83dc82886ae50d23a545e37e8c Mon Sep 17 00:00:00 2001 From: zhzhuang-zju Date: Wed, 12 Jun 2024 12:04:10 +0800 Subject: [PATCH] Adding TLS Certificate Authentication to gRPC Signed-off-by: zhzhuang-zju --- artifacts/deploy/karmada-descheduler.yaml | 9 ++ .../deploy/karmada-scheduler-estimator.yaml | 10 ++ artifacts/deploy/karmada-scheduler.yaml | 9 ++ charts/karmada/templates/_helpers.tpl | 7 + .../templates/karmada-descheduler.yaml | 7 + .../karmada-scheduler-estimator.yaml | 8 ++ .../karmada/templates/karmada-scheduler.yaml | 7 + cmd/descheduler/app/options/options.go | 12 ++ .../app/options/options.go | 14 ++ cmd/scheduler/app/options/options.go | 12 ++ cmd/scheduler/app/scheduler.go | 2 +- operator/pkg/controlplane/manifests.go | 18 +++ pkg/descheduler/descheduler.go | 31 +++-- pkg/estimator/client/cache.go | 7 +- pkg/estimator/client/service.go | 2 +- pkg/estimator/server/server.go | 24 +++- .../addons/descheduler/manifests.go | 9 ++ pkg/karmadactl/addons/estimator/manifests.go | 10 ++ .../cmdinit/kubernetes/deployments.go | 16 +++ pkg/scheduler/scheduler.go | 34 +++-- pkg/scheduler/scheduler_test.go | 10 +- pkg/util/dial.go | 43 ------ pkg/util/grpc/grpc.go | 131 ++++++++++++++++++ 23 files changed, 347 insertions(+), 85 deletions(-) delete mode 100644 pkg/util/dial.go create mode 100644 pkg/util/grpc/grpc.go diff --git a/artifacts/deploy/karmada-descheduler.yaml b/artifacts/deploy/karmada-descheduler.yaml index c451a37b4312..db509055d6a7 100644 --- a/artifacts/deploy/karmada-descheduler.yaml +++ b/artifacts/deploy/karmada-descheduler.yaml @@ -27,6 +27,9 @@ spec: - /bin/karmada-descheduler - --kubeconfig=/etc/kubeconfig - --bind-address=0.0.0.0 + - --trusted-ca-file=/etc/karmada/pki/ca.crt + - --cert-file=/etc/karmada/pki/karmada.crt + - --key-file=/etc/karmada/pki/karmada.key - --v=4 livenessProbe: httpGet: @@ -38,10 +41,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/kubeconfig volumes: + - name: karmada-certs + secret: + secretName: karmada-cert-secret - name: kubeconfig secret: secretName: kubeconfig diff --git a/artifacts/deploy/karmada-scheduler-estimator.yaml b/artifacts/deploy/karmada-scheduler-estimator.yaml index ce4af9361e1a..a81d0414905b 100644 --- a/artifacts/deploy/karmada-scheduler-estimator.yaml +++ b/artifacts/deploy/karmada-scheduler-estimator.yaml @@ -27,6 +27,10 @@ spec: - /bin/karmada-scheduler-estimator - --kubeconfig=/etc/{{member_cluster_name}}-kubeconfig - --cluster-name={{member_cluster_name}} + - --cert-file=/etc/karmada/pki/karmada.crt + - --key-file=/etc/karmada/pki/karmada.key + - --client-cert-auth=true + - --trusted-ca-file=/etc/karmada/pki/ca.crt livenessProbe: httpGet: path: /healthz @@ -37,10 +41,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: member-kubeconfig subPath: {{member_cluster_name}}-kubeconfig mountPath: /etc/{{member_cluster_name}}-kubeconfig volumes: + - name: karmada-certs + secret: + secretName: karmada-cert-secret - name: member-kubeconfig secret: secretName: {{member_cluster_name}}-kubeconfig diff --git a/artifacts/deploy/karmada-scheduler.yaml b/artifacts/deploy/karmada-scheduler.yaml index 1401e7a4cca0..5db522a96439 100644 --- a/artifacts/deploy/karmada-scheduler.yaml +++ b/artifacts/deploy/karmada-scheduler.yaml @@ -39,11 +39,20 @@ spec: - --secure-port=10351 - --enable-scheduler-estimator=true - --v=4 + - --trusted-ca-file=/etc/karmada/pki/ca.crt + - --cert-file=/etc/karmada/pki/karmada.crt + - --key-file=/etc/karmada/pki/karmada.key volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/kubeconfig volumes: + - name: karmada-certs + secret: + secretName: karmada-cert-secret - name: kubeconfig secret: secretName: kubeconfig diff --git a/charts/karmada/templates/_helpers.tpl b/charts/karmada/templates/_helpers.tpl index 3753e814cfd4..2d129b7cea09 100644 --- a/charts/karmada/templates/_helpers.tpl +++ b/charts/karmada/templates/_helpers.tpl @@ -349,6 +349,13 @@ app: {{- include "karmada.name" .}}-search {{- end }} {{- end -}} +{{- define "karmada.scheduler.cert.volume" -}} +{{ $name := include "karmada.name" . }} +- name: karmada-certs + secret: + secretName: {{ $name }}-cert +{{- end -}} + {{/* Return the proper karmada internal etcd image name */}} diff --git a/charts/karmada/templates/karmada-descheduler.yaml b/charts/karmada/templates/karmada-descheduler.yaml index c55109244aa3..0829f3f7ec00 100644 --- a/charts/karmada/templates/karmada-descheduler.yaml +++ b/charts/karmada/templates/karmada-descheduler.yaml @@ -52,6 +52,9 @@ spec: - --kubeconfig=/etc/kubeconfig - --bind-address=0.0.0.0 - --leader-elect-resource-namespace={{ $systemNamespace }} + - --trusted-ca-file=/etc/karmada/pki/ca.crt + - --cert-file=/etc/karmada/pki/karmada.crt + - --key-file=/etc/karmada/pki/karmada.key - --v=4 livenessProbe: httpGet: @@ -63,12 +66,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true {{- include "karmada.kubeconfig.volumeMount" . | nindent 12 }} resources: {{- toYaml .Values.descheduler.resources | nindent 12 }} volumes: {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.descheduler.kubeconfig.volume" . | nindent 8 }} + {{- include "karmada.scheduler.cert.volume" . | nindent 8 }} {{ if .Values.descheduler.podDisruptionBudget }} --- diff --git a/charts/karmada/templates/karmada-scheduler-estimator.yaml b/charts/karmada/templates/karmada-scheduler-estimator.yaml index 33bdd7846911..036f3a38b387 100644 --- a/charts/karmada/templates/karmada-scheduler-estimator.yaml +++ b/charts/karmada/templates/karmada-scheduler-estimator.yaml @@ -48,6 +48,10 @@ spec: - /bin/karmada-scheduler-estimator - --kubeconfig=/etc/{{ $clusterName }}-kubeconfig - --cluster-name={{ $clusterName }} + - --cert-file=/etc/karmada/pki/karmada.crt + - --key-file=/etc/karmada/pki/karmada.key + - --client-cert-auth=true + - --trusted-ca-file=/etc/karmada/pki/ca.crt {{- with (include "karmada.schedulerEstimator.featureGates" (dict "featureGatesArg" $.Values.schedulerEstimator.featureGates)) }} - {{ . }} {{- end}} @@ -61,12 +65,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: member-kubeconfig subPath: {{ $clusterName }}-kubeconfig mountPath: /etc/{{ $clusterName }}-kubeconfig resources: {{- toYaml $.Values.schedulerEstimator.resources | nindent 12 }} volumes: + {{- include "karmada.scheduler.cert.volume" $ | nindent 8 }} - name: member-kubeconfig secret: secretName: {{ $clusterName }}-kubeconfig diff --git a/charts/karmada/templates/karmada-scheduler.yaml b/charts/karmada/templates/karmada-scheduler.yaml index a6933df792f4..d6172c31f4eb 100644 --- a/charts/karmada/templates/karmada-scheduler.yaml +++ b/charts/karmada/templates/karmada-scheduler.yaml @@ -53,6 +53,9 @@ spec: - --bind-address=0.0.0.0 - --secure-port=10351 - --leader-elect-resource-namespace={{ $systemNamespace }} + - --trusted-ca-file=/etc/karmada/pki/ca.crt + - --cert-file=/etc/karmada/pki/karmada.crt + - --key-file=/etc/karmada/pki/karmada.key livenessProbe: httpGet: path: /healthz @@ -63,12 +66,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true {{- include "karmada.kubeconfig.volumeMount" . | nindent 12 }} resources: {{- toYaml .Values.scheduler.resources | nindent 12 }} volumes: {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.kubeconfig.volume" . | nindent 8 }} + {{- include "karmada.scheduler.cert.volume" . | nindent 8 }} {{ if .Values.scheduler.podDisruptionBudget }} --- diff --git a/cmd/descheduler/app/options/options.go b/cmd/descheduler/app/options/options.go index 0dd0d0b816cb..90a1cca3a2d8 100644 --- a/cmd/descheduler/app/options/options.go +++ b/cmd/descheduler/app/options/options.go @@ -62,6 +62,14 @@ type Options struct { SchedulerEstimatorServicePrefix string // SchedulerEstimatorPort is the port that the accurate scheduler estimator server serves at. SchedulerEstimatorPort int + // InsecureSkipVerify controls whether verifies the grpc server's certificate chain and host name. + InsecureSkipVerify bool + // CertFile the certificate used for SSL/TLS connections. + CertFile string + // KeyFile the key for the certificate. + KeyFile string + // TrustedCAFile Trusted certificate authority. + TrustedCAFile string // DeschedulingInterval specifies time interval for descheduler to run. DeschedulingInterval metav1.Duration // UnschedulableThreshold specifies the period of pod unschedulable condition. @@ -99,6 +107,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver.") fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.") fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.") + fs.StringVar(&o.CertFile, "cert-file", "", "The certificate used for SSL/TLS connections.") + fs.StringVar(&o.KeyFile, "key-file", "", "The key for the certificate.") + fs.StringVar(&o.TrustedCAFile, "trusted-ca-file", "", "Trusted certificate authority.") + fs.BoolVar(&o.InsecureSkipVerify, "insecure-skip-verify", false, "Controls whether verifies the grpc server's certificate chain and host name.") fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name") fs.DurationVar(&o.DeschedulingInterval.Duration, "descheduling-interval", defaultDeschedulingInterval, "Time interval between two consecutive descheduler executions. Setting this value instructs the descheduler to run in a continuous loop at the interval specified.") fs.DurationVar(&o.UnschedulableThreshold.Duration, "unschedulable-threshold", defaultUnschedulableThreshold, "The period of pod unschedulable condition. This value is considered as a classification standard of unschedulable replicas.") diff --git a/cmd/scheduler-estimator/app/options/options.go b/cmd/scheduler-estimator/app/options/options.go index f10bf3ca9cb2..bca1a349b45f 100644 --- a/cmd/scheduler-estimator/app/options/options.go +++ b/cmd/scheduler-estimator/app/options/options.go @@ -40,6 +40,16 @@ type Options struct { SecurePort int // ServerPort is the port that the server gRPC serves at. ServerPort int + // ClientCertAuth when this is set, server will check all incoming HTTPS requests for a client certificate signed by the trusted CA, + // requests that don’t supply a valid client certificate will fail. If authentication is enabled, + // the certificate provides credentials for the user name given by the Common Name field. + ClientCertAuth bool + // CertFile the certificate used for SSL/TLS connections. + CertFile string + // KeyFile the key for the certificate. + KeyFile string + // TrustedCAFile Trusted certificate authority. + TrustedCAFile string // ClusterAPIQPS is the QPS to use while talking with cluster kube-apiserver. ClusterAPIQPS float32 // ClusterAPIBurst is the burst to allow while talking with cluster kube-apiserver. @@ -64,6 +74,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.ClusterName, "cluster-name", o.ClusterName, "Name of member cluster that the estimator serves for.") fs.StringVar(&o.BindAddress, "bind-address", defaultBindAddress, "The IP address on which to listen for the --secure-port port.") fs.IntVar(&o.ServerPort, "server-port", defaultServerPort, "The secure port on which to serve gRPC.") + fs.StringVar(&o.CertFile, "cert-file", "", "The certificate used for SSL/TLS connections.") + fs.StringVar(&o.KeyFile, "key-file", "", "The key for the certificate.") + fs.BoolVar(&o.ClientCertAuth, "client-cert-auth", false, "When this is set, server will check all incoming HTTPS requests for a client certificate signed by the trusted CA, requests that don’t supply a valid client certificate will fail. If authentication is enabled, the certificate provides credentials for the user name given by the Common Name field.") + fs.StringVar(&o.TrustedCAFile, "trusted-ca-file", "", "Trusted certificate authority.") fs.IntVar(&o.SecurePort, "secure-port", defaultHealthzPort, "The secure port on which to serve HTTPS.") fs.Float32Var(&o.ClusterAPIQPS, "kube-api-qps", 20.0, "QPS to use while talking with apiserver.") fs.IntVar(&o.ClusterAPIBurst, "kube-api-burst", 30, "Burst to use while talking with apiserver.") diff --git a/cmd/scheduler/app/options/options.go b/cmd/scheduler/app/options/options.go index bcb6b2e59634..ae1527779236 100644 --- a/cmd/scheduler/app/options/options.go +++ b/cmd/scheduler/app/options/options.go @@ -71,6 +71,14 @@ type Options struct { SchedulerEstimatorServicePrefix string // SchedulerEstimatorPort is the port that the accurate scheduler estimator server serves at. SchedulerEstimatorPort int + // InsecureSkipVerify controls whether verifies the grpc server's certificate chain and host name. + InsecureSkipVerify bool + // CertFile the certificate used for SSL/TLS connections. + CertFile string + // KeyFile the key for the certificate. + KeyFile string + // TrustedCAFile Trusted certificate authority. + TrustedCAFile string // EnableEmptyWorkloadPropagation represents whether workload with 0 replicas could be propagated to member clusters. EnableEmptyWorkloadPropagation bool @@ -138,6 +146,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.") fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name") fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.") + fs.StringVar(&o.CertFile, "cert-file", "", "The certificate used for SSL/TLS connections.") + fs.StringVar(&o.KeyFile, "key-file", "", "The key for the certificate.") + fs.StringVar(&o.TrustedCAFile, "trusted-ca-file", "", "Trusted certificate authority.") + fs.BoolVar(&o.InsecureSkipVerify, "insecure-skip-verify", false, "Controls whether verifies the grpc server's certificate chain and host name.") fs.BoolVar(&o.EnableEmptyWorkloadPropagation, "enable-empty-workload-propagation", false, "Enable workload with replicas 0 to be propagated to member clusters.") fs.StringSliceVar(&o.Plugins, "plugins", []string{"*"}, fmt.Sprintf("A list of plugins to enable. '*' enables all build-in and customized plugins, 'foo' enables the plugin named 'foo', '*,-foo' disables the plugin named 'foo'.\nAll build-in plugins: %s.", strings.Join(frameworkplugins.NewInTreeRegistry().FactoryNames(), ","))) diff --git a/cmd/scheduler/app/scheduler.go b/cmd/scheduler/app/scheduler.go index 826adbf99a0c..a0a5d3d45709 100644 --- a/cmd/scheduler/app/scheduler.go +++ b/cmd/scheduler/app/scheduler.go @@ -170,7 +170,7 @@ func run(opts *options.Options, stopChan <-chan struct{}, registryOptions ...Opt scheduler.WithEnableSchedulerEstimator(opts.EnableSchedulerEstimator), scheduler.WithDisableSchedulerEstimatorInPullMode(opts.DisableSchedulerEstimatorInPullMode), scheduler.WithSchedulerEstimatorServicePrefix(opts.SchedulerEstimatorServicePrefix), - scheduler.WithSchedulerEstimatorPort(opts.SchedulerEstimatorPort), + scheduler.WithGRPC(opts.SchedulerEstimatorPort, opts.CertFile, opts.KeyFile, opts.TrustedCAFile, opts.InsecureSkipVerify), scheduler.WithSchedulerEstimatorTimeout(opts.SchedulerEstimatorTimeout), scheduler.WithEnableEmptyWorkloadPropagation(opts.EnableEmptyWorkloadPropagation), scheduler.WithEnableSchedulerPlugin(opts.Plugins), diff --git a/operator/pkg/controlplane/manifests.go b/operator/pkg/controlplane/manifests.go index e7480b91d25d..9cb5655edf01 100644 --- a/operator/pkg/controlplane/manifests.go +++ b/operator/pkg/controlplane/manifests.go @@ -188,6 +188,9 @@ spec: - --secure-port=10351 - --enable-scheduler-estimator=true - --leader-elect-resource-namespace={{ .SystemNamespace }} + - --trusted-ca-file=/etc/karmada/pki/ca.crt + - --cert-file=/etc/karmada/pki/karmada.crt + - --key-file=/etc/karmada/pki/karmada.key - --v=4 livenessProbe: httpGet: @@ -199,10 +202,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/karmada/kubeconfig volumes: + - name: karmada-certs + secret: + secretName: {{ .KarmadaCertsSecret }} - name: kubeconfig secret: secretName: {{ .KubeconfigSecret }} @@ -241,6 +250,9 @@ spec: - --kubeconfig=/etc/karmada/kubeconfig - --bind-address=0.0.0.0 - --leader-elect-resource-namespace={{ .SystemNamespace }} + - --trusted-ca-file=/etc/karmada/pki/ca.crt + - --cert-file=/etc/karmada/pki/karmada.crt + - --key-file=/etc/karmada/pki/karmada.key - --v=4 livenessProbe: httpGet: @@ -252,10 +264,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: karmada-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/karmada/kubeconfig volumes: + - name: karmada-certs + secret: + secretName: {{ .KarmadaCertsSecret }} - name: kubeconfig secret: secretName: {{ .KubeconfigSecret }} diff --git a/pkg/descheduler/descheduler.go b/pkg/descheduler/descheduler.go index f5689672fa81..f7de7fdb10e8 100644 --- a/pkg/descheduler/descheduler.go +++ b/pkg/descheduler/descheduler.go @@ -45,6 +45,7 @@ import ( "github.com/karmada-io/karmada/pkg/util" "github.com/karmada-io/karmada/pkg/util/fedinformer" "github.com/karmada-io/karmada/pkg/util/gclient" + grpcutil "github.com/karmada-io/karmada/pkg/util/grpc" ) const ( @@ -65,7 +66,7 @@ type Descheduler struct { schedulerEstimatorCache *estimatorclient.SchedulerEstimatorCache schedulerEstimatorServicePrefix string - schedulerEstimatorPort int + grpcConfig *grpcutil.Config schedulerEstimatorWorker util.AsyncWorker unschedulableThreshold time.Duration @@ -77,15 +78,21 @@ type Descheduler struct { func NewDescheduler(karmadaClient karmadaclientset.Interface, kubeClient kubernetes.Interface, opts *options.Options) *Descheduler { factory := informerfactory.NewSharedInformerFactory(karmadaClient, 0) desched := &Descheduler{ - KarmadaClient: karmadaClient, - KubeClient: kubeClient, - informerFactory: factory, - bindingInformer: factory.Work().V1alpha2().ResourceBindings().Informer(), - bindingLister: factory.Work().V1alpha2().ResourceBindings().Lister(), - clusterInformer: factory.Cluster().V1alpha1().Clusters().Informer(), - clusterLister: factory.Cluster().V1alpha1().Clusters().Lister(), - schedulerEstimatorCache: estimatorclient.NewSchedulerEstimatorCache(), - schedulerEstimatorPort: opts.SchedulerEstimatorPort, + KarmadaClient: karmadaClient, + KubeClient: kubeClient, + informerFactory: factory, + bindingInformer: factory.Work().V1alpha2().ResourceBindings().Informer(), + bindingLister: factory.Work().V1alpha2().ResourceBindings().Lister(), + clusterInformer: factory.Cluster().V1alpha1().Clusters().Informer(), + clusterLister: factory.Cluster().V1alpha1().Clusters().Lister(), + schedulerEstimatorCache: estimatorclient.NewSchedulerEstimatorCache(), + grpcConfig: &grpcutil.Config{ + ServerPort: opts.SchedulerEstimatorPort, + TrustedCAFile: opts.TrustedCAFile, + CertFile: opts.CertFile, + KeyFile: opts.KeyFile, + InsecureSkipVerify: opts.InsecureSkipVerify, + }, schedulerEstimatorServicePrefix: opts.SchedulerEstimatorServicePrefix, unschedulableThreshold: opts.UnschedulableThreshold.Duration, deschedulingInterval: opts.DeschedulingInterval.Duration, @@ -273,7 +280,7 @@ func (d *Descheduler) establishEstimatorConnections() { return } for i := range clusterList.Items { - if err = estimatorclient.EstablishConnection(d.KubeClient, clusterList.Items[i].Name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorPort); err != nil { + if err = estimatorclient.EstablishConnection(d.KubeClient, clusterList.Items[i].Name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.grpcConfig); err != nil { klog.Error(err) } } @@ -293,7 +300,7 @@ func (d *Descheduler) reconcileEstimatorConnection(key util.QueueKey) error { } return err } - return estimatorclient.EstablishConnection(d.KubeClient, name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorPort) + return estimatorclient.EstablishConnection(d.KubeClient, name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.grpcConfig) } func (d *Descheduler) recordDescheduleResultEventForResourceBinding(rb *workv1alpha2.ResourceBinding, message string, err error) { diff --git a/pkg/estimator/client/cache.go b/pkg/estimator/client/cache.go index 3c278e240f0d..d4809f7862fd 100644 --- a/pkg/estimator/client/cache.go +++ b/pkg/estimator/client/cache.go @@ -27,6 +27,7 @@ import ( estimatorservice "github.com/karmada-io/karmada/pkg/estimator/service" "github.com/karmada-io/karmada/pkg/util" + grpcutil "github.com/karmada-io/karmada/pkg/util/grpc" "github.com/karmada-io/karmada/pkg/util/names" ) @@ -96,19 +97,19 @@ func (c *SchedulerEstimatorCache) GetClient(name string) (estimatorservice.Estim } // EstablishConnection establishes a new gRPC connection with the specified cluster scheduler estimator. -func EstablishConnection(kubeClient kubernetes.Interface, name string, estimatorCache *SchedulerEstimatorCache, estimatorServicePrefix string, port int) error { +func EstablishConnection(kubeClient kubernetes.Interface, name string, estimatorCache *SchedulerEstimatorCache, estimatorServicePrefix string, grpcConfig *grpcutil.Config) error { if estimatorCache.IsEstimatorExist(name) { return nil } serverAddr, err := resolveCluster(kubeClient, util.NamespaceKarmadaSystem, - names.GenerateEstimatorServiceName(estimatorServicePrefix, name), int32(port)) + names.GenerateEstimatorServiceName(estimatorServicePrefix, name), int32(grpcConfig.ServerPort)) if err != nil { return err } klog.Infof("Start dialing estimator server(%s) of cluster(%s).", serverAddr, name) - cc, err := util.Dial(serverAddr, 5*time.Second) + cc, err := grpcConfig.DialWithTimeOut(serverAddr, 5*time.Second) if err != nil { klog.Errorf("Failed to dial cluster(%s): %v.", name, err) return err diff --git a/pkg/estimator/client/service.go b/pkg/estimator/client/service.go index 323cd7ca8661..c1216557763c 100644 --- a/pkg/estimator/client/service.go +++ b/pkg/estimator/client/service.go @@ -41,7 +41,7 @@ func resolveCluster(kubeClient kubernetes.Interface, namespace, id string, port * But the Service resource is defined in Host Kubernetes Cluster. So we cannot get its content here. * The best thing we can do is just glue host:port together, and try to connect to it. */ - return net.JoinHostPort(id, fmt.Sprintf("%d", port)), nil + return net.JoinHostPort(fmt.Sprintf("%s.%s.svc.cluster.local", id, namespace), fmt.Sprintf("%d", port)), nil } return "", err diff --git a/pkg/estimator/server/server.go b/pkg/estimator/server/server.go index d8b6156c7f26..10b6756d9e9c 100644 --- a/pkg/estimator/server/server.go +++ b/pkg/estimator/server/server.go @@ -23,7 +23,6 @@ import ( "time" "github.com/kr/pretty" - "google.golang.org/grpc" "google.golang.org/grpc/metadata" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -53,6 +52,7 @@ import ( "github.com/karmada-io/karmada/pkg/util/fedinformer" "github.com/karmada-io/karmada/pkg/util/fedinformer/genericmanager" "github.com/karmada-io/karmada/pkg/util/fedinformer/keys" + grpcutil "github.com/karmada-io/karmada/pkg/util/grpc" "github.com/karmada-io/karmada/pkg/util/helper" schedcache "github.com/karmada-io/karmada/pkg/util/lifted/scheduler/cache" "github.com/karmada-io/karmada/pkg/util/lifted/scheduler/framework/parallelize" @@ -73,7 +73,6 @@ var ( // AccurateSchedulerEstimatorServer is the gRPC server of a cluster accurate scheduler estimator. // Please see https://github.com/karmada-io/karmada/pull/580 (#580). type AccurateSchedulerEstimatorServer struct { - port int clusterName string kubeClient kubernetes.Interface restMapper meta.RESTMapper @@ -85,6 +84,8 @@ type AccurateSchedulerEstimatorServer struct { estimateFramework framework.Framework Cache schedcache.Cache + + GrpcConfig *grpcutil.Config } // NewEstimatorServer creates an instance of AccurateSchedulerEstimatorServer. @@ -101,7 +102,6 @@ func NewEstimatorServer( informerFactory.InformerFor(&corev1.Pod{}, newPodInformer) es := &AccurateSchedulerEstimatorServer{ - port: opts.ServerPort, clusterName: opts.ClusterName, kubeClient: kubeClient, restMapper: restMapper, @@ -113,6 +113,13 @@ func NewEstimatorServer( }, parallelizer: parallelize.NewParallelizer(opts.Parallelism), Cache: schedcache.New(durationToExpireAssumedPod, stopChan), + GrpcConfig: &grpcutil.Config{ + ClientCertAuth: opts.ClientCertAuth, + ServerPort: opts.ServerPort, + TrustedCAFile: opts.TrustedCAFile, + CertFile: opts.CertFile, + KeyFile: opts.KeyFile, + }, } // ignore the error here because the informers haven't been started _ = informerFactory.Core().V1().Nodes().Informer().SetTransform(fedinformer.StripUnusedFields) @@ -154,14 +161,17 @@ func (es *AccurateSchedulerEstimatorServer) Start(ctx context.Context) error { } // Listen a port and register the gRPC server. - l, err := net.Listen("tcp", fmt.Sprintf(":%d", es.port)) + l, err := net.Listen("tcp", fmt.Sprintf(":%d", es.GrpcConfig.ServerPort)) if err != nil { - return fmt.Errorf("failed to listen port %d: %v", es.port, err) + return fmt.Errorf("failed to listen port %d: %v", es.GrpcConfig.ServerPort, err) } - klog.Infof("Listening port: %d", es.port) + klog.Infof("Listening port: %d", es.GrpcConfig.ServerPort) defer l.Close() - s := grpc.NewServer() + s, err := es.GrpcConfig.NewServer() + if err != nil { + return fmt.Errorf("failed to create grpc server: %v", err) + } estimatorservice.RegisterEstimatorServer(s, es) // Graceful stop when the context is cancelled. diff --git a/pkg/karmadactl/addons/descheduler/manifests.go b/pkg/karmadactl/addons/descheduler/manifests.go index af1bd357c56b..0c0a9fe3e70f 100644 --- a/pkg/karmadactl/addons/descheduler/manifests.go +++ b/pkg/karmadactl/addons/descheduler/manifests.go @@ -47,6 +47,9 @@ spec: - --kubeconfig=/etc/kubeconfig - --bind-address=0.0.0.0 - --leader-elect-resource-namespace={{ .Namespace }} + - --trusted-ca-file=/etc/karmada/pki/ca.crt + - --cert-file=/etc/karmada/pki/karmada.crt + - --key-file=/etc/karmada/pki/karmada.key - --v=4 livenessProbe: httpGet: @@ -58,10 +61,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: k8s-certs + mountPath: /etc/karmada/pki + readOnly: true - name: kubeconfig subPath: kubeconfig mountPath: /etc/kubeconfig volumes: + - name: k8s-certs + secret: + secretName: karmada-cert - name: kubeconfig secret: secretName: kubeconfig diff --git a/pkg/karmadactl/addons/estimator/manifests.go b/pkg/karmadactl/addons/estimator/manifests.go index a11e29212f99..14834150f739 100644 --- a/pkg/karmadactl/addons/estimator/manifests.go +++ b/pkg/karmadactl/addons/estimator/manifests.go @@ -48,6 +48,10 @@ spec: - /bin/karmada-scheduler-estimator - --kubeconfig=/etc/{{ .MemberClusterName}}-kubeconfig - --cluster-name={{ .MemberClusterName}} + - --cert-file=/etc/karmada/pki/karmada.crt + - --key-file=/etc/karmada/pki/karmada.key + - --client-cert-auth=true + - --trusted-ca-file=/etc/karmada/pki/ca.crt livenessProbe: httpGet: path: /healthz @@ -58,10 +62,16 @@ spec: periodSeconds: 15 timeoutSeconds: 5 volumeMounts: + - name: k8s-certs + mountPath: /etc/karmada/pki + readOnly: true - name: member-kubeconfig subPath: {{ .MemberClusterName}}-kubeconfig mountPath: /etc/{{ .MemberClusterName}}-kubeconfig volumes: + - name: k8s-certs + secret: + secretName: karmada-cert - name: member-kubeconfig secret: secretName: {{ .MemberClusterName}}-kubeconfig diff --git a/pkg/karmadactl/cmdinit/kubernetes/deployments.go b/pkg/karmadactl/cmdinit/kubernetes/deployments.go index bb0ea0479e69..7ecbdcc4aac5 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deployments.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deployments.go @@ -454,6 +454,9 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment "--secure-port=10351", "--enable-scheduler-estimator=true", "--leader-elect=true", + "--trusted-ca-file=/etc/karmada/pki/ca.crt", + "--cert-file=/etc/karmada/pki/karmada.crt", + "--key-file=/etc/karmada/pki/karmada.key", fmt.Sprintf("--leader-elect-resource-namespace=%s", i.Namespace), "--v=4", }, @@ -465,6 +468,11 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment MountPath: kubeConfigContainerMountPath, SubPath: KubeConfigSecretAndMountName, }, + { + Name: globaloptions.KarmadaCertsName, + ReadOnly: true, + MountPath: karmadaCertsVolumeMountPath, + }, }, }, }, @@ -477,6 +485,14 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment }, }, }, + { + Name: globaloptions.KarmadaCertsName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: globaloptions.KarmadaCertsName, + }, + }, + }, }, Tolerations: []corev1.Toleration{ { diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 2f4e7737d895..bab641af1c15 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -56,6 +56,7 @@ import ( "github.com/karmada-io/karmada/pkg/scheduler/metrics" "github.com/karmada-io/karmada/pkg/sharedcli/ratelimiterflag" "github.com/karmada-io/karmada/pkg/util" + grpcutil "github.com/karmada-io/karmada/pkg/util/grpc" "github.com/karmada-io/karmada/pkg/util/helper" utilmetrics "github.com/karmada-io/karmada/pkg/util/metrics" ) @@ -106,11 +107,12 @@ type Scheduler struct { disableSchedulerEstimatorInPullMode bool schedulerEstimatorCache *estimatorclient.SchedulerEstimatorCache schedulerEstimatorServicePrefix string - schedulerEstimatorPort int schedulerEstimatorWorker util.AsyncWorker schedulerName string enableEmptyWorkloadPropagation bool + + grpcConfig *grpcutil.Config } type schedulerOptions struct { @@ -122,8 +124,6 @@ type schedulerOptions struct { schedulerEstimatorTimeout metav1.Duration // SchedulerEstimatorServicePrefix presents the prefix of the accurate scheduler estimator service name. schedulerEstimatorServicePrefix string - // schedulerEstimatorPort is the port that the accurate scheduler estimator server serves at. - schedulerEstimatorPort int // schedulerName is the name of the scheduler. Default is "default-scheduler". schedulerName string //enableEmptyWorkloadPropagation represents whether allow workload with replicas 0 propagated to member clusters should be enabled @@ -134,6 +134,8 @@ type schedulerOptions struct { plugins []string // contains the options for rate limiter. RateLimiterOptions ratelimiterflag.Options + // grpcConfig contains the configuration of GRPC. + grpcConfig *grpcutil.Config } // Option configures a Scheduler @@ -146,6 +148,19 @@ func WithEnableSchedulerEstimator(enableSchedulerEstimator bool) Option { } } +// WithGRPC sets the grpc config for scheduler +func WithGRPC(port int, certFile, keyFile, trustedCAFile string, insecureSkipVerify bool) Option { + return func(o *schedulerOptions) { + o.grpcConfig = &grpcutil.Config{ + ServerPort: port, + CertFile: certFile, + KeyFile: keyFile, + TrustedCAFile: trustedCAFile, + InsecureSkipVerify: insecureSkipVerify, + } + } +} + // WithDisableSchedulerEstimatorInPullMode sets the disableSchedulerEstimatorInPullMode for scheduler func WithDisableSchedulerEstimatorInPullMode(disableSchedulerEstimatorInPullMode bool) Option { return func(o *schedulerOptions) { @@ -167,13 +182,6 @@ func WithSchedulerEstimatorServicePrefix(schedulerEstimatorServicePrefix string) } } -// WithSchedulerEstimatorPort sets the schedulerEstimatorPort for scheduler -func WithSchedulerEstimatorPort(schedulerEstimatorPort int) Option { - return func(o *schedulerOptions) { - o.schedulerEstimatorPort = schedulerEstimatorPort - } -} - // WithSchedulerName sets the schedulerName for scheduler func WithSchedulerName(schedulerName string) Option { return func(o *schedulerOptions) { @@ -255,7 +263,7 @@ func NewScheduler(dynamicClient dynamic.Interface, karmadaClient karmadaclientse sched.enableSchedulerEstimator = options.enableSchedulerEstimator sched.disableSchedulerEstimatorInPullMode = options.disableSchedulerEstimatorInPullMode sched.schedulerEstimatorServicePrefix = options.schedulerEstimatorServicePrefix - sched.schedulerEstimatorPort = options.schedulerEstimatorPort + sched.grpcConfig = options.grpcConfig sched.schedulerEstimatorCache = estimatorclient.NewSchedulerEstimatorCache() schedulerEstimatorWorkerOptions := util.Options{ Name: "scheduler-estimator", @@ -769,7 +777,7 @@ func (s *Scheduler) reconcileEstimatorConnection(key util.QueueKey) error { return nil } - return estimatorclient.EstablishConnection(s.KubeClient, name, s.schedulerEstimatorCache, s.schedulerEstimatorServicePrefix, s.schedulerEstimatorPort) + return estimatorclient.EstablishConnection(s.KubeClient, name, s.schedulerEstimatorCache, s.schedulerEstimatorServicePrefix, s.grpcConfig) } func (s *Scheduler) establishEstimatorConnections() { @@ -782,7 +790,7 @@ func (s *Scheduler) establishEstimatorConnections() { if clusterList.Items[i].Spec.SyncMode == clusterv1alpha1.Pull && s.disableSchedulerEstimatorInPullMode { continue } - if err = estimatorclient.EstablishConnection(s.KubeClient, clusterList.Items[i].Name, s.schedulerEstimatorCache, s.schedulerEstimatorServicePrefix, s.schedulerEstimatorPort); err != nil { + if err = estimatorclient.EstablishConnection(s.KubeClient, clusterList.Items[i].Name, s.schedulerEstimatorCache, s.schedulerEstimatorServicePrefix, s.grpcConfig); err != nil { klog.Error(err) } } diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 584eb226e224..1fc8badbd489 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -53,16 +53,16 @@ func TestCreateScheduler(t *testing.T) { name: "scheduler with enableSchedulerEstimator enabled", opts: []Option{ WithEnableSchedulerEstimator(true), - WithSchedulerEstimatorPort(port), + WithGRPC(port, "", "", "", false), }, enableSchedulerEstimator: true, schedulerEstimatorPort: port, }, { - name: "scheduler with enableSchedulerEstimator disabled, WithSchedulerEstimatorPort enabled", + name: "scheduler with enableSchedulerEstimator disabled, WithGRPC enabled", opts: []Option{ WithEnableSchedulerEstimator(false), - WithSchedulerEstimatorPort(port), + WithGRPC(port, "", "", "", false), }, enableSchedulerEstimator: false, }, @@ -79,8 +79,8 @@ func TestCreateScheduler(t *testing.T) { t.Errorf("unexpected enableSchedulerEstimator want %v, got %v", tc.enableSchedulerEstimator, sche.enableSchedulerEstimator) } - if tc.schedulerEstimatorPort != sche.schedulerEstimatorPort { - t.Errorf("unexpected schedulerEstimatorPort want %v, got %v", tc.schedulerEstimatorPort, sche.schedulerEstimatorPort) + if tc.enableSchedulerEstimator && tc.schedulerEstimatorPort != sche.grpcConfig.ServerPort { + t.Errorf("unexpected schedulerEstimatorPort want %v, got %v", tc.schedulerEstimatorPort, sche.grpcConfig.ServerPort) } }) } diff --git a/pkg/util/dial.go b/pkg/util/dial.go deleted file mode 100644 index ab12bcda5ee1..000000000000 --- a/pkg/util/dial.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2021 The Karmada Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "context" - "fmt" - "time" - - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -// Dial establishes the gRPC communication. -func Dial(path string, timeout time.Duration) (*grpc.ClientConn, error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - opts := []grpc.DialOption{ - grpc.WithBlock(), - grpc.WithTransportCredentials(insecure.NewCredentials()), - } - cc, err := grpc.DialContext(ctx, path, opts...) - if err != nil { - return nil, fmt.Errorf("dial %s error: %v", path, err) - } - - return cc, nil -} diff --git a/pkg/util/grpc/grpc.go b/pkg/util/grpc/grpc.go new file mode 100644 index 000000000000..b7a526cbf6e4 --- /dev/null +++ b/pkg/util/grpc/grpc.go @@ -0,0 +1,131 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package grpc + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "os" + "time" + + "google.golang.org/grpc" + grpccredentials "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" +) + +// Config the config of GRPC connection. +type Config struct { + // InsecureSkipVerify controls whether a client verifies the server's + // certificate chain and host name. If InsecureSkipVerify is true, crypto/tls + // accepts any certificate presented by the server and any host name in that + // certificate. In this mode, TLS is susceptible to machine-in-the-middle + // attacks unless custom verification is used. This should be used only for + // testing or in combination with VerifyConnection or VerifyPeerCertificate. + InsecureSkipVerify bool + // When this is set, server will check all incoming HTTPS requests for a client certificate signed by the trusted CA, + // requests that don’t supply a valid client certificate will fail. If authentication is enabled, + // the certificate provides credentials for the user name given by the Common Name field. + ClientCertAuth bool + // The secure port on which to serve gRPC. + ServerPort int + // Trusted certificate authority. + TrustedCAFile string + // Certificate used for SSL/TLS connections. + CertFile string + // Key for the certificate. + KeyFile string +} + +// NewServer creates a gRPC server which has no service registered and has not +// started to accept requests yet. +func (g *Config) NewServer() (*grpc.Server, error) { + if g.CertFile == "" || g.KeyFile == "" { + return grpc.NewServer(), nil + } + + cert, err := tls.LoadX509KeyPair(g.CertFile, g.KeyFile) + if err != nil { + return nil, err + } + config := &tls.Config{ + Certificates: []tls.Certificate{cert}, + MinVersion: tls.VersionTLS13, + } + if g.ClientCertAuth { + config.ClientAuth = tls.RequireAndVerifyClientCert + certPool := x509.NewCertPool() + ca, err := os.ReadFile(g.TrustedCAFile) + if err != nil { + return nil, err + } + if ok := certPool.AppendCertsFromPEM(ca); !ok { + return nil, fmt.Errorf("failed to append ca into certPool") + } + config.ClientCAs = certPool + } + + return grpc.NewServer(grpc.Creds(grpccredentials.NewTLS(config))), nil +} + +// DialWithTimeOut creates a client connection to the given target. +func (g *Config) DialWithTimeOut(path string, timeout time.Duration) (*grpc.ClientConn, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + opts := []grpc.DialOption{ + grpc.WithBlock(), + } + + var cred grpccredentials.TransportCredentials + if g.TrustedCAFile == "" && !g.InsecureSkipVerify { + // insecure connection + cred = insecure.NewCredentials() + } else { + // server-side TLS + config := &tls.Config{InsecureSkipVerify: g.InsecureSkipVerify} // nolint:gosec // G402: TLS InsecureSkipVerify may be true. + if g.TrustedCAFile != "" { + certPool := x509.NewCertPool() + ca, err := os.ReadFile(g.TrustedCAFile) + if err != nil { + return nil, err + } + if ok := certPool.AppendCertsFromPEM(ca); !ok { + return nil, fmt.Errorf("failed to append ca certs") + } + config.RootCAs = certPool + } + if g.CertFile != "" && g.KeyFile != "" { + // mutual TLS + certificate, err := tls.LoadX509KeyPair(g.CertFile, g.KeyFile) + if err != nil { + return nil, err + } + config.Certificates = []tls.Certificate{certificate} + } + cred = grpccredentials.NewTLS(config) + } + + opts = append(opts, grpc.WithTransportCredentials(cred)) + cc, err := grpc.DialContext(ctx, path, opts...) + if err != nil { + return nil, fmt.Errorf("dial %s error: %v", path, err) + } + + return cc, nil +}