Skip to content

Commit

Permalink
Adding TLS Certificate Authentication to gRPC
Browse files Browse the repository at this point in the history
Signed-off-by: zhzhuang-zju <m17799853869@163.com>
  • Loading branch information
zhzhuang-zju committed Jun 21, 2024
1 parent 0bc96a2 commit acdd211
Show file tree
Hide file tree
Showing 19 changed files with 330 additions and 85 deletions.
9 changes: 9 additions & 0 deletions artifacts/deploy/karmada-descheduler.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ spec:
- /bin/karmada-descheduler
- --kubeconfig=/etc/kubeconfig
- --bind-address=0.0.0.0
- --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
- --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
- --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key
- --v=4
livenessProbe:
httpGet:
Expand All @@ -38,10 +41,16 @@ spec:
periodSeconds: 15
timeoutSeconds: 5
volumeMounts:
- name: karmada-certs
mountPath: /etc/karmada/pki
readOnly: true
- name: kubeconfig
subPath: kubeconfig
mountPath: /etc/kubeconfig
volumes:
- name: karmada-certs
secret:
secretName: karmada-cert-secret
- name: kubeconfig
secret:
secretName: kubeconfig
9 changes: 9 additions & 0 deletions artifacts/deploy/karmada-scheduler-estimator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ spec:
- /bin/karmada-scheduler-estimator
- --kubeconfig=/etc/{{member_cluster_name}}-kubeconfig
- --cluster-name={{member_cluster_name}}
- --grpc-auth-cert-file=/etc/karmada/pki/karmada.crt
- --grpc-auth-key-file=/etc/karmada/pki/karmada.key
- --grpc-client-ca-file=/etc/karmada/pki/ca.crt
livenessProbe:
httpGet:
path: /healthz
Expand All @@ -37,10 +40,16 @@ spec:
periodSeconds: 15
timeoutSeconds: 5
volumeMounts:
- name: karmada-certs
mountPath: /etc/karmada/pki
readOnly: true
- name: member-kubeconfig
subPath: {{member_cluster_name}}-kubeconfig
mountPath: /etc/{{member_cluster_name}}-kubeconfig
volumes:
- name: karmada-certs
secret:
secretName: karmada-cert-secret
- name: member-kubeconfig
secret:
secretName: {{member_cluster_name}}-kubeconfig
Expand Down
9 changes: 9 additions & 0 deletions artifacts/deploy/karmada-scheduler.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,21 @@ spec:
- --bind-address=0.0.0.0
- --secure-port=10351
- --enable-scheduler-estimator=true
- --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
- --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
- --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key
- --v=4
volumeMounts:
- name: karmada-certs
mountPath: /etc/karmada/pki
readOnly: true
- name: kubeconfig
subPath: kubeconfig
mountPath: /etc/kubeconfig
volumes:
- name: karmada-certs
secret:
secretName: karmada-cert-secret
- name: kubeconfig
secret:
secretName: kubeconfig
12 changes: 12 additions & 0 deletions cmd/descheduler/app/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,14 @@ type Options struct {
SchedulerEstimatorServicePrefix string
// SchedulerEstimatorPort is the port that the accurate scheduler estimator server serves at.
SchedulerEstimatorPort int
// SchedulerEstimatorCertFile SSL certification file used to secure scheduler estimator communication.
SchedulerEstimatorCertFile string
// SchedulerEstimatorKeyFile SSL key file used to secure scheduler estimator communication.
SchedulerEstimatorKeyFile string
// SchedulerEstimatorCaFile SSL Certificate Authority file used to secure scheduler estimator communication.
SchedulerEstimatorCaFile string
// InsecureSkipEstimatorVerify controls whether verifies the grpc server's certificate chain and host name.
InsecureSkipEstimatorVerify bool
// DeschedulingInterval specifies time interval for descheduler to run.
DeschedulingInterval metav1.Duration
// UnschedulableThreshold specifies the period of pod unschedulable condition.
Expand Down Expand Up @@ -99,6 +107,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
fs.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver.")
fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.")
fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.")
fs.StringVar(&o.SchedulerEstimatorCertFile, "scheduler-estimator-cert-file", "", "SSL certification file used to secure scheduler estimator communication.")
fs.StringVar(&o.SchedulerEstimatorKeyFile, "scheduler-estimator-key-file", "", "SSL key file used to secure scheduler estimator communication.")
fs.StringVar(&o.SchedulerEstimatorCaFile, "scheduler-estimator-ca-file", "", "SSL Certificate Authority file used to secure scheduler estimator communication.")
fs.BoolVar(&o.InsecureSkipEstimatorVerify, "insecure-skip-estimator-verify", false, "Controls whether verifies the scheduler estimator's certificate chain and host name.")
fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name")
fs.DurationVar(&o.DeschedulingInterval.Duration, "descheduling-interval", defaultDeschedulingInterval, "Time interval between two consecutive descheduler executions. Setting this value instructs the descheduler to run in a continuous loop at the interval specified.")
fs.DurationVar(&o.UnschedulableThreshold.Duration, "unschedulable-threshold", defaultUnschedulableThreshold, "The period of pod unschedulable condition. This value is considered as a classification standard of unschedulable replicas.")
Expand Down
12 changes: 12 additions & 0 deletions cmd/scheduler-estimator/app/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,14 @@ type Options struct {
SecurePort int
// ServerPort is the port that the server gRPC serves at.
ServerPort int
// InsecureSkipGrpcClientVerify controls whether verifies the grpc client's certificate chain and host name.
InsecureSkipGrpcClientVerify bool
// GrpcAuthCertFile SSL certification file used for grpc SSL/TLS connections.
GrpcAuthCertFile string
// GrpcAuthKeyFile SSL key file used for grpc SSL/TLS connections.
GrpcAuthKeyFile string
// GrpcClientCaFile SSL Certificate Authority file used to verify grpc client certificates on incoming requests.
GrpcClientCaFile string
// ClusterAPIQPS is the QPS to use while talking with cluster kube-apiserver.
ClusterAPIQPS float32
// ClusterAPIBurst is the burst to allow while talking with cluster kube-apiserver.
Expand All @@ -64,6 +72,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&o.ClusterName, "cluster-name", o.ClusterName, "Name of member cluster that the estimator serves for.")
fs.StringVar(&o.BindAddress, "bind-address", defaultBindAddress, "The IP address on which to listen for the --secure-port port.")
fs.IntVar(&o.ServerPort, "server-port", defaultServerPort, "The secure port on which to serve gRPC.")
fs.StringVar(&o.GrpcAuthCertFile, "grpc-auth-cert-file", "", "SSL certification file used for grpc SSL/TLS connections.")
fs.StringVar(&o.GrpcAuthKeyFile, "grpc-auth-key-file", "", "SSL key file used for grpc SSL/TLS connections.")
fs.BoolVar(&o.InsecureSkipGrpcClientVerify, "insecure-skip-grpc-client-verify", false, "If set to true, the estimator will not verify the grpc client's certificate chain and host name. When the relevant certificates are not configured, it will not take effect.")
fs.StringVar(&o.GrpcClientCaFile, "grpc-client-ca-file", "", "SSL Certificate Authority file used to verify grpc client certificates on incoming requests if --client-cert-auth flag is set.")
fs.IntVar(&o.SecurePort, "secure-port", defaultHealthzPort, "The secure port on which to serve HTTPS.")
fs.Float32Var(&o.ClusterAPIQPS, "kube-api-qps", 20.0, "QPS to use while talking with apiserver.")
fs.IntVar(&o.ClusterAPIBurst, "kube-api-burst", 30, "Burst to use while talking with apiserver.")
Expand Down
12 changes: 12 additions & 0 deletions cmd/scheduler/app/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,14 @@ type Options struct {
SchedulerEstimatorServicePrefix string
// SchedulerEstimatorPort is the port that the accurate scheduler estimator server serves at.
SchedulerEstimatorPort int
// InsecureSkipEstimatorVerify controls whether verifies the grpc server's certificate chain and host name.
InsecureSkipEstimatorVerify bool
// SchedulerEstimatorCertFile SSL certification file used to secure scheduler estimator communication.
SchedulerEstimatorCertFile string
// SchedulerEstimatorKeyFile SSL key file used to secure scheduler estimator communication.
SchedulerEstimatorKeyFile string
// SchedulerEstimatorCaFile SSL Certificate Authority file used to secure scheduler estimator communication.
SchedulerEstimatorCaFile string

// EnableEmptyWorkloadPropagation represents whether workload with 0 replicas could be propagated to member clusters.
EnableEmptyWorkloadPropagation bool
Expand Down Expand Up @@ -138,6 +146,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.")
fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name")
fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.")
fs.StringVar(&o.SchedulerEstimatorCertFile, "scheduler-estimator-cert-file", "", "SSL certification file used to secure scheduler estimator communication.")
fs.StringVar(&o.SchedulerEstimatorKeyFile, "scheduler-estimator-key-file", "", "SSL key file used to secure scheduler estimator communication.")
fs.StringVar(&o.SchedulerEstimatorCaFile, "scheduler-estimator-ca-file", "", "SSL Certificate Authority file used to secure scheduler estimator communication.")
fs.BoolVar(&o.InsecureSkipEstimatorVerify, "insecure-skip-estimator-verify", false, "Controls whether verifies the scheduler estimator's certificate chain and host name.")
fs.BoolVar(&o.EnableEmptyWorkloadPropagation, "enable-empty-workload-propagation", false, "Enable workload with replicas 0 to be propagated to member clusters.")
fs.StringSliceVar(&o.Plugins, "plugins", []string{"*"},
fmt.Sprintf("A list of plugins to enable. '*' enables all build-in and customized plugins, 'foo' enables the plugin named 'foo', '*,-foo' disables the plugin named 'foo'.\nAll build-in plugins: %s.", strings.Join(frameworkplugins.NewInTreeRegistry().FactoryNames(), ",")))
Expand Down
2 changes: 1 addition & 1 deletion cmd/scheduler/app/scheduler.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ func run(opts *options.Options, stopChan <-chan struct{}, registryOptions ...Opt
scheduler.WithEnableSchedulerEstimator(opts.EnableSchedulerEstimator),
scheduler.WithDisableSchedulerEstimatorInPullMode(opts.DisableSchedulerEstimatorInPullMode),
scheduler.WithSchedulerEstimatorServicePrefix(opts.SchedulerEstimatorServicePrefix),
scheduler.WithSchedulerEstimatorPort(opts.SchedulerEstimatorPort),
scheduler.WithSchedulerEstimatorConnection(opts.SchedulerEstimatorPort, opts.SchedulerEstimatorCertFile, opts.SchedulerEstimatorKeyFile, opts.SchedulerEstimatorCaFile, opts.InsecureSkipEstimatorVerify),
scheduler.WithSchedulerEstimatorTimeout(opts.SchedulerEstimatorTimeout),
scheduler.WithEnableEmptyWorkloadPropagation(opts.EnableEmptyWorkloadPropagation),
scheduler.WithEnableSchedulerPlugin(opts.Plugins),
Expand Down
18 changes: 18 additions & 0 deletions operator/pkg/controlplane/manifests.go
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,9 @@ spec:
- --secure-port=10351
- --enable-scheduler-estimator=true
- --leader-elect-resource-namespace={{ .SystemNamespace }}
- --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
- --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
- --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key
- --v=4
livenessProbe:
httpGet:
Expand All @@ -199,10 +202,16 @@ spec:
periodSeconds: 15
timeoutSeconds: 5
volumeMounts:
- name: karmada-certs
mountPath: /etc/karmada/pki
readOnly: true
- name: kubeconfig
subPath: kubeconfig
mountPath: /etc/karmada/kubeconfig
volumes:
- name: karmada-certs
secret:
secretName: {{ .KarmadaCertsSecret }}
- name: kubeconfig
secret:
secretName: {{ .KubeconfigSecret }}
Expand Down Expand Up @@ -241,6 +250,9 @@ spec:
- --kubeconfig=/etc/karmada/kubeconfig
- --bind-address=0.0.0.0
- --leader-elect-resource-namespace={{ .SystemNamespace }}
- --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
- --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
- --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key
- --v=4
livenessProbe:
httpGet:
Expand All @@ -252,10 +264,16 @@ spec:
periodSeconds: 15
timeoutSeconds: 5
volumeMounts:
- name: karmada-certs
mountPath: /etc/karmada/pki
readOnly: true
- name: kubeconfig
subPath: kubeconfig
mountPath: /etc/karmada/kubeconfig
volumes:
- name: karmada-certs
secret:
secretName: {{ .KarmadaCertsSecret }}
- name: kubeconfig
secret:
secretName: {{ .KubeconfigSecret }}
Expand Down
31 changes: 19 additions & 12 deletions pkg/descheduler/descheduler.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import (
"github.com/karmada-io/karmada/pkg/util"
"github.com/karmada-io/karmada/pkg/util/fedinformer"
"github.com/karmada-io/karmada/pkg/util/gclient"
"github.com/karmada-io/karmada/pkg/util/grpcconnection"
)

const (
Expand All @@ -65,7 +66,7 @@ type Descheduler struct {

schedulerEstimatorCache *estimatorclient.SchedulerEstimatorCache
schedulerEstimatorServicePrefix string
schedulerEstimatorPort int
schedulerEstimatorClientConfig *grpcconnection.ClientConfig
schedulerEstimatorWorker util.AsyncWorker

unschedulableThreshold time.Duration
Expand All @@ -77,15 +78,21 @@ type Descheduler struct {
func NewDescheduler(karmadaClient karmadaclientset.Interface, kubeClient kubernetes.Interface, opts *options.Options) *Descheduler {
factory := informerfactory.NewSharedInformerFactory(karmadaClient, 0)
desched := &Descheduler{
KarmadaClient: karmadaClient,
KubeClient: kubeClient,
informerFactory: factory,
bindingInformer: factory.Work().V1alpha2().ResourceBindings().Informer(),
bindingLister: factory.Work().V1alpha2().ResourceBindings().Lister(),
clusterInformer: factory.Cluster().V1alpha1().Clusters().Informer(),
clusterLister: factory.Cluster().V1alpha1().Clusters().Lister(),
schedulerEstimatorCache: estimatorclient.NewSchedulerEstimatorCache(),
schedulerEstimatorPort: opts.SchedulerEstimatorPort,
KarmadaClient: karmadaClient,
KubeClient: kubeClient,
informerFactory: factory,
bindingInformer: factory.Work().V1alpha2().ResourceBindings().Informer(),
bindingLister: factory.Work().V1alpha2().ResourceBindings().Lister(),
clusterInformer: factory.Cluster().V1alpha1().Clusters().Informer(),
clusterLister: factory.Cluster().V1alpha1().Clusters().Lister(),
schedulerEstimatorCache: estimatorclient.NewSchedulerEstimatorCache(),
schedulerEstimatorClientConfig: &grpcconnection.ClientConfig{
InsecureSkipServerVerify: opts.InsecureSkipEstimatorVerify,
ServerAuthCAFile: opts.SchedulerEstimatorCaFile,
CertFile: opts.SchedulerEstimatorCertFile,
KeyFile: opts.SchedulerEstimatorKeyFile,
TargetPort: opts.SchedulerEstimatorPort,
},
schedulerEstimatorServicePrefix: opts.SchedulerEstimatorServicePrefix,
unschedulableThreshold: opts.UnschedulableThreshold.Duration,
deschedulingInterval: opts.DeschedulingInterval.Duration,
Expand Down Expand Up @@ -273,7 +280,7 @@ func (d *Descheduler) establishEstimatorConnections() {
return
}
for i := range clusterList.Items {
if err = estimatorclient.EstablishConnection(d.KubeClient, clusterList.Items[i].Name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorPort); err != nil {
if err = estimatorclient.EstablishConnection(d.KubeClient, clusterList.Items[i].Name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorClientConfig); err != nil {
klog.Error(err)
}
}
Expand All @@ -293,7 +300,7 @@ func (d *Descheduler) reconcileEstimatorConnection(key util.QueueKey) error {
}
return err
}
return estimatorclient.EstablishConnection(d.KubeClient, name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorPort)
return estimatorclient.EstablishConnection(d.KubeClient, name, d.schedulerEstimatorCache, d.schedulerEstimatorServicePrefix, d.schedulerEstimatorClientConfig)
}

func (d *Descheduler) recordDescheduleResultEventForResourceBinding(rb *workv1alpha2.ResourceBinding, message string, err error) {
Expand Down
7 changes: 4 additions & 3 deletions pkg/estimator/client/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (

estimatorservice "github.com/karmada-io/karmada/pkg/estimator/service"
"github.com/karmada-io/karmada/pkg/util"
"github.com/karmada-io/karmada/pkg/util/grpcconnection"
"github.com/karmada-io/karmada/pkg/util/names"
)

Expand Down Expand Up @@ -96,19 +97,19 @@ func (c *SchedulerEstimatorCache) GetClient(name string) (estimatorservice.Estim
}

// EstablishConnection establishes a new gRPC connection with the specified cluster scheduler estimator.
func EstablishConnection(kubeClient kubernetes.Interface, name string, estimatorCache *SchedulerEstimatorCache, estimatorServicePrefix string, port int) error {
func EstablishConnection(kubeClient kubernetes.Interface, name string, estimatorCache *SchedulerEstimatorCache, estimatorServicePrefix string, grpcConfig *grpcconnection.ClientConfig) error {
if estimatorCache.IsEstimatorExist(name) {
return nil
}

serverAddr, err := resolveCluster(kubeClient, util.NamespaceKarmadaSystem,
names.GenerateEstimatorServiceName(estimatorServicePrefix, name), int32(port))
names.GenerateEstimatorServiceName(estimatorServicePrefix, name), int32(grpcConfig.TargetPort))
if err != nil {
return err
}

klog.Infof("Start dialing estimator server(%s) of cluster(%s).", serverAddr, name)
cc, err := util.Dial(serverAddr, 5*time.Second)
cc, err := grpcConfig.DialWithTimeOut(serverAddr, 5*time.Second)
if err != nil {
klog.Errorf("Failed to dial cluster(%s): %v.", name, err)
return err
Expand Down
2 changes: 1 addition & 1 deletion pkg/estimator/client/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func resolveCluster(kubeClient kubernetes.Interface, namespace, id string, port
* But the Service resource is defined in Host Kubernetes Cluster. So we cannot get its content here.
* The best thing we can do is just glue host:port together, and try to connect to it.
*/
return net.JoinHostPort(id, fmt.Sprintf("%d", port)), nil
return net.JoinHostPort(fmt.Sprintf("%s.%s.svc.cluster.local", id, namespace), fmt.Sprintf("%d", port)), nil
}

return "", err
Expand Down
Loading

0 comments on commit acdd211

Please sign in to comment.