fix: Separate the certificate configuration to solve the problem of k…

…ubeclient initialization failure

cmd/yurthub/app/config/cert/config.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2022 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cert
+
+import (
+	"fmt"
+	"strings"
+
+	"k8s.io/klog/v2"
+
+	"github.com/openyurtio/openyurt/cmd/yurthub/app/config"
+	"github.com/openyurtio/openyurt/cmd/yurthub/app/options"
+	"github.com/openyurtio/openyurt/pkg/yurthub/certificate"
+	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself"
+	"github.com/openyurtio/openyurt/pkg/yurthub/util"
+)
+
+func Complete(options *options.YurtHubOptions) (*config.CertificateConfiguration, error) {
+	us, err := util.ParseRemoteServers(options.ServerAddr)
+	if err != nil {
+		return nil, err
+	}
+
+	hubCertOrgs := make([]string, 0)
+	if options.YurtHubCertOrganizations != "" {
+		for _, orgStr := range strings.Split(options.YurtHubCertOrganizations, ",") {
+			hubCertOrgs = append(hubCertOrgs, orgStr)
+		}
+	}
+
+	klog.Infof("register cert managers")
+	cmr := certificate.NewCertificateManagerRegistry()
+	hubself.Register(cmr)
+
+	cfg := &config.CertificateConfiguration{
+		CertMgrMode:              options.CertMgrMode,
+		KubeletRootCAFilePath:    options.KubeletRootCAFilePath,
+		KubeletPairFilePath:      options.KubeletPairFilePath,
+		YurtHubCertOrganizations: hubCertOrgs,
+		NodeName:                 options.NodeName,
+		JoinToken:                options.JoinToken,
+		RootDir:                  options.RootDir,
+		RemoteServers:            us,
+	}
+	klog.Infof("create cert manager with %s mode", cfg.CertMgrMode)
+	certManager, err := cmr.New(cfg.CertMgrMode, cfg)
+	if err != nil {
+		return nil, fmt.Errorf("could not create certificate manager, %v", err)
+	}
+	cfg.CertManager = certManager
+	return cfg, nil
+}

cmd/yurthub/app/config/config.go

@@ -41,7 +41,6 @@ import (
 	"sigs.k8s.io/yaml"
 
 	"github.com/openyurtio/openyurt/cmd/yurthub/app/options"
-	"github.com/openyurtio/openyurt/pkg/projectinfo"
 	"github.com/openyurtio/openyurt/pkg/yurthub/cachemanager"
 	"github.com/openyurtio/openyurt/pkg/yurthub/filter"
 	"github.com/openyurtio/openyurt/pkg/yurthub/filter/initializer"
@@ -68,15 +67,11 @@ type YurtHubConfiguration struct {
 	YurtHubProxyServerDummyAddr       string
 	YurtHubProxyServerSecureDummyAddr string
 	GCFrequency                       int
-	CertMgrMode                       string
-	KubeletRootCAFilePath             string
-	KubeletPairFilePath               string
 	NodeName                          string
 	HeartbeatFailedRetry              int
 	HeartbeatHealthyThreshold         int
 	HeartbeatTimeoutSeconds           int
 	MaxRequestInFlight                int
-	JoinToken                         string
 	RootDir                           string
 	EnableProfiling                   bool
 	EnableDummyIf                     bool
@@ -93,6 +88,19 @@ type YurtHubConfiguration struct {
 	FilterChain                       filter.Interface
 }
 
+// CertificateConfiguration represents configuration of yurthub
+type CertificateConfiguration struct {
+	CertMgrMode              string
+	KubeletRootCAFilePath    string
+	KubeletPairFilePath      string
+	YurtHubCertOrganizations []string
+	JoinToken                string
+	NodeName                 string
+	RemoteServers            []*url.URL
+	RootDir                  string
+	CertManager              interface{}
+}
+
 const (
 	// External APIs use this configuration to access Kube-apiserver
 	yurtCfgPath = "/var/lib/yurthub/yurthub.conf"
@@ -108,7 +116,7 @@ const (
 
 // Complete converts *options.YurtHubOptions to *YurtHubConfiguration
 func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
-	us, err := parseRemoteServers(options.ServerAddr)
+	us, err := util.ParseRemoteServers(options.ServerAddr)
 	if err != nil {
 		return nil, err
 	}
@@ -172,21 +180,16 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
 		LBMode:                            options.LBMode,
 		RemoteServers:                     us,
 		YurtHubServerAddr:                 hubServerAddr,
-		YurtHubCertOrganizations:          hubCertOrgs,
 		YurtHubProxyServerAddr:            proxyServerAddr,
 		YurtHubProxyServerSecureAddr:      proxySecureServerAddr,
 		YurtHubProxyServerDummyAddr:       proxyServerDummyAddr,
 		YurtHubProxyServerSecureDummyAddr: proxySecureServerDummyAddr,
 		GCFrequency:                       options.GCFrequency,
-		CertMgrMode:                       options.CertMgrMode,
-		KubeletRootCAFilePath:             options.KubeletRootCAFilePath,
-		KubeletPairFilePath:               options.KubeletPairFilePath,
 		NodeName:                          options.NodeName,
 		HeartbeatFailedRetry:              options.HeartbeatFailedRetry,
 		HeartbeatHealthyThreshold:         options.HeartbeatHealthyThreshold,
 		HeartbeatTimeoutSeconds:           options.HeartbeatTimeoutSeconds,
 		MaxRequestInFlight:                options.MaxRequestInFlight,
-		JoinToken:                         options.JoinToken,
 		RootDir:                           options.RootDir,
 		EnableProfiling:                   options.EnableProfiling,
 		EnableDummyIf:                     options.EnableDummyIf,
@@ -205,36 +208,6 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
 	return cfg, nil
 }
 
-func parseRemoteServers(serverAddr string) ([]*url.URL, error) {
-	if serverAddr == "" {
-		return make([]*url.URL, 0), fmt.Errorf("--server-addr should be set for hub agent")
-	}
-	servers := strings.Split(serverAddr, ",")
-	us := make([]*url.URL, 0, len(servers))
-	remoteServers := make([]string, 0, len(servers))
-	for _, server := range servers {
-		u, err := url.Parse(server)
-		if err != nil {
-			klog.Errorf("failed to parse server address %s, %v", servers, err)
-			return us, err
-		}
-		if u.Scheme == "" {
-			u.Scheme = "https"
-		} else if u.Scheme != "https" {
-			return us, fmt.Errorf("only https scheme is supported for server address(%s)", serverAddr)
-		}
-		us = append(us, u)
-		remoteServers = append(remoteServers, u.String())
-	}
-
-	if len(us) < 1 {
-		return us, fmt.Errorf("no server address is set, can not connect remote server")
-	}
-	klog.Infof("%s would connect remote servers: %s", projectinfo.GetHubName(), strings.Join(remoteServers, ","))
-
-	return us, nil
-}
-
 // createSharedInformers create sharedInformers from the given proxyAddr.
 func createSharedInformers(proxyAddr string) (informers.SharedInformerFactory, yurtinformers.SharedInformerFactory, error) {
 	var kubeConfig *rest.Config

cmd/yurthub/app/start.go

@@ -20,16 +20,14 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
 	"k8s.io/klog/v2"
 
 	"github.com/openyurtio/openyurt/cmd/yurthub/app/config"
+	"github.com/openyurtio/openyurt/cmd/yurthub/app/config/cert"
 	"github.com/openyurtio/openyurt/cmd/yurthub/app/options"
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
 	"github.com/openyurtio/openyurt/pkg/yurthub/cachemanager"
-	"github.com/openyurtio/openyurt/pkg/yurthub/certificate"
-	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/hubself"
+	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces"
 	"github.com/openyurtio/openyurt/pkg/yurthub/gc"
 	"github.com/openyurtio/openyurt/pkg/yurthub/healthchecker"
 	"github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest"
@@ -38,6 +36,8 @@ import (
 	"github.com/openyurtio/openyurt/pkg/yurthub/server"
 	"github.com/openyurtio/openyurt/pkg/yurthub/transport"
 	"github.com/openyurtio/openyurt/pkg/yurthub/util"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
 // NewCmdStartYurtHub creates a *cobra.Command object with default parameters
@@ -61,14 +61,17 @@ func NewCmdStartYurtHub(stopCh <-chan struct{}) *cobra.Command {
 			if err := options.ValidateOptions(yurtHubOptions); err != nil {
 				klog.Fatalf("validate options: %v", err)
 			}
-
+			certCfg, err := cert.Complete(yurtHubOptions)
+			if err != nil {
+				klog.Fatal(err)
+			}
 			yurtHubCfg, err := config.Complete(yurtHubOptions)
 			if err != nil {
 				klog.Fatalf("complete %s configuration error, %v", projectinfo.GetHubName(), err)
 			}
 			klog.Infof("%s cfg: %#+v", projectinfo.GetHubName(), yurtHubCfg)
 
-			if err := Run(yurtHubCfg, stopCh); err != nil {
+			if err := Run(yurtHubCfg, certCfg, stopCh); err != nil {
 				klog.Fatalf("run %s failed, %v", projectinfo.GetHubName(), err)
 			}
 		},
@@ -79,21 +82,10 @@ func NewCmdStartYurtHub(stopCh <-chan struct{}) *cobra.Command {
 }
 
 // Run runs the YurtHubConfiguration. This should never exit
-func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
+func Run(cfg *config.YurtHubConfiguration, certCfg *config.CertificateConfiguration, stopCh <-chan struct{}) error {
 	trace := 1
-	klog.Infof("%d. register cert managers", trace)
-	cmr := certificate.NewCertificateManagerRegistry()
-	hubself.Register(cmr)
-	trace++
-
-	klog.Infof("%d. create cert manager with %s mode", trace, cfg.CertMgrMode)
-	certManager, err := cmr.New(cfg.CertMgrMode, cfg)
-	if err != nil {
-		return fmt.Errorf("could not create certificate manager, %v", err)
-	}
-	trace++
-
 	klog.Infof("%d. new transport manager", trace)
+	certManager := certCfg.CertManager.(interfaces.YurtCertificateManager)
 	transportManager, err := transport.NewTransportManager(certManager, stopCh)
 	if err != nil {
 		return fmt.Errorf("could not new transport manager, %v", err)
@@ -116,8 +108,8 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
 	healthChecker.Run()
 	trace++
 
-	klog.Infof("%d. new restConfig manager for %s mode", trace, cfg.CertMgrMode)
-	restConfigMgr, err := rest.NewRestConfigManager(cfg, certManager, healthChecker)
+	klog.Infof("%d. new restConfig manager for %s mode", trace, certCfg.CertMgrMode)
+	restConfigMgr, err := rest.NewRestConfigManager(certCfg, certManager, healthChecker)
 	if err != nil {
 		return fmt.Errorf("could not new restConfig manager, %v", err)
 	}

pkg/yurthub/certificate/certificate.go

@@ -21,16 +21,16 @@ import (
 	"sync"
 	"time"
 
-	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/klog/v2"
-
 	"github.com/openyurtio/openyurt/cmd/yurthub/app/config"
 	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/interfaces"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog/v2"
 )
 
 // Factory is a function that returns an YurtCertificateManager.
 // The cfg parameter provides the common info for certificate manager
-type Factory func(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error)
+type Factory func(cfg *config.CertificateConfiguration) (interfaces.YurtCertificateManager, error)
 
 // CertificateManagerRegistry is a object for holding all certificate managers
 type CertificateManagerRegistry struct {
@@ -62,7 +62,7 @@ func (cmr *CertificateManagerRegistry) Register(name string, cm Factory) {
 }
 
 // New creates a YurtCertificateManager with specified name of registered certificate manager
-func (cmr *CertificateManagerRegistry) New(name string, cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) {
+func (cmr *CertificateManagerRegistry) New(name string, cfg *config.CertificateConfiguration) (interfaces.YurtCertificateManager, error) {
 	f, found := cmr.registry[name]
 	if !found {
 		return nil, fmt.Errorf("certificate manager %s is not registered", name)

pkg/yurthub/certificate/hubself/cert_mgr.go

@@ -67,7 +67,7 @@ const (
 
 // Register registers a YurtCertificateManager
 func Register(cmr *hubcert.CertificateManagerRegistry) {
-	cmr.Register(util.YurtHubCertificateManagerName, func(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) {
+	cmr.Register(util.YurtHubCertificateManagerName, func(cfg *config.CertificateConfiguration) (interfaces.YurtCertificateManager, error) {
 		return NewYurtHubCertManager(cfg)
 	})
 }
@@ -90,7 +90,7 @@ type yurtHubCertManager struct {
 }
 
 // NewYurtHubCertManager new a YurtCertificateManager instance
-func NewYurtHubCertManager(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) {
+func NewYurtHubCertManager(cfg *config.CertificateConfiguration) (interfaces.YurtCertificateManager, error) {
 	if cfg == nil || len(cfg.NodeName) == 0 || len(cfg.RemoteServers) == 0 {
 		return nil, fmt.Errorf("hub agent configuration is invalid, could not new hub agent cert manager")
 	}
@@ -217,7 +217,7 @@ func (ycm *yurtHubCertManager) ServerHealthy() bool {
 }
 
 // Update update bootstrap conf file by new bearer token.
-func (ycm *yurtHubCertManager) Update(cfg *config.YurtHubConfiguration) error {
+func (ycm *yurtHubCertManager) Update(cfg *config.CertificateConfiguration) error {
 	if cfg == nil {
 		return nil
 	}

pkg/yurthub/certificate/hubself/fake_cert_mgr.go

@@ -133,7 +133,7 @@ func (fyc *fakeYurtHubCertManager) ServerHealthy() bool {
 }
 
 // Update do nothing
-func (fyc *fakeYurtHubCertManager) Update(_ *config.YurtHubConfiguration) error {
+func (fyc *fakeYurtHubCertManager) Update(_ *config.CertificateConfiguration) error {
 	return nil
 }
 

pkg/yurthub/certificate/interfaces/interfaces.go

@@ -25,7 +25,7 @@ import (
 // YurtCertificateManager is responsible for managing node certificate for yurthub
 type YurtCertificateManager interface {
 	certificate.Manager
-	Update(cfg *config.YurtHubConfiguration) error
+	Update(cfg *config.CertificateConfiguration) error
 	GetConfFilePath() string
 	GetCaFile() string
 	NotExpired() bool

pkg/yurthub/kubernetes/rest/config.go

@@ -37,7 +37,7 @@ type RestConfigManager struct {
 }
 
 // NewRestConfigManager creates a *RestConfigManager object
-func NewRestConfigManager(cfg *config.YurtHubConfiguration, certMgr interfaces.YurtCertificateManager, healthChecker healthchecker.HealthChecker) (*RestConfigManager, error) {
+func NewRestConfigManager(cfg *config.CertificateConfiguration, certMgr interfaces.YurtCertificateManager, healthChecker healthchecker.HealthChecker) (*RestConfigManager, error) {
 	mgr := &RestConfigManager{
 		remoteServers: cfg.RemoteServers,
 		certMgrMode:   cfg.CertMgrMode,

pkg/yurthub/kubernetes/rest/config_test.go

@@ -113,7 +113,7 @@ func TestGetRestConfig(t *testing.T) {
 	}
 
 	// set the YurtHubConfiguration
-	cfg := &config.YurtHubConfiguration{
+	cfg := &config.CertificateConfiguration{
 		RootDir:               testDir,
 		RemoteServers:         []*url.URL{u},
 		KubeletRootCAFilePath: caFile,

pkg/yurthub/server/certificate.go

@@ -49,7 +49,7 @@ func updateTokenHandler(certificateMgr interfaces.YurtCertificateManager) http.H
 			return
 		}
 
-		err = certificateMgr.Update(&config.YurtHubConfiguration{JoinToken: joinToken})
+		err = certificateMgr.Update(&config.CertificateConfiguration{JoinToken: joinToken})
 		if err != nil {
 			w.WriteHeader(http.StatusInternalServerError)
 			fmt.Fprintf(w, "could not update bootstrap token, %v", err)

pkg/yurthub/util/util.go

@@ -433,3 +433,33 @@ func CreateKubeConfigFile(kubeClientConfig *rest.Config, kubeconfigPath string)
 	// Marshal to disk
 	return clientcmd.WriteToFile(kubeconfigData, kubeconfigPath)
 }
+
+func ParseRemoteServers(serverAddr string) ([]*url.URL, error) {
+	if serverAddr == "" {
+		return make([]*url.URL, 0), fmt.Errorf("--server-addr should be set for hub agent")
+	}
+	servers := strings.Split(serverAddr, ",")
+	us := make([]*url.URL, 0, len(servers))
+	remoteServers := make([]string, 0, len(servers))
+	for _, server := range servers {
+		u, err := url.Parse(server)
+		if err != nil {
+			klog.Errorf("failed to parse server address %s, %v", servers, err)
+			return us, err
+		}
+		if u.Scheme == "" {
+			u.Scheme = "https"
+		} else if u.Scheme != "https" {
+			return us, fmt.Errorf("only https scheme is supported for server address(%s)", serverAddr)
+		}
+		us = append(us, u)
+		remoteServers = append(remoteServers, u.String())
+	}
+
+	if len(us) < 1 {
+		return us, fmt.Errorf("no server address is set, can not connect remote server")
+	}
+	klog.Infof("%s would connect remote servers: %s", projectinfo.GetHubName(), strings.Join(remoteServers, ","))
+
+	return us, nil
+}