fix the error when cert-mgr-mode set to kubelet (#359)

cmd/yurthub/app/config/config.go

@@ -40,6 +40,8 @@ type YurtHubConfiguration struct {
 	YurtHubProxyServerDummyAddr string
 	GCFrequency                 int
 	CertMgrMode                 string
+	KubeletRootCAFilePath       string
+	KubeletPairFilePath         string
 	NodeName                    string
 	HeartbeatFailedRetry        int
 	HeartbeatHealthyThreshold   int
@@ -81,6 +83,8 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
 		YurtHubProxyServerDummyAddr: proxyServerDummyAddr,
 		GCFrequency:                 options.GCFrequency,
 		CertMgrMode:                 options.CertMgrMode,
+		KubeletRootCAFilePath:       options.KubeletRootCAFilePath,
+		KubeletPairFilePath:         options.KubeletPairFilePath,
 		NodeName:                    options.NodeName,
 		HeartbeatFailedRetry:        options.HeartbeatFailedRetry,
 		HeartbeatHealthyThreshold:   options.HeartbeatHealthyThreshold,

cmd/yurthub/app/options/options.go

@@ -40,6 +40,8 @@ type YurtHubOptions struct {
 	YurtHubProxyPort          string
 	GCFrequency               int
 	CertMgrMode               string
+	KubeletRootCAFilePath     string
+	KubeletPairFilePath       string
 	NodeName                  string
 	LBMode                    string
 	HeartbeatFailedRetry      int
@@ -64,7 +66,9 @@ func NewYurtHubOptions() *YurtHubOptions {
 		YurtHubProxyPort:          "10261",
 		YurtHubPort:               "10267",
 		GCFrequency:               120,
-		CertMgrMode:               "hubself",
+		CertMgrMode:               util.YurtHubCertificateManagerName,
+		KubeletRootCAFilePath:     util.DefaultKubeletRootCAFilePath,
+		KubeletPairFilePath:       util.DefaultKubeletPairFilePath,
 		LBMode:                    "rr",
 		HeartbeatFailedRetry:      3,
 		HeartbeatHealthyThreshold: 2,
@@ -114,6 +118,8 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&o.YurtHubProxyPort, "proxy-port", o.YurtHubProxyPort, "the port on which to proxy HTTP requests to kube-apiserver")
 	fs.StringVar(&o.ServerAddr, "server-addr", o.ServerAddr, "the address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"")
 	fs.StringVar(&o.CertMgrMode, "cert-mgr-mode", o.CertMgrMode, "the cert manager mode, kubelet: use certificates that belongs to kubelet, hubself: auto generate client cert for hub agent.")
+	fs.StringVar(&o.KubeletRootCAFilePath, "kubelet-ca-file", o.KubeletRootCAFilePath, "the ca file path used by kubelet.")
+	fs.StringVar(&o.KubeletPairFilePath, "kubelet-client-certificate", o.KubeletPairFilePath, "the path of kubelet client certificate file.")
 	fs.IntVar(&o.GCFrequency, "gc-frequency", o.GCFrequency, "the frequency to gc cache in storage(unit: minute).")
 	fs.StringVar(&o.NodeName, "node-name", o.NodeName, "the name of node that runs hub agent")
 	fs.StringVar(&o.LBMode, "lb-mode", o.LBMode, "the mode of load balancer to connect remote servers(rr, priority)")

cmd/yurthub/app/start.go

@@ -28,6 +28,7 @@ import (
 	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/kubelet"
 	"github.com/openyurtio/openyurt/pkg/yurthub/gc"
 	"github.com/openyurtio/openyurt/pkg/yurthub/healthchecker"
+	"github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest"
 	"github.com/openyurtio/openyurt/pkg/yurthub/network"
 	"github.com/openyurtio/openyurt/pkg/yurthub/proxy"
 	"github.com/openyurtio/openyurt/pkg/yurthub/server"
@@ -110,6 +111,14 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
 	healthChecker.Run()
 	trace++
 
+	klog.Infof("%d. new restConfig manager for %s mode", trace, cfg.CertMgrMode)
+	restConfigMgr, err := rest.NewRestConfigManager(cfg, certManager, healthChecker)
+	if err != nil {
+		klog.Errorf("could not new restConfig manager, %v", err)
+		return err
+	}
+	trace++
+
 	klog.Infof("%d. new cache manager with storage wrapper and serializer manager", trace)
 	cacheMgr, err := cachemanager.NewCacheManager(cfg.StorageWrapper, cfg.SerializerManager)
 	if err != nil {
@@ -119,7 +128,7 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
 	trace++
 
 	klog.Infof("%d. new gc manager for node %s, and gc frequency is a random time between %d min and %d min", trace, cfg.NodeName, cfg.GCFrequency, 3*cfg.GCFrequency)
-	gcMgr, err := gc.NewGCManager(cfg, transportManager, stopCh)
+	gcMgr, err := gc.NewGCManager(cfg, restConfigMgr, stopCh)
 	if err != nil {
 		klog.Errorf("could not new gc manager, %v", err)
 		return err

pkg/yurtctl/util/edgenode/common.go

@@ -68,6 +68,10 @@ spec:
     hostPath:
       path: /etc/kubernetes
       type: Directory
+  - name: pem-dir
+    hostPath:
+      path: /var/lib/kubelet/pki
+      type: Directory
   containers:
   - name: yurt-hub
     image: __yurthub_image__
@@ -77,6 +81,8 @@ spec:
       mountPath: /var/lib/yurthub
     - name: kubernetes
       mountPath: /etc/kubernetes
+    - name: pem-dir
+      mountPath: /var/lib/kubelet/pki
     command:
     - yurthub
     - --v=2

pkg/yurthub/certificate/hubself/cert_mgr.go

@@ -50,38 +50,39 @@ import (
 )
 
 const (
-	CertificateManagerName  = "hubself"
-	HubName                 = "yurthub"
-	HubRootDir              = "/var/lib/"
-	HubPkiDirName           = "pki"
-	HubCaFileName           = "ca.crt"
-	HubConfigFileName       = "%s.conf"
-	BootstrapConfigFileName = "bootstrap-hub.conf"
-	BootstrapUser           = "token-bootstrap-client"
-	DefaultClusterName      = "kubernetes"
-	ClusterInfoName         = "cluster-info"
-	KubeconfigName          = "kubeconfig"
+	hubName                 = "yurthub"
+	hubRootDir              = "/var/lib/"
+	hubPkiDirName           = "pki"
+	hubCaFileName           = "ca.crt"
+	hubConfigFileName       = "%s.conf"
+	bootstrapConfigFileName = "bootstrap-hub.conf"
+	bootstrapUser           = "token-bootstrap-client"
+	defaultClusterName      = "kubernetes"
+	clusterInfoName         = "cluster-info"
+	kubeconfigName          = "kubeconfig"
 )
 
 // Register registers a YurtCertificateManager
 func Register(cmr *hubcert.CertificateManagerRegistry) {
-	cmr.Register(CertificateManagerName, func(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) {
+	cmr.Register(util.YurtHubCertificateManagerName, func(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) {
 		return NewYurtHubCertManager(cfg)
 	})
 }
 
 type yurtHubCertManager struct {
-	remoteServers        []*url.URL
-	bootstrapConfStore   storage.Store
-	hubClientCertManager certificate.Manager
-	hubClientCertPath    string
-	joinToken            string
-	caFile               string
-	nodeName             string
-	rootDir              string
-	hubName              string
-	dialer               *util.Dialer
-	stopCh               chan struct{}
+	remoteServers         []*url.URL
+	bootstrapConfStore    storage.Store
+	hubClientCertManager  certificate.Manager
+	hubClientCertPath     string
+	joinToken             string
+	caFile                string
+	nodeName              string
+	rootDir               string
+	hubName               string
+	kubeletRootCAFilePath string
+	kubeletPairFilePath   string
+	dialer                *util.Dialer
+	stopCh                chan struct{}
 }
 
 // NewYurtHubCertManager new a YurtCertificateManager instance
@@ -90,24 +91,26 @@ func NewYurtHubCertManager(cfg *config.YurtHubConfiguration) (interfaces.YurtCer
 		return nil, fmt.Errorf("hub agent configuration is invalid, could not new hub agent cert manager")
 	}
 
-	hubName := projectinfo.GetHubName()
-	if len(hubName) == 0 {
-		hubName = HubName
+	hn := projectinfo.GetHubName()
+	if len(hn) == 0 {
+		hn = hubName
 	}
 
 	rootDir := cfg.RootDir
 	if len(rootDir) == 0 {
-		rootDir = filepath.Join(HubRootDir, hubName)
+		rootDir = filepath.Join(hubRootDir, hn)
 	}
 
 	ycm := &yurtHubCertManager{
-		remoteServers: cfg.RemoteServers,
-		nodeName:      cfg.NodeName,
-		joinToken:     cfg.JoinToken,
-		rootDir:       rootDir,
-		hubName:       hubName,
-		dialer:        util.NewDialer("hub certificate manager"),
-		stopCh:        make(chan struct{}),
+		remoteServers:         cfg.RemoteServers,
+		nodeName:              cfg.NodeName,
+		joinToken:             cfg.JoinToken,
+		kubeletRootCAFilePath: cfg.KubeletRootCAFilePath,
+		kubeletPairFilePath:   cfg.KubeletPairFilePath,
+		rootDir:               rootDir,
+		hubName:               hn,
+		dialer:                util.NewDialer("hub certificate manager"),
+		stopCh:                make(chan struct{}),
 	}
 
 	return ycm, nil
@@ -185,44 +188,16 @@ func (ycm *yurtHubCertManager) Update(cfg *config.YurtHubConfiguration) error {
 	return nil
 }
 
-// GetRestConfig get rest client config from hub agent conf file.
-func (ycm *yurtHubCertManager) GetRestConfig() *restclient.Config {
-	healthyServer := ycm.remoteServers[0]
-	if healthyServer == nil {
-		klog.Infof("all of remote servers are unhealthy, so return nil for rest config")
-		return nil
-	}
-
-	// certificate expired, rest config can not be used to connect remote server,
-	// so return nil for rest config
-	if ycm.Current() == nil {
-		klog.Infof("certificate expired, so return nil for rest config")
-		return nil
-	}
-
-	hubConfFile := ycm.getHubConfFile()
-	if isExist, _ := util.FileExists(hubConfFile); isExist {
-		cfg, err := util.LoadRESTClientConfig(hubConfFile)
-		if err != nil {
-			klog.Errorf("could not get rest config for %s, %v", hubConfFile, err)
-			return nil
-		}
-
-		// re-fix host connecting healthy server
-		cfg.Host = healthyServer.String()
-		klog.Infof("re-fix hub rest config host successfully with server %s", cfg.Host)
-		return cfg
-	}
-
-	klog.Errorf("%s config file(%s) is not exist", ycm.hubName, hubConfFile)
-	return nil
-}
-
 // GetCaFile returns the path of ca file
 func (ycm *yurtHubCertManager) GetCaFile() string {
 	return ycm.caFile
 }
 
+// GetConfFilePath returns the path of yurtHub config file path
+func (ycm *yurtHubCertManager) GetConfFilePath() string {
+	return ycm.getHubConfFile()
+}
+
 // NotExpired returns hub client cert is expired or not.
 // True: not expired
 // False: expired
@@ -258,13 +233,13 @@ func (ycm *yurtHubCertManager) initCaCert() error {
 	}
 
 	// make sure configMap kube-public/cluster-info in k8s cluster beforehand
-	insecureClusterInfo, err := insecureClient.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(context.Background(), ClusterInfoName, metav1.GetOptions{})
+	insecureClusterInfo, err := insecureClient.CoreV1().ConfigMaps(metav1.NamespacePublic).Get(context.Background(), clusterInfoName, metav1.GetOptions{})
 	if err != nil {
 		klog.Errorf("failed to get cluster-info configmap, %v", err)
 		return err
 	}
 
-	kubeconfigStr, ok := insecureClusterInfo.Data[KubeconfigName]
+	kubeconfigStr, ok := insecureClusterInfo.Data[kubeconfigName]
 	if !ok || len(kubeconfigStr) == 0 {
 		return fmt.Errorf("no kubeconfig in cluster-info configmap of kube-public namespace")
 	}
@@ -300,7 +275,7 @@ func (ycm *yurtHubCertManager) initBootstrap() error {
 	}
 	ycm.bootstrapConfStore = bootstrapConfStore
 
-	contents, err := ycm.bootstrapConfStore.Get(BootstrapConfigFileName)
+	contents, err := ycm.bootstrapConfStore.Get(bootstrapConfigFileName)
 	if err == storage.ErrStorageNotFound {
 		klog.Infof("%s bootstrap conf file does not exist, so create it", ycm.hubName)
 		return ycm.createBootstrapConfFile(ycm.joinToken)
@@ -370,7 +345,7 @@ func (ycm *yurtHubCertManager) getBootstrapClientConfig(healthyServer *url.URL)
 
 	klog.Infof("no join token, so use kubelet config to bootstrap hub")
 	// use kubelet.conf to bootstrap hub agent
-	return util.LoadKubeletRestClientConfig(healthyServer)
+	return util.LoadKubeletRestClientConfig(healthyServer, ycm.kubeletRootCAFilePath, ycm.kubeletPairFilePath)
 }
 
 func (ycm *yurtHubCertManager) generateCertClientFn(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
@@ -462,39 +437,39 @@ func (ycm *yurtHubCertManager) initHubConf() error {
 
 // getPkiDir returns the directory for storing hub agent pki
 func (ycm *yurtHubCertManager) getPkiDir() string {
-	return filepath.Join(ycm.rootDir, HubPkiDirName)
+	return filepath.Join(ycm.rootDir, hubPkiDirName)
 }
 
 // getCaFile returns the path of ca file
 func (ycm *yurtHubCertManager) getCaFile() string {
-	return filepath.Join(ycm.getPkiDir(), HubCaFileName)
+	return filepath.Join(ycm.getPkiDir(), hubCaFileName)
 }
 
 // getBootstrapConfFile returns the path of bootstrap conf file
 func (ycm *yurtHubCertManager) getBootstrapConfFile() string {
-	return filepath.Join(ycm.rootDir, BootstrapConfigFileName)
+	return filepath.Join(ycm.rootDir, bootstrapConfigFileName)
 }
 
 // getHubConfFile returns the path of hub agent conf file.
 func (ycm *yurtHubCertManager) getHubConfFile() string {
-	return filepath.Join(ycm.rootDir, fmt.Sprintf(HubConfigFileName, ycm.hubName))
+	return filepath.Join(ycm.rootDir, fmt.Sprintf(hubConfigFileName, ycm.hubName))
 }
 
 // createBasic create basic client cmd config
 func createBasic(apiServerAddr string, caCert []byte) *clientcmdapi.Config {
-	contextName := fmt.Sprintf("%s@%s", BootstrapUser, DefaultClusterName)
+	contextName := fmt.Sprintf("%s@%s", bootstrapUser, defaultClusterName)
 
 	return &clientcmdapi.Config{
 		Clusters: map[string]*clientcmdapi.Cluster{
-			DefaultClusterName: {
+			defaultClusterName: {
 				Server:                   apiServerAddr,
 				CertificateAuthorityData: caCert,
 			},
 		},
 		Contexts: map[string]*clientcmdapi.Context{
 			contextName: {
-				Cluster:  DefaultClusterName,
-				AuthInfo: BootstrapUser,
+				Cluster:  defaultClusterName,
+				AuthInfo: bootstrapUser,
 			},
 		},
 		AuthInfos:      map[string]*clientcmdapi.AuthInfo{},
@@ -508,7 +483,7 @@ func createInsecureRestClientConfig(remoteServer *url.URL) (*restclient.Config,
 		return nil, fmt.Errorf("no healthy remote server")
 	}
 	cfg := createBasic(remoteServer.String(), []byte{})
-	cfg.Clusters[DefaultClusterName].InsecureSkipTLSVerify = true
+	cfg.Clusters[defaultClusterName].InsecureSkipTLSVerify = true
 
 	restConfig, err := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{}).ClientConfig()
 	if err != nil {
@@ -536,7 +511,7 @@ func createBootstrapConf(apiServerAddr, caFile, joinToken string) *clientcmdapi.
 	}
 
 	cfg := createBasic(apiServerAddr, caCert)
-	cfg.AuthInfos[BootstrapUser] = &clientcmdapi.AuthInfo{Token: joinToken}
+	cfg.AuthInfos[bootstrapUser] = &clientcmdapi.AuthInfo{Token: joinToken}
 
 	return cfg
 }
@@ -559,7 +534,7 @@ func (ycm *yurtHubCertManager) createBootstrapConfFile(joinToken string) error {
 		return err
 	}
 
-	err = ycm.bootstrapConfStore.Update(BootstrapConfigFileName, content)
+	err = ycm.bootstrapConfStore.Update(bootstrapConfigFileName, content)
 	if err != nil {
 		klog.Errorf("could not create bootstrap conf file(%s), %v", ycm.getBootstrapConfFile(), err)
 		return err
@@ -586,21 +561,21 @@ func (ycm *yurtHubCertManager) updateBootstrapConfFile(joinToken string) error {
 		return fmt.Errorf("could not load bootstrap conf file(%s), %v", ycm.getBootstrapConfFile(), err)
 	}
 
-	if curKubeConfig.AuthInfos[BootstrapUser] != nil {
-		if curKubeConfig.AuthInfos[BootstrapUser].Token == joinToken {
+	if curKubeConfig.AuthInfos[bootstrapUser] != nil {
+		if curKubeConfig.AuthInfos[bootstrapUser].Token == joinToken {
 			klog.Infof("join token for %s bootstrap conf file is not changed", ycm.hubName)
 			return nil
 		}
 	}
 
-	curKubeConfig.AuthInfos[BootstrapUser] = &clientcmdapi.AuthInfo{Token: joinToken}
+	curKubeConfig.AuthInfos[bootstrapUser] = &clientcmdapi.AuthInfo{Token: joinToken}
 	content, err := clientcmd.Write(*curKubeConfig)
 	if err != nil {
 		klog.Errorf("could not update bootstrap config into bytes, %v", err)
 		return err
 	}
 
-	err = ycm.bootstrapConfStore.Update(BootstrapConfigFileName, content)
+	err = ycm.bootstrapConfStore.Update(bootstrapConfigFileName, content)
 	if err != nil {
 		klog.Errorf("could not update bootstrap config, %v", err)
 		return err