fix the error when cert-mgr-mode set to kubelet

cmd/yurthub/app/config/config.go

@@ -40,6 +40,8 @@ type YurtHubConfiguration struct {
 	YurtHubProxyServerDummyAddr string
 	GCFrequency                 int
 	CertMgrMode                 string
+	KubeletRootCAFilePath       string
+	KubeletPairFilePath         string
 	NodeName                    string
 	HeartbeatFailedRetry        int
 	HeartbeatHealthyThreshold   int
@@ -81,6 +83,8 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
 		YurtHubProxyServerDummyAddr: proxyServerDummyAddr,
 		GCFrequency:                 options.GCFrequency,
 		CertMgrMode:                 options.CertMgrMode,
+		KubeletRootCAFilePath:       options.KubeletRootCAFilePath,
+		KubeletPairFilePath:         options.KubeletPairFilePath,
 		NodeName:                    options.NodeName,
 		HeartbeatFailedRetry:        options.HeartbeatFailedRetry,
 		HeartbeatHealthyThreshold:   options.HeartbeatHealthyThreshold,

cmd/yurthub/app/options/options.go

@@ -39,6 +39,8 @@ type YurtHubOptions struct {
 	YurtHubProxyPort          string
 	GCFrequency               int
 	CertMgrMode               string
+	KubeletRootCAFilePath     string
+	KubeletPairFilePath       string
 	NodeName                  string
 	LBMode                    string
 	HeartbeatFailedRetry      int
@@ -63,6 +65,8 @@ func NewYurtHubOptions() *YurtHubOptions {
 		YurtHubPort:               "10267",
 		GCFrequency:               120,
 		CertMgrMode:               util.YurtHubCertificateManagerName,
+		KubeletRootCAFilePath:     util.DefaultKubeletRootCAFilePath,
+		KubeletPairFilePath:       util.DefaultKubeletPairFilePath,
 		LBMode:                    "rr",
 		HeartbeatFailedRetry:      3,
 		HeartbeatHealthyThreshold: 2,
@@ -111,6 +115,8 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&o.YurtHubProxyPort, "proxy-port", o.YurtHubProxyPort, "the port on which to proxy HTTP requests to kube-apiserver")
 	fs.StringVar(&o.ServerAddr, "server-addr", o.ServerAddr, "the address of Kubernetes kube-apiserver,the format is: \"server1,server2,...\"")
 	fs.StringVar(&o.CertMgrMode, "cert-mgr-mode", o.CertMgrMode, "the cert manager mode, kubelet: use certificates that belongs to kubelet, hubself: auto generate client cert for hub agent.")
+	fs.StringVar(&o.KubeletRootCAFilePath, "kubelet-ca-file", o.KubeletRootCAFilePath, "the ca file path used by kubelet.")
+	fs.StringVar(&o.KubeletPairFilePath, "kubelet-client-certificate", o.KubeletPairFilePath, "the path of kubelet client certificate file.")
 	fs.IntVar(&o.GCFrequency, "gc-frequency", o.GCFrequency, "the frequency to gc cache in storage(unit: minute).")
 	fs.StringVar(&o.NodeName, "node-name", o.NodeName, "the name of node that runs hub agent")
 	fs.StringVar(&o.LBMode, "lb-mode", o.LBMode, "the mode of load balancer to connect remote servers(rr, priority)")

cmd/yurthub/app/start.go

@@ -28,9 +28,9 @@ import (
 	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/kubelet"
 	"github.com/openyurtio/openyurt/pkg/yurthub/gc"
 	"github.com/openyurtio/openyurt/pkg/yurthub/healthchecker"
+	"github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest"
 	"github.com/openyurtio/openyurt/pkg/yurthub/network"
 	"github.com/openyurtio/openyurt/pkg/yurthub/proxy"
-	"github.com/openyurtio/openyurt/pkg/yurthub/restconfig"
 	"github.com/openyurtio/openyurt/pkg/yurthub/server"
 	"github.com/openyurtio/openyurt/pkg/yurthub/transport"
 
@@ -112,7 +112,7 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
 	trace++
 
 	klog.Infof("%d. new restConfig manager for %s mode", trace, cfg.CertMgrMode)
-	restConfigMgr, err := restconfig.NewRestConfigManager(cfg, certManager, healthChecker)
+	restConfigMgr, err := rest.NewRestConfigManager(cfg, certManager, healthChecker)
 	if err != nil {
 		klog.Errorf("could not new restConfig manager, %v", err)
 		return err

pkg/yurthub/certificate/hubself/cert_mgr.go

@@ -50,7 +50,7 @@ import (
 )
 
 const (
-	yurtHubName             = "yurthub"
+	hubName                 = "yurthub"
 	hubRootDir              = "/var/lib/"
 	hubPkiDirName           = "pki"
 	hubCaFileName           = "ca.crt"
@@ -70,17 +70,19 @@ func Register(cmr *hubcert.CertificateManagerRegistry) {
 }
 
 type yurtHubCertManager struct {
-	remoteServers        []*url.URL
-	bootstrapConfStore   storage.Store
-	hubClientCertManager certificate.Manager
-	hubClientCertPath    string
-	joinToken            string
-	caFile               string
-	nodeName             string
-	rootDir              string
-	hubName              string
-	dialer               *util.Dialer
-	stopCh               chan struct{}
+	remoteServers         []*url.URL
+	bootstrapConfStore    storage.Store
+	hubClientCertManager  certificate.Manager
+	hubClientCertPath     string
+	joinToken             string
+	caFile                string
+	nodeName              string
+	rootDir               string
+	hubName               string
+	kubeletRootCAFilePath string
+	kubeletPairFilePath   string
+	dialer                *util.Dialer
+	stopCh                chan struct{}
 }
 
 // NewYurtHubCertManager new a YurtCertificateManager instance
@@ -89,24 +91,26 @@ func NewYurtHubCertManager(cfg *config.YurtHubConfiguration) (interfaces.YurtCer
 		return nil, fmt.Errorf("hub agent configuration is invalid, could not new hub agent cert manager")
 	}
 
-	hubName := projectinfo.GetHubName()
-	if len(hubName) == 0 {
-		hubName = yurtHubName
+	hn := projectinfo.GetHubName()
+	if len(hn) == 0 {
+		hn = hubName
 	}
 
 	rootDir := cfg.RootDir
 	if len(rootDir) == 0 {
-		rootDir = filepath.Join(hubRootDir, hubName)
+		rootDir = filepath.Join(hubRootDir, hn)
 	}
 
 	ycm := &yurtHubCertManager{
-		remoteServers: cfg.RemoteServers,
-		nodeName:      cfg.NodeName,
-		joinToken:     cfg.JoinToken,
-		rootDir:       rootDir,
-		hubName:       hubName,
-		dialer:        util.NewDialer("hub certificate manager"),
-		stopCh:        make(chan struct{}),
+		remoteServers:         cfg.RemoteServers,
+		nodeName:              cfg.NodeName,
+		joinToken:             cfg.JoinToken,
+		kubeletRootCAFilePath: cfg.KubeletRootCAFilePath,
+		kubeletPairFilePath:   cfg.KubeletPairFilePath,
+		rootDir:               rootDir,
+		hubName:               hn,
+		dialer:                util.NewDialer("hub certificate manager"),
+		stopCh:                make(chan struct{}),
 	}
 
 	return ycm, nil
@@ -341,7 +345,7 @@ func (ycm *yurtHubCertManager) getBootstrapClientConfig(healthyServer *url.URL)
 
 	klog.Infof("no join token, so use kubelet config to bootstrap hub")
 	// use kubelet.conf to bootstrap hub agent
-	return util.LoadKubeletRestClientConfig(healthyServer)
+	return util.LoadKubeletRestClientConfig(healthyServer, ycm.kubeletRootCAFilePath, ycm.kubeletPairFilePath)
 }
 
 func (ycm *yurtHubCertManager) generateCertClientFn(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {

pkg/yurthub/certificate/kubelet/cert_mgr.go

@@ -21,7 +21,6 @@ import (
 	"crypto/x509"
 	"fmt"
 	"net/url"
-	"path/filepath"
 	"sync"
 	"time"
 
@@ -35,14 +34,13 @@ import (
 )
 
 const (
-	defaultPairFile    = "kubelet-client-current.pem"
 	certVerifyDuration = 30 * time.Minute
 )
 
 // Register registers a YurtCertificateManager
 func Register(cmr *certificate.CertificateManagerRegistry) {
 	cmr.Register(util.KubeletCertificateManagerName, func(cfg *config.YurtHubConfiguration) (interfaces.YurtCertificateManager, error) {
-		return NewKubeletCertManager(cfg, 0, "")
+		return NewKubeletCertManager(cfg, 0)
 	})
 }
 
@@ -58,9 +56,9 @@ type kubeletCertManager struct {
 }
 
 // NewKubeletCertManager creates a YurtCertificateManager
-func NewKubeletCertManager(cfg *config.YurtHubConfiguration, period time.Duration, certDir string) (interfaces.YurtCertificateManager, error) {
+func NewKubeletCertManager(cfg *config.YurtHubConfiguration, period time.Duration) (interfaces.YurtCertificateManager, error) {
 	var cert *tls.Certificate
-	var pairFile string
+	pairFile := cfg.KubeletPairFilePath
 	if cfg == nil || len(cfg.RemoteServers) == 0 {
 		return nil, fmt.Errorf("hub configuration is invalid")
 	}
@@ -69,12 +67,6 @@ func NewKubeletCertManager(cfg *config.YurtHubConfiguration, period time.Duratio
 		period = certVerifyDuration
 	}
 
-	if len(certDir) == 0 {
-		pairFile = util.DefaultKubeletPairFilePath
-	} else {
-		pairFile = filepath.Join(certDir, defaultPairFile)
-	}
-
 	if pairFileExists, err := util.FileExists(pairFile); err != nil {
 		return nil, err
 	} else if pairFileExists {
@@ -89,7 +81,7 @@ func NewKubeletCertManager(cfg *config.YurtHubConfiguration, period time.Duratio
 		pairFile:           pairFile,
 		cert:               cert,
 		remoteServers:      cfg.RemoteServers,
-		caFile:             util.DefaultKubeletRootCAFilePath,
+		caFile:             cfg.KubeletRootCAFilePath,
 		certVerifyDuration: period,
 		stopCh:             make(chan struct{}),
 	}, nil

pkg/yurthub/certificate/kubelet/cert_mgr_test.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/openyurtio/openyurt/cmd/yurthub/app/config"
+	"github.com/openyurtio/openyurt/pkg/yurthub/util"
 )
 
 var storeCertData = newCertificateData(`-----BEGIN CERTIFICATE-----
@@ -138,10 +139,12 @@ func TestCurrent(t *testing.T) {
 
 	u, _ := url.Parse("http://127.0.0.1:8080")
 	cfg := &config.YurtHubConfiguration{
-		RemoteServers: []*url.URL{u},
+		RemoteServers:         []*url.URL{u},
+		KubeletPairFilePath:   pairFile,
+		KubeletRootCAFilePath: util.DefaultKubeletRootCAFilePath,
 	}
 	// new kubelet cert manager
-	m, err := NewKubeletCertManager(cfg, 10*time.Second, dir)
+	m, err := NewKubeletCertManager(cfg, 10*time.Second)
 	if err != nil {
 		t.Errorf("failed to new kubelet cert manager, %v", err)
 	}

pkg/yurthub/gc/gc.go

@@ -23,7 +23,7 @@ import (
 
 	"github.com/openyurtio/openyurt/cmd/yurthub/app/config"
 	"github.com/openyurtio/openyurt/pkg/yurthub/cachemanager"
-	"github.com/openyurtio/openyurt/pkg/yurthub/restconfig"
+	"github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest"
 	"github.com/openyurtio/openyurt/pkg/yurthub/util"
 
 	v1 "k8s.io/api/core/v1"
@@ -42,15 +42,15 @@ var (
 // GCManager is responsible for cleanup garbage of yurthub
 type GCManager struct {
 	store             cachemanager.StorageWrapper
-	restConfigManager *restconfig.RestConfigManager
+	restConfigManager *rest.RestConfigManager
 	nodeName          string
 	eventsGCFrequency time.Duration
 	lastTime          time.Time
 	stopCh            <-chan struct{}
 }
 
 // NewGCManager creates a *GCManager object
-func NewGCManager(cfg *config.YurtHubConfiguration, restConfigManager *restconfig.RestConfigManager, stopCh <-chan struct{}) (*GCManager, error) {
+func NewGCManager(cfg *config.YurtHubConfiguration, restConfigManager *rest.RestConfigManager, stopCh <-chan struct{}) (*GCManager, error) {
 	gcFrequency := cfg.GCFrequency
 	if gcFrequency == 0 {
 		gcFrequency = defaultEventGcInterval

pkg/yurthub/restconfig/restconfig.go → pkg/yurthub/kubernetes/rest/config.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restconfig
+package rest
 
 import (
 	"net/url"
@@ -30,19 +30,23 @@ import (
 )
 
 type RestConfigManager struct {
-	remoteServers []*url.URL
-	certMgrMode   string
-	checker       healthchecker.HealthChecker
-	certManager   interfaces.YurtCertificateManager
+	remoteServers         []*url.URL
+	certMgrMode           string
+	kubeletRootCAFilePath string
+	kubeletPairFilePath   string
+	checker               healthchecker.HealthChecker
+	certManager           interfaces.YurtCertificateManager
 }
 
 // NewRestConfigManager creates a *RestConfigManager object
 func NewRestConfigManager(cfg *config.YurtHubConfiguration, certMgr interfaces.YurtCertificateManager, healthChecker healthchecker.HealthChecker) (*RestConfigManager, error) {
 	mgr := &RestConfigManager{
-		remoteServers: cfg.RemoteServers,
-		certMgrMode:   cfg.CertMgrMode,
-		checker:       healthChecker,
-		certManager:   certMgr,
+		remoteServers:         cfg.RemoteServers,
+		certMgrMode:           cfg.CertMgrMode,
+		kubeletRootCAFilePath: cfg.KubeletRootCAFilePath,
+		kubeletPairFilePath:   cfg.KubeletPairFilePath,
+		checker:               healthChecker,
+		certManager:           certMgr,
 	}
 	return mgr, nil
 }
@@ -53,19 +57,19 @@ func (rcm *RestConfigManager) GetRestConfig() *rest.Config {
 	case util.YurtHubCertificateManagerName:
 		return rcm.getHubselfRestConfig()
 	case util.KubeletCertificateManagerName:
-		return rcm.getKubeletRestConfig()
+		return rcm.getKubeletRestConfig(rcm.kubeletRootCAFilePath, rcm.kubeletPairFilePath)
 	default:
 		return nil
 	}
 }
 
-func (rcm *RestConfigManager) getKubeletRestConfig() *rest.Config {
+func (rcm *RestConfigManager) getKubeletRestConfig(kubeletRootCAFilePath, kubeletPairFilePath string) *rest.Config {
 	healthyServer := rcm.getHealthyServer()
 	if healthyServer == nil {
 		klog.Infof("all of remote servers are unhealthy, so return nil for rest config")
 		return nil
 	}
-	cfg, err := util.LoadKubeletRestClientConfig(healthyServer)
+	cfg, err := util.LoadKubeletRestClientConfig(healthyServer, kubeletRootCAFilePath, kubeletPairFilePath)
 	if err != nil {
 		klog.Errorf("could not load kubelet rest client config, %v", err)
 		return nil