diff --git a/pkg/util/certmanager/pki.go b/pkg/util/certmanager/pki.go index 087e4d4adc4..bce98c5a2e1 100644 --- a/pkg/util/certmanager/pki.go +++ b/pkg/util/certmanager/pki.go @@ -185,3 +185,10 @@ func GenCertPoolUseCA(caFile string) (*x509.CertPool, error) { certPool.AppendCertsFromPEM(caData) return certPool, nil } + +// GenCertPoolUseCAData generates a x509 CertPool based on the given CA data +func GenCertPoolUseCAData(caData []byte) (*x509.CertPool, error) { + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(caData) + return certPool, nil +} diff --git a/pkg/yurthub/certificate/interfaces.go b/pkg/yurthub/certificate/interfaces.go index 0a8c74d7b03..4d0d9a94b88 100644 --- a/pkg/yurthub/certificate/interfaces.go +++ b/pkg/yurthub/certificate/interfaces.go @@ -45,6 +45,7 @@ type YurtClientCertificateManager interface { Stop() UpdateBootstrapConf(joinToken string) error GetHubConfFile() string + GetCAData() []byte GetCaFile() string GetAPIServerClientCert() *tls.Certificate } diff --git a/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go b/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go index 0db01ee19b4..c827437f1d2 100644 --- a/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go +++ b/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go @@ -21,6 +21,7 @@ import ( "crypto/x509" "errors" "fmt" + "os" "time" "k8s.io/klog/v2" @@ -40,6 +41,7 @@ type kubeletCertManager struct { kubeletCAFile string kubeletPemFile string cert *tls.Certificate + caData []byte } func NewKubeletCertManager(kubeConfFile, kubeletCAFile, kubeletPemFile string) (certificate.YurtClientCertificateManager, error) { @@ -50,6 +52,10 @@ func NewKubeletCertManager(kubeConfFile, kubeletCAFile, kubeletPemFile string) ( if exist, _ := util.FileExists(kubeletCAFile); !exist { return nil, KubeletCANotExistErr } + caData, err := os.ReadFile(kubeletCAFile) + if err != nil { + return nil, err + } if exist, _ := util.FileExists(kubeletPemFile); !exist { return nil, KubeletPemNotExistErr @@ -65,6 +71,7 @@ func NewKubeletCertManager(kubeConfFile, kubeletCAFile, kubeletPemFile string) ( kubeletCAFile: kubeletCAFile, kubeletPemFile: kubeletPemFile, cert: cert, + caData: caData, }, nil } @@ -84,6 +91,10 @@ func (kcm *kubeletCertManager) GetHubConfFile() string { return kcm.kubeConfFile } +func (kcm *kubeletCertManager) GetCAData() []byte { + return kcm.caData +} + func (kcm *kubeletCertManager) GetCaFile() string { return kcm.kubeletCAFile } diff --git a/pkg/yurthub/certificate/manager/manager.go b/pkg/yurthub/certificate/manager/manager.go index c06b7dec172..a17f7f34aaf 100644 --- a/pkg/yurthub/certificate/manager/manager.go +++ b/pkg/yurthub/certificate/manager/manager.go @@ -32,7 +32,6 @@ import ( "github.com/openyurtio/openyurt/pkg/yurthub/certificate/kubeletcertificate" hubServerCert "github.com/openyurtio/openyurt/pkg/yurthub/certificate/server" "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" - "github.com/openyurtio/openyurt/pkg/yurthub/util" ) const ( @@ -123,12 +122,8 @@ func (hcm *yurtHubCertManager) Ready() bool { errs = append(errs, apiServerClientCertNotReadyError) } - if exist, err := util.FileExists(hcm.YurtClientCertificateManager.GetCaFile()); !exist { - if err == nil { - errs = append(errs, caCertIsNotReadyError) - } else { - errs = append(errs, err) - } + if len(hcm.YurtClientCertificateManager.GetCAData()) == 0 { + errs = append(errs, caCertIsNotReadyError) } if hcm.GetHubServerCert() == nil { diff --git a/pkg/yurthub/certificate/manager/manager_test.go b/pkg/yurthub/certificate/manager/manager_test.go index fda1226619c..385a0fbcb30 100644 --- a/pkg/yurthub/certificate/manager/manager_test.go +++ b/pkg/yurthub/certificate/manager/manager_test.go @@ -24,16 +24,11 @@ import ( "testing" "time" - "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/tools/clientcmd" - certutil "k8s.io/client-go/util/cert" "github.com/openyurtio/openyurt/cmd/yurthub/app/options" "github.com/openyurtio/openyurt/pkg/projectinfo" - kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig" "github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata" - "github.com/openyurtio/openyurt/pkg/yurthub/util" ) func TestGetHubServerCertFile(t *testing.T) { @@ -108,33 +103,6 @@ func TestReady(t *testing.T) { if mgr.Ready() { return true, nil } - - if exist, err := util.FileExists(mgr.GetCaFile()); !exist { - if err != nil { - t.Logf("could not get ca file(%s), %v", mgr.GetCaFile(), err) - return false, err - } - - if exist, err := util.FileExists(mgr.GetHubConfFile()); err != nil { - t.Logf("could not get hub conf file(%s), %v", mgr.GetHubConfFile(), err) - return false, nil - } else if exist { - t.Logf("%s file already exists, so use it to create ca file", mgr.GetHubConfFile()) - hubKubeConfig, err := clientcmd.LoadFromFile(mgr.GetHubConfFile()) - if err != nil { - return false, err - } - - cluster := kubeconfigutil.GetClusterFromKubeConfig(hubKubeConfig) - if cluster != nil { - if err := certutil.WriteCert(mgr.GetCaFile(), cluster.CertificateAuthorityData); err != nil { - return false, errors.Wrap(err, "couldn't save the CA certificate to disk") - } - } else { - return false, errors.Errorf("couldn't prepare ca.crt(%s) file", mgr.GetCaFile()) - } - } - } return false, nil }) @@ -161,8 +129,14 @@ func TestReady(t *testing.T) { return } newMgr.Start() - if !newMgr.Ready() { - t.Errorf("certificates can not be reused") + err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) { + if mgr.Ready() { + return true, nil + } + return false, nil + }) + if err != nil { + t.Errorf("certificates are not reused, %v", err) } newMgr.Stop() diff --git a/pkg/yurthub/certificate/token/token.go b/pkg/yurthub/certificate/token/token.go index 8a9f49bba4a..b332a23de7b 100644 --- a/pkg/yurthub/certificate/token/token.go +++ b/pkg/yurthub/certificate/token/token.go @@ -80,6 +80,7 @@ type yurtHubClientCertManager struct { joinToken string bootstrapFile string dialer *util.Dialer + caData []byte } // NewYurtHubClientCertManager new a YurtCertificateManager instance @@ -203,11 +204,17 @@ func (ycm *yurtHubClientCertManager) prepareConfigAndCaFile() error { if err := certutil.WriteCert(ycm.GetCaFile(), cluster.CertificateAuthorityData); err != nil { return errors.Wrap(err, "couldn't save the CA certificate to disk") } + ycm.caData = cluster.CertificateAuthorityData } else { return errors.Errorf("couldn't prepare ca.crt(%s) file", ycm.GetCaFile()) } } else { klog.V(2).Infof("%s file already exists, so reuse it", ycm.GetCaFile()) + caData, err := os.ReadFile(ycm.GetCaFile()) + if err != nil { + return err + } + ycm.caData = caData } return nil } @@ -250,11 +257,17 @@ func (ycm *yurtHubClientCertManager) prepareConfigAndCaFile() error { if err := certutil.WriteCert(ycm.GetCaFile(), cluster.CertificateAuthorityData); err != nil { return errors.Wrap(err, "couldn't save the CA certificate to disk") } + ycm.caData = cluster.CertificateAuthorityData } else { return errors.Errorf("couldn't prepare ca.crt(%s) file", ycm.GetCaFile()) } } else { klog.V(2).Infof("%s file already exists, so reuse it", ycm.GetCaFile()) + caData, err := os.ReadFile(ycm.GetCaFile()) + if err != nil { + return err + } + ycm.caData = caData } return nil @@ -284,6 +297,10 @@ func (ycm *yurtHubClientCertManager) getBootstrapConfFile() string { return filepath.Join(ycm.hubRunDir, bootstrapConfigFileName) } +func (ycm *yurtHubClientCertManager) GetCAData() []byte { + return ycm.caData +} + // GetCaFile returns the path of ca file func (ycm *yurtHubClientCertManager) GetCaFile() string { return filepath.Join(ycm.getPkiDir(), hubCaFileName) diff --git a/pkg/yurthub/transport/transport.go b/pkg/yurthub/transport/transport.go index 378e19a4dd4..b112dbc7c95 100644 --- a/pkg/yurthub/transport/transport.go +++ b/pkg/yurthub/transport/transport.go @@ -34,8 +34,8 @@ type CertGetter interface { // GetAPIServerClientCert returns the currently selected certificate, as well as // the associated certificate and key data in PEM format. GetAPIServerClientCert() *tls.Certificate - // Return CA file path. - GetCaFile() string + // GetCAData returns CA file data. + GetCAData() []byte } // Interface is an transport interface for managing clients that used to connecting kube-apiserver @@ -60,13 +60,12 @@ type transportManager struct { // NewTransportManager create a transport interface object. func NewTransportManager(certGetter CertGetter, stopCh <-chan struct{}) (Interface, error) { - caFile := certGetter.GetCaFile() - if len(caFile) == 0 { - return nil, fmt.Errorf("ca cert file was not prepared when new transport") + caData := certGetter.GetCAData() + if len(caData) == 0 { + return nil, fmt.Errorf("ca cert data was not prepared when new transport") } - klog.V(2).Infof("use %s ca cert file to access remote server", caFile) - cfg, err := tlsConfig(certGetter.GetAPIServerClientCert, caFile) + cfg, err := tlsConfig(certGetter.GetAPIServerClientCert, caData) if err != nil { klog.Errorf("could not get tls config when new transport, %v", err) return nil, err @@ -81,7 +80,7 @@ func NewTransportManager(certGetter CertGetter, stopCh <-chan struct{}) (Interfa DialContext: d.DialContext, }) - bearerTLSCfg, err := tlsConfig(nil, caFile) + bearerTLSCfg, err := tlsConfig(nil, caData) if err != nil { klog.Errorf("could not get tls config when new bearer transport, %v", err) return nil, err @@ -151,9 +150,9 @@ func (tm *transportManager) start() { }, 10*time.Second, tm.stopCh) } -func tlsConfig(current func() *tls.Certificate, caFile string) (*tls.Config, error) { +func tlsConfig(current func() *tls.Certificate, caData []byte) (*tls.Config, error) { // generate the TLS configuration based on the latest certificate - rootCert, err := certmanager.GenCertPoolUseCA(caFile) + rootCert, err := certmanager.GenCertPoolUseCAData(caData) if err != nil { klog.Errorf("could not generate a x509 CertPool based on the given CA file, %v", err) return nil, err diff --git a/pkg/yurthub/yurtcoordinator/certmanager/certmanager.go b/pkg/yurthub/yurtcoordinator/certmanager/certmanager.go index 6e914ef80d2..0facbbf407b 100644 --- a/pkg/yurthub/yurtcoordinator/certmanager/certmanager.go +++ b/pkg/yurthub/yurtcoordinator/certmanager/certmanager.go @@ -98,6 +98,7 @@ type CertManager struct { coordinatorCert *tls.Certificate nodeLeaseProxyCert *tls.Certificate store fs.FileSystemOperator + caData []byte // Used for unit test. secret *corev1.Secret @@ -115,6 +116,10 @@ func (c *CertManager) GetNodeLeaseProxyClientCert() *tls.Certificate { return c.nodeLeaseProxyCert } +func (c *CertManager) GetCAData() []byte { + return c.caData +} + func (c *CertManager) GetCaFile() string { return c.GetFilePath(RootCA) } @@ -162,6 +167,7 @@ func (c *CertManager) updateCerts(secret *corev1.Secret) { if err := c.createOrUpdateFile(c.GetFilePath(RootCA), ca); err != nil { klog.Errorf("could not update ca, %v", err) } + c.caData = ca } if cook { diff --git a/pkg/yurthub/yurtcoordinator/coordinator.go b/pkg/yurthub/yurtcoordinator/coordinator.go index 591660d1225..730a59dd5b4 100644 --- a/pkg/yurthub/yurtcoordinator/coordinator.go +++ b/pkg/yurthub/yurtcoordinator/coordinator.go @@ -105,8 +105,8 @@ type coordinator struct { statusInfoChan chan statusInfo isPoolCacheSynced bool certMgr *certmanager.CertManager - // cloudCAFilePath is the file path of cloud kubernetes cluster CA cert. - cloudCAFilePath string + // cloudCAFileData is the file data of cloud kubernetes cluster CA cert. + cloudCAFileData []byte // cloudHealthChecker is health checker of cloud APIServers. It is used to // pick a healthy cloud APIServer to proxy heartbeats. cloudHealthChecker healthchecker.MultipleBackendsHealthChecker @@ -153,7 +153,7 @@ func NewCoordinator( coordinator := &coordinator{ ctx: ctx, - cloudCAFilePath: cfg.CertManager.GetCaFile(), + cloudCAFileData: cfg.CertManager.GetCAData(), cloudHealthChecker: cloudHealthChecker, etcdStorageCfg: etcdStorageCfg, restConfigMgr: restMgr, @@ -425,7 +425,7 @@ func (coordinator *coordinator) newNodeLeaseProxyClient() (coordclientset.LeaseI restCfg := &rest.Config{ Host: healthyCloudServer.String(), TLSClientConfig: rest.TLSClientConfig{ - CAFile: coordinator.cloudCAFilePath, + CAData: coordinator.cloudCAFileData, CertFile: coordinator.certMgr.GetFilePath(certmanager.NodeLeaseProxyClientCert), KeyFile: coordinator.certMgr.GetFilePath(certmanager.NodeLeaseProxyClientKey), },